Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa |
No files matched your search
@@ -6,7 +6,7 @@ jobs:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -37,12 +37,14 @@ jobs:
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: sudo apt install -y --no-install-recommends ffmpeg
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
@@ -57,7 +59,7 @@ jobs:
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
|
||||
@@ -8,7 +8,7 @@ install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.68.0"
|
||||
__version__ = "0.72.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+23
-9
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -198,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -319,8 +327,14 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
+4
-3
@@ -691,9 +691,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
@@ -19,7 +19,7 @@ from functools import wraps, partial
|
||||
import ecdsa
|
||||
import base58
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter
|
||||
from google.protobuf.message import DecodeError
|
||||
from lbry.wallet import (
|
||||
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
|
||||
@@ -290,6 +290,11 @@ class JSONRPCServerType(type):
|
||||
return klass
|
||||
|
||||
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
LBRYnet daemon, a jsonrpc interface to lbry functions
|
||||
@@ -297,6 +302,28 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
callable_methods: dict
|
||||
deprecated_methods: dict
|
||||
|
||||
pending_requests_metric = Gauge(
|
||||
"pending_requests", "Number of running api requests", namespace="daemon_api",
|
||||
labelnames=("method",)
|
||||
)
|
||||
|
||||
requests_count_metric = Counter(
|
||||
"requests_count", "Number of requests received", namespace="daemon_api",
|
||||
labelnames=("method",)
|
||||
)
|
||||
failed_request_metric = Counter(
|
||||
"failed_request_count", "Number of failed requests", namespace="daemon_api",
|
||||
labelnames=("method",)
|
||||
)
|
||||
cancelled_request_metric = Counter(
|
||||
"cancelled_request_count", "Number of cancelled requests", namespace="daemon_api",
|
||||
labelnames=("method",)
|
||||
)
|
||||
response_time_metric = Histogram(
|
||||
"response_time", "Response times", namespace="daemon_api", buckets=HISTOGRAM_BUCKETS,
|
||||
labelnames=("method",)
|
||||
)
|
||||
|
||||
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
|
||||
self.conf = conf
|
||||
self.platform_info = system_info.get_platform()
|
||||
@@ -457,7 +484,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
|
||||
|
||||
self.need_connection_status_refresh.set()
|
||||
self._connection_status_task = self.component_manager.loop.create_task(
|
||||
self.keep_connection_status_up_to_date()
|
||||
@@ -663,20 +689,27 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
JSONRPCError.CODE_INVALID_PARAMS,
|
||||
params_error_message,
|
||||
)
|
||||
|
||||
self.pending_requests_metric.labels(method=function_name).inc()
|
||||
self.requests_count_metric.labels(method=function_name).inc()
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = method(self, *_args, **_kwargs)
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
self.cancelled_request_metric.labels(method=function_name).inc()
|
||||
log.info("cancelled API call for: %s", function_name)
|
||||
raise
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
self.failed_request_metric.labels(method=function_name).inc()
|
||||
log.exception("error handling api request")
|
||||
return JSONRPCError.create_command_exception(
|
||||
command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc()
|
||||
)
|
||||
finally:
|
||||
self.pending_requests_metric.labels(method=function_name).dec()
|
||||
self.response_time_metric.labels(method=function_name).observe(time.perf_counter() - start)
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
@@ -1917,9 +1950,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_list(
|
||||
self, sort=None, reverse=False, comparison=None,
|
||||
wallet_id=None, page=None, page_size=None, **kwargs):
|
||||
async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None,
|
||||
page_size=None, **kwargs):
|
||||
"""
|
||||
List files limited by optional filters
|
||||
|
||||
@@ -1940,17 +1972,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--stream_hash=<stream_hash> : (str) get file with matching stream hash
|
||||
--rowid=<rowid> : (int) get file with matching row id
|
||||
--added_on=<added_on> : (int) get file with matching time of insertion
|
||||
--claim_id=<claim_id> : (str) get file with matching claim id
|
||||
--outpoint=<outpoint> : (str) get file with matching claim outpoint
|
||||
--claim_id=<claim_id> : (str) get file with matching claim id(s)
|
||||
--outpoint=<outpoint> : (str) get file with matching claim outpoint(s)
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id(s)
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le)
|
||||
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in)
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
--wallet_id=<wallet_id> : (str) add purchase receipts from this wallet
|
||||
@@ -1960,6 +1992,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
sort = sort or 'rowid'
|
||||
comparison = comparison or 'eq'
|
||||
|
||||
paginated = paginate_list(
|
||||
self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size
|
||||
)
|
||||
@@ -2237,7 +2270,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
|
||||
if 'is_spent' not in kwargs:
|
||||
if not kwargs.get('is_spent', False):
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(**kwargs)
|
||||
|
||||
@@ -5154,10 +5187,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--comment_ids=<comment_ids> : (str, list) one or more comment_id to hide.
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
|
||||
Returns:
|
||||
(dict) keyed by comment_id, containing success info
|
||||
'<comment_id>': {
|
||||
"hidden": (bool) flag indicating if comment_id was hidden
|
||||
Returns: lists containing the ids comments that are hidden and visible.
|
||||
|
||||
{
|
||||
"hidden": (list) IDs of hidden comments.
|
||||
"visible": (list) IDs of visible comments.
|
||||
}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
@@ -5168,6 +5202,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
comments = await comment_client.jsonrpc_post(
|
||||
self.conf.comment_server, 'get_comments_by_id', comment_ids=comment_ids
|
||||
)
|
||||
comments = comments['items']
|
||||
claim_ids = {comment['claim_id'] for comment in comments}
|
||||
claims = {cid: await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id=cid) for cid in claim_ids}
|
||||
pieces = []
|
||||
|
||||
@@ -108,7 +108,9 @@ def encode_file_doc():
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
}
|
||||
|
||||
|
||||
@@ -307,7 +309,9 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import logging
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self, logger=None):
|
||||
self.runner = None
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
|
||||
async def start(self, interface: str, port: int):
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
+17
-10
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def file_reader(file_path: str):
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -245,7 +252,7 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
for blob_bytes in file_reader(file_path):
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
|
||||
@@ -65,6 +65,7 @@ class ManagedStream:
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'reflector_progress',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
@@ -73,7 +74,8 @@ class ManagedStream:
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt'
|
||||
'finished_write_attempt',
|
||||
'uploading_to_reflector'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
@@ -101,6 +103,8 @@ class ManagedStream:
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.reflector_progress = 0
|
||||
self.uploading_to_reflector = False
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
@@ -339,9 +343,10 @@ class ManagedStream:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -351,12 +356,12 @@ class ManagedStream:
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
open(output_path, 'wb').close()
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
@@ -430,6 +435,7 @@ class ManagedStream:
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
self.uploading_to_reflector = True
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
@@ -445,9 +451,10 @@ class ManagedStream:
|
||||
]
|
||||
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
||||
self.claim_name, self.claim_id)
|
||||
for blob_hash in we_have:
|
||||
for i, blob_hash in enumerate(we_have):
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
@@ -455,6 +462,7 @@ class ManagedStream:
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
self.uploading_to_reflector = False
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
|
||||
@@ -23,6 +23,9 @@ if typing.TYPE_CHECKING:
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.wallet.manager import WalletManager
|
||||
from lbry.wallet.wallet import Wallet
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,6 +49,12 @@ FILTER_FIELDS = [
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
SET_FILTER_FIELDS = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
@@ -53,6 +62,7 @@ COMPARISON_OPERATORS = {
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
'in': lambda a, b: a in b
|
||||
}
|
||||
|
||||
|
||||
@@ -276,15 +286,34 @@ class StreamManager:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
matched = False
|
||||
for set_search, val in compare_sets.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val):
|
||||
streams.append(stream)
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
this_stream = getattr(stream, search)
|
||||
if COMPARISON_OPERATORS[comparison](this_stream, val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -3,6 +3,7 @@ import codecs
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import time
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
@@ -282,3 +283,25 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
|
||||
|
||||
class LockWithMetrics(asyncio.Lock):
|
||||
def __init__(self, acquire_metric, held_time_metric, loop=None):
|
||||
super().__init__(loop=loop)
|
||||
self._acquire_metric = acquire_metric
|
||||
self._lock_held_time_metric = held_time_metric
|
||||
self._lock_acquired_time = None
|
||||
|
||||
async def acquire(self):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await super().acquire()
|
||||
finally:
|
||||
self._lock_acquired_time = time.perf_counter()
|
||||
self._acquire_metric.observe(self._lock_acquired_time - start)
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
+94
-18
@@ -10,6 +10,8 @@ from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
from datetime import date
|
||||
from prometheus_client import Gauge, Counter, Histogram
|
||||
from lbry.utils import LockWithMetrics
|
||||
|
||||
from .bip32 import PubKey
|
||||
from .transaction import Transaction, Output, OutputScript, TXRefImmutable
|
||||
@@ -20,6 +22,10 @@ from .util import date_to_julian_day
|
||||
log = logging.getLogger(__name__)
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReaderProcessState:
|
||||
@@ -64,15 +70,36 @@ else:
|
||||
class AIOSQLite:
|
||||
reader_executor: ReaderExecutorClass
|
||||
|
||||
waiting_writes_metric = Gauge(
|
||||
"waiting_writes_count", "Number of waiting db writes", namespace="daemon_database"
|
||||
)
|
||||
waiting_reads_metric = Gauge(
|
||||
"waiting_reads_count", "Number of waiting db writes", namespace="daemon_database"
|
||||
)
|
||||
write_count_metric = Counter(
|
||||
"write_count", "Number of database writes", namespace="daemon_database"
|
||||
)
|
||||
read_count_metric = Counter(
|
||||
"read_count", "Number of database reads", namespace="daemon_database"
|
||||
)
|
||||
acquire_write_lock_metric = Histogram(
|
||||
f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
held_write_lock_metric = Histogram(
|
||||
f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database",
|
||||
buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
# has to be single threaded as there is no mapping of thread:connection
|
||||
self.writer_executor = ThreadPoolExecutor(max_workers=1)
|
||||
self.writer_connection: Optional[sqlite3.Connection] = None
|
||||
self._closing = False
|
||||
self.query_count = 0
|
||||
self.write_lock = asyncio.Lock()
|
||||
self.write_lock = LockWithMetrics(self.acquire_write_lock_metric, self.held_write_lock_metric)
|
||||
self.writers = 0
|
||||
self.read_ready = asyncio.Event()
|
||||
self.urgent_read_done = asyncio.Event()
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
@@ -88,6 +115,7 @@ class AIOSQLite:
|
||||
)
|
||||
await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer)
|
||||
db.read_ready.set()
|
||||
db.urgent_read_done.set()
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
@@ -112,12 +140,28 @@ class AIOSQLite:
|
||||
read_only=False, fetch_all: bool = False) -> List[dict]:
|
||||
read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone
|
||||
parameters = parameters if parameters is not None else []
|
||||
still_waiting = False
|
||||
urgent_read = False
|
||||
if read_only:
|
||||
while self.writers:
|
||||
await self.read_ready.wait()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
self.waiting_reads_metric.inc()
|
||||
self.read_count_metric.inc()
|
||||
try:
|
||||
while self.writers: # more writes can come in while we are waiting for the first
|
||||
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
|
||||
# throttle the writes if they pile up
|
||||
self.urgent_read_done.clear()
|
||||
urgent_read = True
|
||||
# wait until the running writes have finished
|
||||
await self.read_ready.wait()
|
||||
still_waiting = True
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
finally:
|
||||
if urgent_read:
|
||||
# unthrottle the writers if they had to be throttled
|
||||
self.urgent_read_done.set()
|
||||
self.waiting_reads_metric.dec()
|
||||
if fetch_all:
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchall())
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchone())
|
||||
@@ -135,17 +179,32 @@ class AIOSQLite:
|
||||
return self.run(lambda conn: conn.execute(sql, parameters))
|
||||
|
||||
async def run(self, fun, *args, **kwargs):
|
||||
self.write_count_metric.inc()
|
||||
self.waiting_writes_metric.inc()
|
||||
# it's possible many writes are coming in one after the other, these can
|
||||
# block reader calls for a long time
|
||||
# if the reader waits for the writers to finish and then has to wait for
|
||||
# yet more, it will clear the urgent_read_done event to block more writers
|
||||
# piling on
|
||||
try:
|
||||
await self.urgent_read_done.wait()
|
||||
except Exception as e:
|
||||
self.waiting_writes_metric.dec()
|
||||
raise e
|
||||
self.writers += 1
|
||||
# block readers
|
||||
self.read_ready.clear()
|
||||
async with self.write_lock:
|
||||
try:
|
||||
try:
|
||||
async with self.write_lock:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
|
||||
)
|
||||
finally:
|
||||
self.writers -= 1
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
finally:
|
||||
self.writers -= 1
|
||||
self.waiting_writes_metric.dec()
|
||||
if not self.writers:
|
||||
# unblock the readers once the last enqueued writer finishes
|
||||
self.read_ready.set()
|
||||
|
||||
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
|
||||
self.writer_connection.execute('begin')
|
||||
@@ -160,10 +219,26 @@ class AIOSQLite:
|
||||
log.warning("rolled back")
|
||||
raise
|
||||
|
||||
def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable:
|
||||
return asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
|
||||
)
|
||||
async def run_with_foreign_keys_disabled(self, fun, *args, **kwargs):
|
||||
self.write_count_metric.inc()
|
||||
self.waiting_writes_metric.inc()
|
||||
try:
|
||||
await self.urgent_read_done.wait()
|
||||
except Exception as e:
|
||||
self.waiting_writes_metric.dec()
|
||||
raise e
|
||||
self.writers += 1
|
||||
self.read_ready.clear()
|
||||
try:
|
||||
async with self.write_lock:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
|
||||
)
|
||||
finally:
|
||||
self.writers -= 1
|
||||
self.waiting_writes_metric.dec()
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
|
||||
def __run_transaction_with_foreign_keys_disabled(self,
|
||||
fun: Callable[[sqlite3.Connection, Any, Any], Any],
|
||||
@@ -579,7 +654,7 @@ class Database(SQLiteMixin):
|
||||
return self.db.run(__many)
|
||||
|
||||
async def reserve_outputs(self, txos, is_reserved=True):
|
||||
txoids = ((is_reserved, txo.id) for txo in txos)
|
||||
txoids = [(is_reserved, txo.id) for txo in txos]
|
||||
await self.db.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", txoids)
|
||||
|
||||
async def release_outputs(self, txos):
|
||||
@@ -733,8 +808,9 @@ class Database(SQLiteMixin):
|
||||
include_is_my_input = True
|
||||
constraints['exclude_internal_payments__or'] = {
|
||||
'txo.txo_type__not': TXO_TYPES['other'],
|
||||
'txo.address__not_in': my_addresses,
|
||||
'txi.address__is_null': True,
|
||||
'txi.address__not_in': my_addresses
|
||||
'txi.address__not_in': my_addresses,
|
||||
}
|
||||
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
|
||||
if is_spent:
|
||||
|
||||
+44
-47
@@ -5,7 +5,6 @@ import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
@@ -42,23 +41,22 @@ class Headers:
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.io = None
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.io = BytesIO()
|
||||
if self.path != ':memory:':
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
def _readit():
|
||||
if os.path.exists(self.path):
|
||||
with open(self.path, 'r+b') as header_file:
|
||||
self.io.seek(0)
|
||||
self.io.write(header_file.read())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
@@ -72,10 +70,14 @@ class Headers:
|
||||
await self.get_all_missing_headers()
|
||||
|
||||
async def close(self):
|
||||
if self.executor:
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
self.io.close()
|
||||
if self.io is not None:
|
||||
def _close():
|
||||
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||
with open(self.path, flags) as header_file:
|
||||
header_file.write(self.io.getbuffer())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||
self.io.close()
|
||||
self.io = None
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
@@ -135,28 +137,30 @@ class Headers:
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height):
|
||||
def estimated_timestamp(self, height, try_real_headers=True):
|
||||
if height <= 0:
|
||||
return
|
||||
if try_real_headers and self.has_header(height):
|
||||
offset = height * self.header_size
|
||||
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
|
||||
return self._read(height)
|
||||
|
||||
def _read(self, height, count=1):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size * count)
|
||||
offset = height * self.header_size
|
||||
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
self.io.seek(start * self.header_size, os.SEEK_SET)
|
||||
return self.hash_header(self.io.read(count * self.header_size)).decode()
|
||||
return self.hash_header(self._read(start, count)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
@@ -165,7 +169,7 @@ class Headers:
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if await self.has_header(height):
|
||||
if self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
@@ -179,7 +183,7 @@ class Headers:
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
|
||||
self._write(start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
@@ -189,27 +193,23 @@ class Headers:
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
async def has_header(self, height):
|
||||
def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
def _has_header(height):
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
def _io_checkall():
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
@@ -241,7 +241,7 @@ class Headers:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
|
||||
added += self._write(height, chunk)
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
@@ -306,9 +306,7 @@ class Headers:
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self._read, height, batch_size
|
||||
)
|
||||
headers = self._read(height, batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
@@ -324,12 +322,11 @@ class Headers:
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
def __truncate(at_height):
|
||||
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -87,7 +87,7 @@ class ClientSession(BaseClientSession):
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
@@ -33,13 +33,12 @@ from asyncio import Event, CancelledError
|
||||
import logging
|
||||
import time
|
||||
from contextlib import suppress
|
||||
|
||||
from prometheus_client import Counter, Histogram
|
||||
from lbry.wallet.tasks import TaskGroup
|
||||
|
||||
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
||||
from .jsonrpc import RPCError, ProtocolError
|
||||
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
||||
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT, RESET_CONNECTIONS
|
||||
|
||||
|
||||
class Connector:
|
||||
@@ -372,10 +371,26 @@ class BatchRequest:
|
||||
raise BatchError(self)
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class RPCSession(SessionBase):
|
||||
"""Base class for protocols where a message can lead to a response,
|
||||
for example JSON RPC."""
|
||||
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter(
|
||||
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method", "version")
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||
super().__init__(framer=framer, loop=loop)
|
||||
self.connection = connection or self.default_connection()
|
||||
@@ -388,7 +403,7 @@ class RPCSession(SessionBase):
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self._close()
|
||||
return
|
||||
|
||||
@@ -422,7 +437,7 @@ class RPCSession(SessionBase):
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
message = request.send_result(result)
|
||||
RESPONSE_TIMES.labels(
|
||||
self.RESPONSE_TIMES.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).observe(time.perf_counter() - start)
|
||||
@@ -430,7 +445,7 @@ class RPCSession(SessionBase):
|
||||
await self._send_message(message)
|
||||
if isinstance(result, Exception):
|
||||
self._bump_errors()
|
||||
REQUEST_ERRORS_COUNT.labels(
|
||||
self.REQUEST_ERRORS_COUNT.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).inc()
|
||||
@@ -467,7 +482,7 @@ class RPCSession(SessionBase):
|
||||
async def send_notification(self, method, args=()):
|
||||
"""Send an RPC notification over the network."""
|
||||
message = self.connection.send_notification(Notification(method, args))
|
||||
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
await self._send_message(message)
|
||||
|
||||
def send_batch(self, raise_errors=False):
|
||||
|
||||
@@ -3,6 +3,7 @@ import asyncio
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from typing import Optional
|
||||
from prometheus_client import Gauge, Histogram
|
||||
import lbry
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
@@ -10,7 +11,6 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES, REORG_COUNT
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -129,6 +129,9 @@ class ChainError(Exception):
|
||||
"""Raised on error processing blocks."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class BlockProcessor:
|
||||
"""Process blocks and update the DB state to match.
|
||||
|
||||
@@ -136,6 +139,14 @@ class BlockProcessor:
|
||||
Coordinate backing up in case of chain reorganisations.
|
||||
"""
|
||||
|
||||
block_count_metric = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
block_update_time_metric = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
reorg_count_metric = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
|
||||
def __init__(self, env, db, daemon, notifications):
|
||||
self.env = env
|
||||
self.db = db
|
||||
@@ -199,8 +210,8 @@ class BlockProcessor:
|
||||
cache.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
BLOCK_UPDATE_TIMES.observe(processed_time)
|
||||
self.block_count_metric.set(self.height)
|
||||
self.block_update_time_metric.observe(processed_time)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
@@ -255,7 +266,7 @@ class BlockProcessor:
|
||||
last -= len(raw_blocks)
|
||||
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
REORG_COUNT.inc()
|
||||
self.reorg_count_metric.inc()
|
||||
|
||||
async def reorg_hashes(self, count):
|
||||
"""Return a pair (start, last, hashes) of blocks to back up during a
|
||||
|
||||
@@ -6,11 +6,12 @@ from functools import wraps
|
||||
from pylru import lrucache
|
||||
|
||||
import aiohttp
|
||||
from prometheus_client import Gauge, Histogram
|
||||
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||
from lbry.wallet.rpc import JSONRPC
|
||||
from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT
|
||||
|
||||
|
||||
class DaemonError(Exception):
|
||||
"""Raised when the daemon returns an error in its results."""
|
||||
@@ -24,12 +25,23 @@ class WorkQueueFullError(Exception):
|
||||
"""Internal - when the daemon's work queue is full."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class Daemon:
|
||||
"""Handles connections to a daemon at the given URL."""
|
||||
|
||||
WARMING_UP = -28
|
||||
id_counter = itertools.count()
|
||||
|
||||
lbrycrd_request_time_metric = Histogram(
|
||||
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
lbrycrd_pending_count_metric = Gauge(
|
||||
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
|
||||
labelnames=("method",)
|
||||
)
|
||||
|
||||
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
|
||||
max_retry=4.0):
|
||||
self.coin = coin
|
||||
@@ -129,7 +141,7 @@ class Daemon:
|
||||
while True:
|
||||
try:
|
||||
for method in methods:
|
||||
LBRYCRD_PENDING_COUNT.labels(method=method).inc()
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).inc()
|
||||
result = await self._send_data(data)
|
||||
result = processor(result)
|
||||
if on_good_message:
|
||||
@@ -154,7 +166,7 @@ class Daemon:
|
||||
on_good_message = 'running normally'
|
||||
finally:
|
||||
for method in methods:
|
||||
LBRYCRD_PENDING_COUNT.labels(method=method).dec()
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).dec()
|
||||
await asyncio.sleep(retry)
|
||||
retry = max(min(self.max_retry, retry * 2), self.init_retry)
|
||||
|
||||
@@ -175,7 +187,7 @@ class Daemon:
|
||||
if params:
|
||||
payload['params'] = params
|
||||
result = await self._send(payload, processor)
|
||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
|
||||
async def _send_vector(self, method, params_iterable, replace_errs=False):
|
||||
@@ -200,7 +212,7 @@ class Daemon:
|
||||
result = []
|
||||
if payload:
|
||||
result = await self._send(payload, processor)
|
||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
|
||||
async def _is_rpc_available(self, method):
|
||||
|
||||
@@ -545,11 +545,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
if attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
@@ -596,11 +604,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
if attr == 'tag':
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
import os
|
||||
from aiohttp import web
|
||||
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
|
||||
from lbry import __version__ as version
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server import util
|
||||
import lbry.wallet.server.version as wallet_server_version
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
CPU_COUNT = f"{os.cpu_count()}"
|
||||
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
|
||||
VERSION_INFO.info({
|
||||
'build': BUILD,
|
||||
"commit": COMMIT_HASH,
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': version,
|
||||
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
|
||||
"cpu_count": CPU_COUNT
|
||||
})
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version", ))
|
||||
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
)
|
||||
SQLITE_INTERNAL_ERROR_COUNT = Counter(
|
||||
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
|
||||
)
|
||||
SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE)
|
||||
SQLITE_PENDING_COUNT = Gauge(
|
||||
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
|
||||
)
|
||||
LBRYCRD_REQUEST_TIMES = Histogram(
|
||||
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
LBRYCRD_PENDING_COUNT = Gauge(
|
||||
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
CLIENT_VERSIONS = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
BLOCK_COUNT = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
REORG_COUNT = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self):
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.runner = None
|
||||
|
||||
async def start(self, port: int):
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
@@ -6,7 +6,7 @@ import typing
|
||||
|
||||
import lbry
|
||||
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
|
||||
from lbry.wallet.server.prometheus import PrometheusServer
|
||||
from lbry.prometheus import PrometheusServer
|
||||
|
||||
|
||||
class Notifications:
|
||||
@@ -143,4 +143,4 @@ class Server:
|
||||
async def start_prometheus(self):
|
||||
if not self.prometheus_server and self.env.prometheus_port:
|
||||
self.prometheus_server = PrometheusServer()
|
||||
await self.prometheus_server.start(self.env.prometheus_port)
|
||||
await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port)
|
||||
@@ -20,16 +20,15 @@ from functools import partial
|
||||
from binascii import hexlify
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
|
||||
from lbry.wallet.rpc.framing import NewlineFramer
|
||||
import lbry.wallet.server.version as VERSION
|
||||
|
||||
@@ -119,9 +118,45 @@ class SessionGroup:
|
||||
self.semaphore = asyncio.Semaphore(20)
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Holds global state about all sessions."""
|
||||
|
||||
version_info_metric = Info(
|
||||
'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE
|
||||
)
|
||||
version_info_metric.info({
|
||||
'build': BUILD,
|
||||
"commit": COMMIT_HASH,
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': lbry.__version__,
|
||||
"min_version": util.version_string(VERSION.PROTOCOL_MIN),
|
||||
"cpu_count": os.cpu_count()
|
||||
})
|
||||
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version",))
|
||||
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
|
||||
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
db_operational_error_metric = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
)
|
||||
db_error_metric = Counter(
|
||||
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
|
||||
)
|
||||
executor_time_metric = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE)
|
||||
pending_query_metric = Gauge(
|
||||
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
|
||||
)
|
||||
|
||||
client_version_metric = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
|
||||
shutdown_event: asyncio.Event):
|
||||
env.max_send = max(350000, env.max_send)
|
||||
@@ -677,7 +712,7 @@ class SessionBase(RPCSession):
|
||||
context = {'conn_id': f'{self.session_id}'}
|
||||
self.logger = util.ConnectionLogger(self.logger, context)
|
||||
self.group = self.session_mgr.add_session(self)
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
|
||||
peer_addr_str = self.peer_address_str()
|
||||
self.logger.info(f'{self.kind} {peer_addr_str}, '
|
||||
f'{self.session_mgr.session_count():,d} total')
|
||||
@@ -686,7 +721,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle client disconnection."""
|
||||
super().connection_lost(exc)
|
||||
self.session_mgr.remove_session(self)
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
msg = ''
|
||||
if not self._can_send.is_set():
|
||||
msg += ' whilst paused'
|
||||
@@ -710,7 +745,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle an incoming request. ElectrumX doesn't receive
|
||||
notifications from client sessions.
|
||||
"""
|
||||
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
|
||||
self.session_mgr.request_count_metric.labels(method=request.method, version=self.client_version).inc()
|
||||
if isinstance(request, Request):
|
||||
handler = self.request_handlers.get(request.method)
|
||||
handler = partial(handler, self)
|
||||
@@ -946,7 +981,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def run_in_executor(self, query_name, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
SQLITE_PENDING_COUNT.inc()
|
||||
self.session_mgr.pending_query_metric.inc()
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
@@ -955,18 +990,18 @@ class LBRYElectrumX(SessionBase):
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
SQLITE_INTERRUPT_COUNT.inc()
|
||||
self.session_mgr.self.session_mgr.SQLITE_INTERRUPT_COUNT.inc()
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, error.metrics)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT.inc()
|
||||
self.session_mgr.db_operational_error_metric.inc()
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except Exception:
|
||||
log.exception("dear devs, please handle this exception better")
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, {})
|
||||
SQLITE_INTERNAL_ERROR_COUNT.inc()
|
||||
self.session_mgr.db_error_metric.inc()
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
else:
|
||||
if self.env.track_metrics:
|
||||
@@ -975,8 +1010,8 @@ class LBRYElectrumX(SessionBase):
|
||||
metrics.query_response(start, metrics_data)
|
||||
return base64.b64encode(result).decode()
|
||||
finally:
|
||||
SQLITE_PENDING_COUNT.dec()
|
||||
SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start)
|
||||
self.session_mgr.pending_query_metric.dec()
|
||||
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
@@ -1443,10 +1478,10 @@ class LBRYElectrumX(SessionBase):
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
if self.client_version != client_name[:17]:
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
CLIENT_VERSIONS.labels(version=self.client_version).inc()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
|
||||
self.session_mgr.client_version_metric.labels(version=self.client_version).inc()
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.wallet.server.prometheus import REORG_COUNT
|
||||
|
||||
|
||||
class BlockchainReorganizationTests(CommandTestCase):
|
||||
@@ -16,7 +15,8 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
)
|
||||
|
||||
async def test_reorg(self):
|
||||
REORG_COUNT.set(0)
|
||||
bp = self.conductor.spv_node.server.bp
|
||||
bp.reorg_count_metric.set(0)
|
||||
# invalidate current block, move forward 2
|
||||
self.assertEqual(self.ledger.headers.height, 206)
|
||||
await self.assertBlockHash(206)
|
||||
@@ -26,7 +26,7 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
self.assertEqual(1, REORG_COUNT._samples()[0][2])
|
||||
self.assertEqual(1, bp.reorg_count_metric._samples()[0][2])
|
||||
|
||||
# invalidate current block, move forward 3
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
@@ -36,7 +36,7 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
self.assertEqual(2, REORG_COUNT._samples()[0][2])
|
||||
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
|
||||
|
||||
async def test_reorg_change_claim_height(self):
|
||||
# sanity check
|
||||
|
||||
@@ -503,22 +503,35 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
|
||||
await self.channel_create('@kept-channel')
|
||||
await self.channel_create('@sent-channel', claim_address=address2)
|
||||
await self.wallet_send('2.9', address2)
|
||||
|
||||
# all txos on second wallet
|
||||
received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True)
|
||||
received_payment, received_channel = await self.txo_list(
|
||||
wallet_id=wallet2.id, is_my_input_or_output=True)
|
||||
self.assertEqual('1.0', received_channel['amount'])
|
||||
self.assertFalse(received_channel['is_my_input'])
|
||||
self.assertTrue(received_channel['is_my_output'])
|
||||
self.assertFalse(received_channel['is_internal_transfer'])
|
||||
self.assertEqual('2.9', received_payment['amount'])
|
||||
self.assertFalse(received_payment['is_my_input'])
|
||||
self.assertTrue(received_payment['is_my_output'])
|
||||
self.assertFalse(received_payment['is_internal_transfer'])
|
||||
|
||||
# all txos on default wallet
|
||||
r = await self.txo_list(is_my_input_or_output=True)
|
||||
self.assertEqual(
|
||||
['1.0', '7.947786', '1.0', '8.973893', '10.0'],
|
||||
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
|
||||
[t['amount'] for t in r]
|
||||
)
|
||||
|
||||
sent_channel, change2, kept_channel, change1, initial_funds = r
|
||||
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
|
||||
|
||||
self.assertTrue(sent_payment['is_my_input'])
|
||||
self.assertFalse(sent_payment['is_my_output'])
|
||||
self.assertFalse(sent_payment['is_internal_transfer'])
|
||||
self.assertTrue(change3['is_my_input'])
|
||||
self.assertTrue(change3['is_my_output'])
|
||||
self.assertTrue(change3['is_internal_transfer'])
|
||||
|
||||
self.assertTrue(sent_channel['is_my_input'])
|
||||
self.assertFalse(sent_channel['is_my_output'])
|
||||
@@ -540,31 +553,31 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
# my stuff and stuff i sent excluding "change"
|
||||
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
|
||||
self.assertEqual([sent_channel, kept_channel, initial_funds], r)
|
||||
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
|
||||
|
||||
# my unspent stuff and stuff i sent excluding "change"
|
||||
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
|
||||
self.assertEqual([sent_channel, kept_channel], r)
|
||||
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
|
||||
|
||||
# only "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
|
||||
self.assertEqual([change2, change1], r)
|
||||
self.assertEqual([change3, change2, change1], r)
|
||||
|
||||
# only unspent "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
|
||||
self.assertEqual([change2], r)
|
||||
self.assertEqual([change3], r)
|
||||
|
||||
# only spent "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
|
||||
self.assertEqual([change1], r)
|
||||
self.assertEqual([change2, change1], r)
|
||||
|
||||
# all my unspent stuff
|
||||
r = await self.txo_list(is_my_output=True, is_not_spent=True)
|
||||
self.assertEqual([change2, kept_channel], r)
|
||||
self.assertEqual([change3, kept_channel], r)
|
||||
|
||||
# stuff i sent
|
||||
r = await self.txo_list(is_not_my_output=True)
|
||||
self.assertEqual([sent_channel], r)
|
||||
self.assertEqual([sent_payment, sent_channel], r)
|
||||
|
||||
async def test_txo_plot(self):
|
||||
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
|
||||
@@ -648,12 +661,15 @@ class ClaimCommands(ClaimTestCase):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
|
||||
await self.stream_update(stream_id, title='foo')
|
||||
|
||||
# type filtering
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type='stream')
|
||||
# catch a bug where cli sends is_spent=False by default
|
||||
r = await self.claim_list(claim_type='stream', is_spent=False)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(file1['claim_name'], 'foo')
|
||||
self.assertEqual(file2['claim_name'], 'foo2')
|
||||
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
|
||||
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
|
||||
|
||||
@@ -106,11 +106,16 @@ class MockedCommentServer:
|
||||
return False
|
||||
|
||||
def hide_comments(self, pieces: list):
|
||||
comments_hidden = []
|
||||
hidden = []
|
||||
for p in pieces:
|
||||
if self.hide_comment(**p):
|
||||
comments_hidden.append(p['comment_id'])
|
||||
return {'hidden': comments_hidden}
|
||||
hidden.append(p['comment_id'])
|
||||
|
||||
comment_ids = {c['comment_id'] for c in pieces}
|
||||
return {
|
||||
'hidden': hidden,
|
||||
'visible': list(comment_ids - set(hidden))
|
||||
}
|
||||
|
||||
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
@@ -138,12 +143,19 @@ class MockedCommentServer:
|
||||
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
|
||||
comment = self.comments[self.get_comment_id(comment_id)]
|
||||
return {
|
||||
'channel_id': comment.get('channel_id'),
|
||||
'channel_name': comment.get('channel_name')
|
||||
'channel_id': comment['channel_id'],
|
||||
'channel_name': comment['channel_name'],
|
||||
}
|
||||
|
||||
def get_comments_by_id(self, comment_ids: list):
|
||||
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
|
||||
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
|
||||
return {
|
||||
'page': 1,
|
||||
'page_size': len(comment_ids),
|
||||
'total_pages': 1,
|
||||
'items': comments,
|
||||
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
|
||||
}
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
|
||||
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
|
||||
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
|
||||
for save_blobs in (False, True):
|
||||
await self.setup_blob_manager(save_blobs=save_blobs)
|
||||
# add a blob file
|
||||
@@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
blob.save_verified_blob(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.blob_manager.blob_completed(blob)
|
||||
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
|
||||
|
||||
@@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
write_blob = blob._write_blob
|
||||
write_called_count = 0
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
async def _wrap_write_blob(blob_bytes):
|
||||
nonlocal write_called_count
|
||||
write_called_count += 1
|
||||
write_blob(blob_bytes)
|
||||
await write_blob(blob_bytes)
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
return asyncio.create_task(_wrap_write_blob(blob_bytes))
|
||||
|
||||
blob._write_blob = wrap_write_blob
|
||||
|
||||
writer1 = blob.get_blob_writer(peer_port=1)
|
||||
@@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
|
||||
self.assertEqual(1, write_called_count)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.assertDictEqual({}, blob.writers)
|
||||
|
||||
|
||||
@@ -46,7 +46,9 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
|
||||
self.assertEqual(100, self.stream.reflector_progress)
|
||||
self.assertSetEqual(
|
||||
set(sent),
|
||||
set(map(lambda b: b.blob_hash,
|
||||
|
||||
@@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
await self.ledger.headers.open()
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase):
|
||||
async def test_deserialize(self):
|
||||
self.maxDiff = None
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS)
|
||||
await h.open()
|
||||
await h.connect(0, HEADERS)
|
||||
self.assertEqual(await h.get(0), {
|
||||
'bits': 520159231,
|
||||
'block_height': 0,
|
||||
@@ -52,8 +52,11 @@ class TestHeaders(AsyncioTestCase):
|
||||
self.assertEqual(headers.height, 19)
|
||||
|
||||
async def test_connect_from_middle(self):
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS[:block_bytes(10)])
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
h = Headers(headers_temporary_file)
|
||||
await h.open()
|
||||
self.assertEqual(h.height, 9)
|
||||
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
|
||||
@@ -115,6 +118,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_bounds(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS)
|
||||
self.assertEqual(19, headers.height)
|
||||
with self.assertRaises(IndexError):
|
||||
@@ -126,6 +130,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_repair(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS[:block_bytes(11)])
|
||||
self.assertEqual(10, headers.height)
|
||||
await headers.repair()
|
||||
@@ -147,24 +152,39 @@ class TestHeaders(AsyncioTestCase):
|
||||
await headers.repair(start_height=10)
|
||||
self.assertEqual(19, headers.height)
|
||||
|
||||
def test_do_not_estimate_unconfirmed(self):
|
||||
async def test_do_not_estimate_unconfirmed(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
self.assertIsNone(headers.estimated_timestamp(-1))
|
||||
self.assertIsNone(headers.estimated_timestamp(0))
|
||||
self.assertIsNotNone(headers.estimated_timestamp(1))
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
async def test_dont_estimate_whats_there(self):
|
||||
headers = Headers(':memory:')
|
||||
headers.io.seek(0)
|
||||
headers.io.write(HEADERS)
|
||||
await headers.open()
|
||||
estimated = headers.estimated_timestamp(10)
|
||||
await headers.connect(0, HEADERS)
|
||||
real_time = (await headers.get(10))['timestamp']
|
||||
after_downloading_header_estimated = headers.estimated_timestamp(10)
|
||||
self.assertNotEqual(estimated, after_downloading_header_estimated)
|
||||
self.assertEqual(after_downloading_header_estimated, real_time)
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS)
|
||||
headers = Headers(headers_temporary_file)
|
||||
with self.assertLogs(level='WARN') as cm:
|
||||
await headers.open()
|
||||
await headers.close()
|
||||
self.assertEqual(cm.output, [])
|
||||
headers.io.seek(0)
|
||||
headers.io.truncate()
|
||||
headers.io.write(HEADERS[:block_bytes(10)])
|
||||
headers.io.write(b'ops')
|
||||
headers.io.write(HEADERS[block_bytes(10):])
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.seek(0)
|
||||
headers_file.truncate()
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
headers_file.write(b'ops')
|
||||
headers_file.write(HEADERS[block_bytes(10):])
|
||||
await headers.open()
|
||||
self.assertEqual(
|
||||
cm.output, [
|
||||
@@ -192,6 +212,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
reader_task = asyncio.create_task(reader())
|
||||
await writer()
|
||||
await reader_task
|
||||
await headers.close()
|
||||
|
||||
|
||||
HEADERS = unhexlify(
|
||||
|
||||
@@ -48,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
self.ledger.headers.checkpoints = {}
|
||||
await self.ledger.headers.open()
|
||||
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
Reference in new issue
Block a user