Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a58ebcbc53 | ||
|
|
3921180d77 | ||
|
|
ef36e0311e | ||
|
|
bf4cbe1204 | ||
|
|
28eee4da25 | ||
|
|
933ccf6deb | ||
|
|
9ac41322e5 |
No files matched your search
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.58.0"
|
||||
__version__ = "0.58.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
@@ -49,6 +50,11 @@ class AIOSQLite:
|
||||
def executescript(self, script: str) -> Awaitable:
|
||||
return self.run(lambda conn: conn.executescript(script))
|
||||
|
||||
def create_function(self, name, num_params, func, deterministic=True) -> Awaitable:
|
||||
if sys.version_info < (3, 8):
|
||||
return self.run(lambda conn: conn.create_function(name, num_params, func))
|
||||
return self.run(lambda conn: conn.create_function(name, num_params, func, deterministic=deterministic))
|
||||
|
||||
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
|
||||
parameters = parameters if parameters is not None else []
|
||||
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
|
||||
|
||||
@@ -380,6 +380,7 @@ class RPCSession(SessionBase):
|
||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||
super().__init__(framer=framer, loop=loop)
|
||||
self.connection = connection or self.default_connection()
|
||||
self.client_version = 'unknown'
|
||||
|
||||
async def _receive_messages(self):
|
||||
while not self.is_closing():
|
||||
@@ -419,12 +420,18 @@ class RPCSession(SessionBase):
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
message = request.send_result(result)
|
||||
RESPONSE_TIMES.labels(method=request.method).observe(time.perf_counter() - start)
|
||||
RESPONSE_TIMES.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).observe(time.perf_counter() - start)
|
||||
if message:
|
||||
await self._send_message(message)
|
||||
if isinstance(result, Exception):
|
||||
self._bump_errors()
|
||||
REQUEST_ERRORS_COUNT.labels(method=request.method).inc()
|
||||
REQUEST_ERRORS_COUNT.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).inc()
|
||||
|
||||
def connection_lost(self, exc):
|
||||
# Cancel pending requests and message processing
|
||||
@@ -458,8 +465,8 @@ class RPCSession(SessionBase):
|
||||
async def send_notification(self, method, args=()):
|
||||
"""Send an RPC notification over the network."""
|
||||
message = self.connection.send_notification(Notification(method, args))
|
||||
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
await self._send_message(message)
|
||||
NOTIFICATION_COUNT.labels(method=method).inc()
|
||||
|
||||
def send_batch(self, raise_errors=False):
|
||||
"""Return a BatchRequest. Intended to be used like so:
|
||||
|
||||
@@ -9,6 +9,7 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -188,14 +189,15 @@ class BlockProcessor:
|
||||
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
|
||||
|
||||
if hprevs == chain:
|
||||
start = time.time()
|
||||
start = time.perf_counter()
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
BLOCK_UPDATE_TIMES.observe(processed_time)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'
|
||||
.format(len(blocks), s,
|
||||
time.time() - start))
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
if self._caught_up_event.is_set():
|
||||
await self.notifications.on_block(self.touched, self.height)
|
||||
self.touched = set()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from aiohttp import web
|
||||
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
|
||||
from lbry import __version__ as version
|
||||
@@ -6,7 +7,7 @@ from lbry.wallet.server import util
|
||||
import lbry.wallet.server.version as wallet_server_version
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
CPU_COUNT = f"{os.cpu_count()}"
|
||||
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
|
||||
VERSION_INFO.info({
|
||||
'build': BUILD,
|
||||
@@ -14,16 +15,17 @@ VERSION_INFO.info({
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': version,
|
||||
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
|
||||
"cpu_count": CPU_COUNT
|
||||
})
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE)
|
||||
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version", ))
|
||||
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method",))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method",))
|
||||
labelnames=("method", "version"))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method",))
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method",))
|
||||
labelnames=("method", "version"))
|
||||
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
@@ -45,6 +47,10 @@ CLIENT_VERSIONS = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
BLOCK_COUNT = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
|
||||
@@ -12,7 +12,6 @@ import asyncio
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from functools import lru_cache
|
||||
|
||||
from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
@@ -327,7 +326,7 @@ class SessionManager:
|
||||
return [(session.session_id,
|
||||
session.flags(),
|
||||
session.peer_address_str(for_log=for_log),
|
||||
session.client,
|
||||
session.client_version,
|
||||
session.protocol_version_string(),
|
||||
session.count_pending_items(),
|
||||
session.txs_sent,
|
||||
@@ -631,7 +630,6 @@ class SessionBase(RPCSession):
|
||||
self.kind = kind # 'RPC', 'TCP' etc.
|
||||
self.env = session_mgr.env
|
||||
self.coin = self.env.coin
|
||||
self.client = 'unknown'
|
||||
self.anon_logs = self.env.anon_logs
|
||||
self.txs_sent = 0
|
||||
self.log_me = False
|
||||
@@ -675,15 +673,16 @@ class SessionBase(RPCSession):
|
||||
context = {'conn_id': f'{self.session_id}'}
|
||||
self.logger = util.ConnectionLogger(self.logger, context)
|
||||
self.group = self.session_mgr.add_session(self)
|
||||
SESSIONS_COUNT.inc()
|
||||
self.logger.info(f'{self.kind} {self.peer_address_str()}, '
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
peer_addr_str = self.peer_address_str()
|
||||
self.logger.info(f'{self.kind} {peer_addr_str}, '
|
||||
f'{self.session_mgr.session_count():,d} total')
|
||||
|
||||
def connection_lost(self, exc):
|
||||
"""Handle client disconnection."""
|
||||
super().connection_lost(exc)
|
||||
self.session_mgr.remove_session(self)
|
||||
SESSIONS_COUNT.dec()
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
msg = ''
|
||||
if not self._can_send.is_set():
|
||||
msg += ' whilst paused'
|
||||
@@ -707,7 +706,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle an incoming request. ElectrumX doesn't receive
|
||||
notifications from client sessions.
|
||||
"""
|
||||
REQUESTS_COUNT.labels(method=request.method).inc()
|
||||
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
|
||||
if isinstance(request, Request):
|
||||
handler = self.request_handlers.get(request.method)
|
||||
handler = partial(handler, self)
|
||||
@@ -838,12 +837,6 @@ class LBRYElectrumX(SessionBase):
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYLevelDB = self.bp.db
|
||||
|
||||
@lru_cache(1000)
|
||||
async def _cached_block_headers(_start_height, _count, _b64, _max_size, _cp_height):
|
||||
return await self._block_headers(_start_height, _count, _b64, _max_size, _cp_height)
|
||||
|
||||
self._cached_block_headers = _cached_block_headers
|
||||
|
||||
@classmethod
|
||||
def protocol_min_max_strings(cls):
|
||||
return [util.version_string(ver)
|
||||
@@ -1306,20 +1299,6 @@ class LBRYElectrumX(SessionBase):
|
||||
height: the header's height"""
|
||||
return await self.block_header(height)
|
||||
|
||||
async def _block_headers(self, start_height, count, b64, max_size, cp_height):
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
if count and cp_height:
|
||||
last_height = start_height + count - 1
|
||||
result.update(await self._merkle_proof(cp_height, last_height))
|
||||
return result
|
||||
|
||||
async def block_headers(self, start_height, count, cp_height=0, b64=False):
|
||||
"""Return count concatenated block headers as hex for the main chain;
|
||||
starting at start_height.
|
||||
@@ -1333,10 +1312,18 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
max_size = self.MAX_CHUNK_SIZE
|
||||
count = min(count, max_size)
|
||||
|
||||
if count + start_height < self.db.db_height:
|
||||
return await self._cached_block_headers(start_height, count, b64, max_size, cp_height)
|
||||
return await self._block_headers(start_height, count, b64, max_size, cp_height)
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
if count and cp_height:
|
||||
last_height = start_height + count - 1
|
||||
result.update(await self._merkle_proof(cp_height, last_height))
|
||||
return result
|
||||
|
||||
async def block_get_chunk(self, index):
|
||||
"""Return a chunk of block headers as a hexadecimal string.
|
||||
@@ -1429,6 +1416,7 @@ class LBRYElectrumX(SessionBase):
|
||||
client_name: a string identifying the client
|
||||
protocol_version: the protocol version spoken by the client
|
||||
"""
|
||||
|
||||
if self.sv_seen and self.protocol_tuple >= (1, 4):
|
||||
raise RPCError(BAD_REQUEST, f'server.version already sent')
|
||||
self.sv_seen = True
|
||||
@@ -1440,8 +1428,11 @@ class LBRYElectrumX(SessionBase):
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
self.client = client_name[:17]
|
||||
CLIENT_VERSIONS.labels(version=str(client_name)).inc()
|
||||
if self.client_version != client_name[:17]:
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
CLIENT_VERSIONS.labels(version=self.client_version).inc()
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from lbry.wallet import database
|
||||
|
||||
|
||||
def enable_logging():
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.DEBUG)
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
|
||||
handler.setFormatter(formatter)
|
||||
root.addHandler(handler)
|
||||
|
||||
|
||||
async def initialize_tables(queue, old_file, db):
|
||||
|
||||
await db.executescript("PRAGMA journal_mode=WAL")
|
||||
await db.executescript("PRAGMA case_sensitive_like=true")
|
||||
await db.executescript("CREATE TABLE IF NOT EXISTS claim_with_metadata "
|
||||
"(claimID BLOB NOT NULL PRIMARY KEY, "
|
||||
"name BLOB NOT NULL, "
|
||||
"originalHeight INTEGER NOT NULL, "
|
||||
"updateHeight INTEGER NOT NULL, "
|
||||
"activationHeight INTEGER NOT NULL, "
|
||||
"metadataField1 TEXT)")
|
||||
|
||||
await db.executescript(f"ATTACH '{old_file}' AS crd")
|
||||
|
||||
await db.create_function("NEEDS_UPDATE", 5, lambda id, name, original, update, activation:
|
||||
queue.put_nowait({
|
||||
"claimID": id, "name": name, "originalHeight": original,
|
||||
"updateHeight": update, "activationHeight": activation
|
||||
}))
|
||||
await db.executescript("CREATE TEMP TRIGGER on_claim_insert AFTER INSERT ON crd.claim "
|
||||
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
|
||||
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
|
||||
await db.executescript("CREATE TEMP TRIGGER on_claim_update AFTER UPDATE ON crd.claim "
|
||||
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
|
||||
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
|
||||
await db.executescript("CREATE TEMP TRIGGER on_claim_delete AFTER DELETE ON crd.claim "
|
||||
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(OLD.claimID, OLD.nodeName, 0, 0, 0); END")
|
||||
|
||||
|
||||
async def catch_up_on_missing(queue, db):
|
||||
# items removed from crd are those that are in claim_with_metadata but not in crd
|
||||
# items inserted into crd are those in crd but not in claim_with_metadata
|
||||
# items updated are those that are in both but some columns don't match
|
||||
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID NOT IN"
|
||||
"(SELECT c.claimID FROM crd.claim c)")
|
||||
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
|
||||
"SELECT c.claimID, c.nodeName, c.originalHeight, c.updateHeight, c.activationHeight "
|
||||
"FROM crd.claim c LEFT JOIN claim_with_metadata m ON c.claimID = m.claimID "
|
||||
"WHERE m.claimID IS NULL OR c.nodeName != m.name OR c.updateHeight != m.updateHeight "
|
||||
"OR c.activationHeight != m.activationHeight ON CONFLICT(claimID) DO UPDATE SET "
|
||||
"name = excluded.name, originalHeight = excluded.originalHeight, "
|
||||
"updateHeight = excluded.updateHeight, activationHeight = excluded.activationHeight")
|
||||
|
||||
|
||||
async def remove_claim(claim, db):
|
||||
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID = :claimID", claim)
|
||||
|
||||
|
||||
async def insert_update(claim, db):
|
||||
# TODO: lookup and parse metadata here
|
||||
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
|
||||
"VALUES(:claimID, :name, :originalHeight, :updateHeight, :activationHeight) "
|
||||
"ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, "
|
||||
"originalHeight = excluded.originalHeight, updateHeight = excluded.updateHeight, "
|
||||
"activationHeight = excluded.activationHeight", claim)
|
||||
|
||||
|
||||
async def handle_changes():
|
||||
old_file = os.path.expanduser("~/.lbrycrd/claims.sqlite")
|
||||
new_file = os.path.expanduser("~/.lbrycrd/duplicate.sqlite")
|
||||
db = database.AIOSQLite()
|
||||
db = await db.connect(new_file) # super confusing that you have to use the return value here
|
||||
|
||||
queue = asyncio.Queue()
|
||||
await initialize_tables(queue, old_file, db)
|
||||
await catch_up_on_missing(queue, db)
|
||||
|
||||
while True:
|
||||
claim = await queue.get()
|
||||
if claim.updateHeight <= 0:
|
||||
await remove_claim(claim, db)
|
||||
else:
|
||||
await insert_update(claim, db)
|
||||
|
||||
|
||||
def main():
|
||||
enable_logging()
|
||||
|
||||
try:
|
||||
asyncio.run(handle_changes())
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Process interrupted")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user