Compare commits

..
Author SHA1 Message Date
Jack Robison 9f5f833d00 headers lru cache 2020-02-03 23:54:08 -05:00
7 changed files with 48 additions and 167 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.58.1"
__version__ = "0.58.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
-6
View File
@@ -1,7 +1,6 @@
import logging
import asyncio
import sqlite3
import sys
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
@@ -50,11 +49,6 @@ class AIOSQLite:
def executescript(self, script: str) -> Awaitable:
return self.run(lambda conn: conn.executescript(script))
def create_function(self, name, num_params, func, deterministic=True) -> Awaitable:
if sys.version_info < (3, 8):
return self.run(lambda conn: conn.create_function(name, num_params, func))
return self.run(lambda conn: conn.create_function(name, num_params, func, deterministic=deterministic))
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
+3 -10
View File
@@ -380,7 +380,6 @@ class RPCSession(SessionBase):
def __init__(self, *, framer=None, loop=None, connection=None):
super().__init__(framer=framer, loop=loop)
self.connection = connection or self.default_connection()
self.client_version = 'unknown'
async def _receive_messages(self):
while not self.is_closing():
@@ -420,18 +419,12 @@ class RPCSession(SessionBase):
'internal server error')
if isinstance(request, Request):
message = request.send_result(result)
RESPONSE_TIMES.labels(
method=request.method,
version=self.client_version
).observe(time.perf_counter() - start)
RESPONSE_TIMES.labels(method=request.method).observe(time.perf_counter() - start)
if message:
await self._send_message(message)
if isinstance(result, Exception):
self._bump_errors()
REQUEST_ERRORS_COUNT.labels(
method=request.method,
version=self.client_version
).inc()
REQUEST_ERRORS_COUNT.labels(method=request.method).inc()
def connection_lost(self, exc):
# Cancel pending requests and message processing
@@ -465,8 +458,8 @@ class RPCSession(SessionBase):
async def send_notification(self, method, args=()):
"""Send an RPC notification over the network."""
message = self.connection.send_notification(Notification(method, args))
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
await self._send_message(message)
NOTIFICATION_COUNT.labels(method=method).inc()
def send_batch(self, raise_errors=False):
"""Return a BatchRequest. Intended to be used like so:
+4 -6
View File
@@ -9,7 +9,6 @@ from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
class Prefetcher:
@@ -189,15 +188,14 @@ class BlockProcessor:
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
if hprevs == chain:
start = time.perf_counter()
start = time.time()
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
await self._maybe_flush()
processed_time = time.perf_counter() - start
BLOCK_COUNT.set(self.height)
BLOCK_UPDATE_TIMES.observe(processed_time)
if not self.db.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
self.logger.info('processed {:,d} block{} in {:.1f}s'
.format(len(blocks), s,
time.time() - start))
if self._caught_up_event.is_set():
await self.notifications.on_block(self.touched, self.height)
self.touched = set()
+7 -13
View File
@@ -1,4 +1,3 @@
import os
from aiohttp import web
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
from lbry import __version__ as version
@@ -7,7 +6,7 @@ from lbry.wallet.server import util
import lbry.wallet.server.version as wallet_server_version
NAMESPACE = "wallet_server"
CPU_COUNT = f"{os.cpu_count()}"
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
VERSION_INFO.info({
'build': BUILD,
@@ -15,17 +14,16 @@ VERSION_INFO.info({
"docker_tag": DOCKER_TAG,
'version': version,
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
"cpu_count": CPU_COUNT
})
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version", ))
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE)
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
labelnames=("method",))
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method",))
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
namespace=NAMESPACE, labelnames=("method", "version"))
namespace=NAMESPACE, labelnames=("method",))
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
labelnames=("method", "version"))
labelnames=("method",))
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
@@ -47,10 +45,6 @@ CLIENT_VERSIONS = Counter(
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
BLOCK_COUNT = Gauge(
"block_count", "Number of processed blocks", namespace=NAMESPACE
)
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
class PrometheusServer:
+33 -24
View File
@@ -12,6 +12,7 @@ import asyncio
import logging
import itertools
import collections
from functools import lru_cache
from asyncio import Event, sleep
from collections import defaultdict
@@ -326,7 +327,7 @@ class SessionManager:
return [(session.session_id,
session.flags(),
session.peer_address_str(for_log=for_log),
session.client_version,
session.client,
session.protocol_version_string(),
session.count_pending_items(),
session.txs_sent,
@@ -630,6 +631,7 @@ class SessionBase(RPCSession):
self.kind = kind # 'RPC', 'TCP' etc.
self.env = session_mgr.env
self.coin = self.env.coin
self.client = 'unknown'
self.anon_logs = self.env.anon_logs
self.txs_sent = 0
self.log_me = False
@@ -673,16 +675,15 @@ class SessionBase(RPCSession):
context = {'conn_id': f'{self.session_id}'}
self.logger = util.ConnectionLogger(self.logger, context)
self.group = self.session_mgr.add_session(self)
SESSIONS_COUNT.labels(version=self.client_version).inc()
peer_addr_str = self.peer_address_str()
self.logger.info(f'{self.kind} {peer_addr_str}, '
SESSIONS_COUNT.inc()
self.logger.info(f'{self.kind} {self.peer_address_str()}, '
f'{self.session_mgr.session_count():,d} total')
def connection_lost(self, exc):
"""Handle client disconnection."""
super().connection_lost(exc)
self.session_mgr.remove_session(self)
SESSIONS_COUNT.labels(version=self.client_version).dec()
SESSIONS_COUNT.dec()
msg = ''
if not self._can_send.is_set():
msg += ' whilst paused'
@@ -706,7 +707,7 @@ class SessionBase(RPCSession):
"""Handle an incoming request. ElectrumX doesn't receive
notifications from client sessions.
"""
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
REQUESTS_COUNT.labels(method=request.method).inc()
if isinstance(request, Request):
handler = self.request_handlers.get(request.method)
handler = partial(handler, self)
@@ -837,6 +838,12 @@ class LBRYElectrumX(SessionBase):
self.bp: LBRYBlockProcessor = self.session_mgr.bp
self.db: LBRYLevelDB = self.bp.db
@lru_cache(1000)
async def _cached_block_headers(_start_height, _count, _b64, _max_size, _cp_height):
return await self._block_headers(_start_height, _count, _b64, _max_size, _cp_height)
self._cached_block_headers = _cached_block_headers
@classmethod
def protocol_min_max_strings(cls):
return [util.version_string(ver)
@@ -1299,6 +1306,20 @@ class LBRYElectrumX(SessionBase):
height: the header's height"""
return await self.block_header(height)
async def _block_headers(self, start_height, count, b64, max_size, cp_height):
headers, count = await self.db.read_headers(start_height, count)
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
result = {
'base64' if b64 else 'hex': headers,
'count': count,
'max': max_size
}
if count and cp_height:
last_height = start_height + count - 1
result.update(await self._merkle_proof(cp_height, last_height))
return result
async def block_headers(self, start_height, count, cp_height=0, b64=False):
"""Return count concatenated block headers as hex for the main chain;
starting at start_height.
@@ -1312,18 +1333,10 @@ class LBRYElectrumX(SessionBase):
max_size = self.MAX_CHUNK_SIZE
count = min(count, max_size)
headers, count = await self.db.read_headers(start_height, count)
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
result = {
'base64' if b64 else 'hex': headers,
'count': count,
'max': max_size
}
if count and cp_height:
last_height = start_height + count - 1
result.update(await self._merkle_proof(cp_height, last_height))
return result
if count + start_height < self.db.db_height:
return await self._cached_block_headers(start_height, count, b64, max_size, cp_height)
return await self._block_headers(start_height, count, b64, max_size, cp_height)
async def block_get_chunk(self, index):
"""Return a chunk of block headers as a hexadecimal string.
@@ -1416,7 +1429,6 @@ class LBRYElectrumX(SessionBase):
client_name: a string identifying the client
protocol_version: the protocol version spoken by the client
"""
if self.sv_seen and self.protocol_tuple >= (1, 4):
raise RPCError(BAD_REQUEST, f'server.version already sent')
self.sv_seen = True
@@ -1428,11 +1440,8 @@ class LBRYElectrumX(SessionBase):
self.close_after_send = True
raise RPCError(BAD_REQUEST,
f'unsupported client: {client_name}')
if self.client_version != client_name[:17]:
SESSIONS_COUNT.labels(version=self.client_version).dec()
self.client_version = client_name[:17]
SESSIONS_COUNT.labels(version=self.client_version).inc()
CLIENT_VERSIONS.labels(version=self.client_version).inc()
self.client = client_name[:17]
CLIENT_VERSIONS.labels(version=str(client_name)).inc()
# Find the highest common protocol version. Disconnect if
# that protocol version in unsupported.
-107
View File
@@ -1,107 +0,0 @@
#!/usr/bin/env python3
import asyncio
import logging
import os
import sys
from lbry.wallet import database
def enable_logging():
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
handler.setFormatter(formatter)
root.addHandler(handler)
async def initialize_tables(queue, old_file, db):
await db.executescript("PRAGMA journal_mode=WAL")
await db.executescript("PRAGMA case_sensitive_like=true")
await db.executescript("CREATE TABLE IF NOT EXISTS claim_with_metadata "
"(claimID BLOB NOT NULL PRIMARY KEY, "
"name BLOB NOT NULL, "
"originalHeight INTEGER NOT NULL, "
"updateHeight INTEGER NOT NULL, "
"activationHeight INTEGER NOT NULL, "
"metadataField1 TEXT)")
await db.executescript(f"ATTACH '{old_file}' AS crd")
await db.create_function("NEEDS_UPDATE", 5, lambda id, name, original, update, activation:
queue.put_nowait({
"claimID": id, "name": name, "originalHeight": original,
"updateHeight": update, "activationHeight": activation
}))
await db.executescript("CREATE TEMP TRIGGER on_claim_insert AFTER INSERT ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_update AFTER UPDATE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_delete AFTER DELETE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(OLD.claimID, OLD.nodeName, 0, 0, 0); END")
async def catch_up_on_missing(queue, db):
# items removed from crd are those that are in claim_with_metadata but not in crd
# items inserted into crd are those in crd but not in claim_with_metadata
# items updated are those that are in both but some columns don't match
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID NOT IN"
"(SELECT c.claimID FROM crd.claim c)")
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"SELECT c.claimID, c.nodeName, c.originalHeight, c.updateHeight, c.activationHeight "
"FROM crd.claim c LEFT JOIN claim_with_metadata m ON c.claimID = m.claimID "
"WHERE m.claimID IS NULL OR c.nodeName != m.name OR c.updateHeight != m.updateHeight "
"OR c.activationHeight != m.activationHeight ON CONFLICT(claimID) DO UPDATE SET "
"name = excluded.name, originalHeight = excluded.originalHeight, "
"updateHeight = excluded.updateHeight, activationHeight = excluded.activationHeight")
async def remove_claim(claim, db):
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID = :claimID", claim)
async def insert_update(claim, db):
# TODO: lookup and parse metadata here
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"VALUES(:claimID, :name, :originalHeight, :updateHeight, :activationHeight) "
"ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, "
"originalHeight = excluded.originalHeight, updateHeight = excluded.updateHeight, "
"activationHeight = excluded.activationHeight", claim)
async def handle_changes():
old_file = os.path.expanduser("~/.lbrycrd/claims.sqlite")
new_file = os.path.expanduser("~/.lbrycrd/duplicate.sqlite")
db = database.AIOSQLite()
db = await db.connect(new_file) # super confusing that you have to use the return value here
queue = asyncio.Queue()
await initialize_tables(queue, old_file, db)
await catch_up_on_missing(queue, db)
while True:
claim = await queue.get()
if claim.updateHeight <= 0:
await remove_claim(claim, db)
else:
await insert_update(claim, db)
def main():
enable_logging()
try:
asyncio.run(handle_changes())
except KeyboardInterrupt:
logging.info("Process interrupted")
if __name__ == '__main__':
main()