Compare commits

..
7 Commits
Author SHA1 Message Date
Victor Shyba 0aaa2ecaed fix parameter 2020-12-20 21:06:08 -03:00
Victor Shyba b9e913ddc7 fix verification 2020-12-20 13:35:43 -03:00
Victor Shyba 048f061988 checkpoint 2020-12-20 13:34:47 -03:00
Victor Shyba 39690a1c45 asyncgens 2020-12-20 03:12:11 -03:00
Victor Shyba f0d328cde7 refactor duplicate code 2020-12-20 00:50:48 -03:00
Victor Shyba 635cab87a7 wip, see jack 2020-12-20 00:09:35 -03:00
Victor Shyba 6762aafe20 join network can only happen after initial header sync returns 2020-12-19 17:00:01 -03:00
46 changed files with 669 additions and 1118 deletions

No files matched your search

+6 -8
View File
@@ -57,11 +57,11 @@ test:other-integration:
- pip install tox-travis
- tox -e other
#test:json-api:
# stage: test
# script:
# - make install tools
# - HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
test:json-api:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
@@ -121,11 +121,9 @@ build:windows:
GIT_DEPTH: 5
before_script:
- ./docker/install_choco.ps1
- choco install -y --x64 python --version=3.7.9
- choco install -y 7zip checksum
- choco install -y --x86 python3 7zip checksum
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
- pip --version # | findstr /E '\(python 3\.7\)'
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
- pip install virtualenv pywin32
- virtualenv venv
- venv/Scripts/activate.ps1
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.87.0"
__version__ = "0.84.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+1 -1
View File
@@ -638,7 +638,7 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
transaction_cache_size = Integer("Transaction cache size", 100_000)
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
+7 -40
View File
@@ -3,7 +3,6 @@ import asyncio
import typing
import binascii
import socket
import time
from lbry.utils import resolve_host
from lbry.dht import constants
from lbry.dht.peer import make_kademlia_peer
@@ -18,16 +17,6 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
class NodeState:
def __init__(self,
routing_table_peers: typing.List[typing.Tuple[bytes, str, int, int]],
datastore: typing.List[typing.Tuple[bytes, str, int, int, bytes, float]]):
# List[Tuple[node_id, address, udp_port, tcp_port]]
self.routing_table_peers = routing_table_peers
# List[Tuple[node_id, address, udp_port, tcp_port, blob_hash, added_at]]
self.datastore = datastore
class Node:
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
@@ -42,7 +31,6 @@ class Node:
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
self._storage = storage
self.started_listening = asyncio.Event()
async def refresh_node(self, force_once=False):
while True:
@@ -115,7 +103,6 @@ class Node:
return stored_to
def stop(self) -> None:
self.started_listening.clear()
if self.joined.is_set():
self.joined.clear()
if self._join_task:
@@ -131,35 +118,18 @@ class Node:
self.listening_port = None
log.info("Stopped DHT node")
def get_state(self) -> NodeState:
return NodeState(
routing_table_peers=[(p.node_id, p.address, p.udp_port, p.tcp_port)
for p in self.protocol.routing_table.get_peers()],
datastore=self.protocol.data_store.dump()
)
def load_state(self, state: NodeState):
now = self.loop.time()
for node_id, address, udp_port, tcp_port, blob_hash, added_at in state.datastore:
if added_at + constants.DATA_EXPIRATION < now:
continue
p = make_kademlia_peer(node_id, address, udp_port, tcp_port)
self.protocol.data_store.add_peer_to_blob(p, blob_hash)
async def start_listening(self, interface: str = '0.0.0.0') -> None:
if not self.listening_port:
self.listening_port, _ = await self.loop.create_datagram_endpoint(
lambda: self.protocol, (interface, self.internal_udp_port)
)
self.started_listening.set()
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
self.protocol.start()
else:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: str = '0.0.0.0',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
persisted_peers: typing.List[typing.Tuple[bytes, str, int, int]] = []):
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
peer_addresses = []
for node_id, address, udp_port, tcp_port in urls:
@@ -178,15 +148,15 @@ class Node:
if not self.joined.is_set():
self.joined.set()
log.info(
"%s: joined dht, %i peers known in %i buckets",
bytes.fromhex(self.protocol.node_id)[:8],
len(self.protocol.routing_table.get_peers()),
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts()
)
else:
if self.joined.is_set():
self.joined.clear()
seed_peers = peers_from_urls(persisted_peers) if persisted_peers else []
seed_peers = peers_from_urls(
await self._storage.get_persisted_kademlia_peers()
) if self._storage else []
if not seed_peers:
try:
seed_peers.extend(peers_from_urls([
@@ -203,11 +173,8 @@ class Node:
await asyncio.sleep(1, loop=self.loop)
def start(self, interface: str,
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
persisted_peers: typing.List[typing.Tuple[bytes, str, int, int]] = []):
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls, persisted_peers))
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
-6
View File
@@ -68,9 +68,3 @@ class DictDataStore:
for _, stored in self._data_store.items():
peers.update(set(map(lambda tup: tup[0], stored)))
return list(peers)
def dump(self) -> typing.List[typing.Tuple[bytes, str, int, int, bytes, float]]:
data = []
for k, peers in self._data_store.items():
data.extend([(p.node_id, p.address, p.udp_port, p.tcp_port, k, added_at) for (p, added_at) in peers])
return data
-6
View File
@@ -287,7 +287,6 @@ class KademliaProtocol(DatagramProtocol):
self._to_add: typing.Set['KademliaPeer'] = set()
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
self.event_queue = asyncio.Queue(maxsize=100)
@functools.lru_cache(128)
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
@@ -429,11 +428,6 @@ class KademliaProtocol(DatagramProtocol):
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
sender_contact.address, sender_contact.udp_port)
try:
self.event_queue.put_nowait((sender_contact.node_id, sender_contact.address, method, args))
except asyncio.QueueFull:
pass
if method == b'ping':
result = self.node_rpc.ping()
elif method == b'store':
+1 -1
View File
@@ -286,7 +286,7 @@ class TreeRoutingTable:
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
if not to_pop:
return
log.debug("%s: join buckets %i", bytes.hex(self._parent_node_id)[:8], len(to_pop))
log.info("join buckets %i", len(to_pop))
bucket_index_to_pop = to_pop[0]
assert len(self.buckets[bucket_index_to_pop]) == 0
can_go_lower = bucket_index_to_pop - 1 >= 0
+22 -25
View File
@@ -26,45 +26,42 @@ def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
except KeyError:
pass
return False
def verify(channel, data, signature, channel_hash=None):
pieces = [
signature['signing_ts'].encode(),
channel_hash or channel.claim_hash,
data
]
return Output.is_signature_valid(
get_encoded_signature(signature['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
comment.update(sign(channel, signing_field.encode()))
def sign(channel, data):
timestamp = str(int(time.time()))
pieces = [timestamp.encode(), channel.claim_hash, data]
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
return {
comment.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
}
})
def sign_reaction(reaction: dict, channel: Output):
timestamp = str(int(time.time()))
signing_field = reaction['channel_name']
reaction.update(sign(channel, signing_field.encode()))
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
reaction.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
+1 -30
View File
@@ -21,7 +21,6 @@ import base58
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter
from google.protobuf.message import DecodeError
from lbry.wallet import (
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
Transaction, Output, Input, Account, database
@@ -2789,34 +2788,6 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_sign(
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
"""
Signs data using the specified channel signing key.
Usage:
channel_sign [<channel_name> | --channel_name=<channel_name>]
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
Returns: {}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
signing_channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
return comment_client.sign(signing_channel, unhexlify(hexdata))
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_abandon(
self, claim_id=None, txid=None, nout=None, account_id=None, wallet_id=None,
@@ -5163,7 +5134,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
comment_create (<comment> | --comment=<comment>)
(<claim_id> | --claim_id=<claim_id>) [--parent_id=<parent_id>]
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
(--channel_id=<channel_id> | --channel_name=<channel_name>)
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
+1 -37
View File
@@ -1,44 +1,14 @@
import time
import logging
import asyncio
import asyncio.tasks
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
from prometheus_client import Counter, Histogram, Gauge
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
def get_loop_metrics(delay=1):
loop = asyncio.get_event_loop()
def callback(started):
PROBE_TIMES.observe(time.perf_counter() - started - delay)
PROBES_FINISHED.inc()
async def monitor_loop_responsiveness():
while True:
now = time.perf_counter()
loop.call_later(delay, callback, now)
PROBES_IN_FLIGHT.inc()
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
await asyncio.sleep(delay)
return loop.create_task(monitor_loop_responsiveness())
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
self._monitor_loop_task = None
async def start(self, interface: str, port: int):
self.logger.info("start prometheus metrics")
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
@@ -46,10 +16,7 @@ class PrometheusServer:
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info(
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
)
self._monitor_loop_task = get_loop_metrics()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
@@ -62,7 +29,4 @@ class PrometheusServer:
raise
async def stop(self):
if self._monitor_loop_task and not self._monitor_loop_task.done():
self._monitor_loop_task.cancel()
self._monitor_loop_task = None
await self.runner.cleanup()
+3 -5
View File
@@ -5,10 +5,8 @@ from binascii import hexlify
from itertools import chain
from lbry.error import ResolveCensoredError
from lbry.schema.types.v2.result_pb2 import \
Outputs as OutputsMessage, \
Output as OutputMessage, \
Error as ErrorMessage
from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage
INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
@@ -72,7 +70,7 @@ class Outputs:
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total'
def __init__(self, txos: List[OutputMessage], extra_txos: List, txs: set,
def __init__(self, txos: List, extra_txos: List, txs: set,
offset: int, total: int, blocked: List, blocked_total: int):
self.txos = txos
self.txs = txs
+8 -1
View File
@@ -6,7 +6,7 @@ from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"[^=&#:$@%*?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
def _named(name, regex):
@@ -23,6 +23,7 @@ def _create_url_regex():
_named(name+"_name", prefix + invalid_names_regex) +
_oneof(
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(r'\*' + _named(name+"_sequence", '[1-9][0-9]*')),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
@@ -49,6 +50,7 @@ def normalize_name(name):
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
@@ -59,6 +61,8 @@ class PathSegment(NamedTuple):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
@@ -66,6 +70,8 @@ class PathSegment(NamedTuple):
def __str__(self):
if self.claim_id is not None:
return f"{self.name}:{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}*{self.sequence}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
@@ -112,6 +118,7 @@ class URL(NamedTuple):
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
+1 -1
View File
@@ -40,7 +40,7 @@ class StreamDownloader:
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2)
if self.blob_manager.decrypted_blob_lru_cache is not None:
if self.blob_manager.decrypted_blob_lru_cache:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob
)
+4 -3
View File
@@ -354,19 +354,20 @@ class ManagedStream(ManagedDownloadSource):
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionError:
except ConnectionRefusedError:
return sent
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
elif isinstance(err, OSError):
raise err
if isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
return sent
raise asyncio.CancelledError()
finally:
if protocol.transport:
protocol.transport.close()
+2 -2
View File
@@ -160,13 +160,13 @@ class StreamManager(SourceManager):
async def reflect_streams(self):
try:
return await self._reflect_streams()
return await self._reflact_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
async def _reflect_streams(self):
async def _reflact_streams(self):
# todo: those debug statements are temporary for #2987 - remove them if its closed
while True:
if self.config.reflect_streams and self.config.reflector_servers:
-2
View File
@@ -340,7 +340,6 @@ class CommandTestCase(IntegrationTestCase):
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_config = Config()
self.server_config.transaction_cache_size = 10000
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
await self.server_storage.open()
@@ -390,7 +389,6 @@ class CommandTestCase(IntegrationTestCase):
conf.fixed_peers = [('127.0.0.1', 5567)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.transaction_cache_size = 10000
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
+5 -61
View File
@@ -20,11 +20,8 @@ import pkg_resources
import certifi
import aiohttp
from prometheus_client import Counter
from prometheus_client.registry import REGISTRY
from lbry.schema.claim import Claim
log = logging.getLogger(__name__)
@@ -209,41 +206,15 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
class LRUCache:
__slots__ = [
'capacity',
'cache',
'_track_metrics',
'hits',
'misses'
'cache'
]
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
def __init__(self, capacity):
self.capacity = capacity
self.cache = collections.OrderedDict()
if metric_name is None:
self._track_metrics = False
self.hits = self.misses = None
else:
self._track_metrics = True
try:
self.hits = Counter(
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
)
self.misses = Counter(
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
)
except ValueError as err:
log.warning("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
self._track_metrics = False
self.hits = self.misses = None
def get(self, key, default=None):
try:
value = self.cache.pop(key)
if self._track_metrics:
self.hits.inc()
except KeyError:
if self._track_metrics:
self.misses.inc()
return default
def get(self, key):
value = self.cache.pop(key)
self.cache[key] = value
return value
@@ -255,43 +226,16 @@ class LRUCache:
self.cache.popitem(last=False)
self.cache[key] = value
def clear(self):
self.cache.clear()
def pop(self, key):
return self.cache.pop(key)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool:
return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
if self._track_metrics: # needed for tests
try:
REGISTRY.unregister(self.hits)
REGISTRY.unregister(self.misses)
except AttributeError:
pass
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCache(cache_size)
lru_cache = override_lru_cache or LRUCache(cache_size)
def wrapper(async_fn):
+6 -12
View File
@@ -121,13 +121,7 @@ class AIOSQLite:
if self._closing:
return
self._closing = True
def __checkpoint_and_close(conn: sqlite3.Connection):
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
log.info("DB checkpoint finished.")
conn.close()
await asyncio.get_event_loop().run_in_executor(
self.writer_executor, __checkpoint_and_close, self.writer_connection)
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
self.writer_executor.shutdown(wait=True)
self.reader_executor.shutdown(wait=True)
self.read_ready.clear()
@@ -160,7 +154,7 @@ class AIOSQLite:
await self.read_ready.wait()
still_waiting = True
if self._closing:
raise asyncio.CancelledError()
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.reader_executor, read_only_fn, sql, parameters
)
@@ -204,7 +198,7 @@ class AIOSQLite:
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError()
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
@@ -241,7 +235,7 @@ class AIOSQLite:
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError()
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
@@ -431,7 +425,7 @@ class SQLiteMixin:
return
await self.db.executescript('\n'.join(
f"DROP TABLE {table};" for table in tables
) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;')
))
await self.db.execute(self.CREATE_VERSION_TABLE)
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
await self.db.executescript(self.CREATE_TABLES_QUERY)
@@ -575,7 +569,7 @@ def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: L
class Database(SQLiteMixin):
SCHEMA_VERSION = "1.5"
SCHEMA_VERSION = "1.3"
PRAGMAS = """
pragma journal_mode=WAL;
+144 -77
View File
@@ -10,11 +10,11 @@ from collections import defaultdict
from binascii import hexlify, unhexlify
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
import pylru
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
from lbry.schema.url import URL
from lbry.crypto.hash import hash160, double_sha256, sha256
from lbry.crypto.base58 import Base58
from lbry.utils import LRUCache
from .tasks import TaskGroup
from .database import Database
@@ -155,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
self._on_ready_controller = StreamController()
self.on_ready = self._on_ready_controller.stream
self._tx_cache = LRUCache(self.config.get("tx_cache_size", 1024), metric_name='tx')
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 100_000))
self._update_tasks = TaskGroup()
self._other_tasks = TaskGroup() # that we dont need to start
self._utxo_reservation_lock = asyncio.Lock()
@@ -167,7 +167,7 @@ class Ledger(metaclass=LedgerRegistry):
self._known_addresses_out_of_sync = set()
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self._balance_cache = LRUCache(2 ** 15)
self._balance_cache = pylru.lrucache(100000)
@classmethod
def get_id(cls):
@@ -415,7 +415,6 @@ class Ledger(metaclass=LedgerRegistry):
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
height, height+rewound
)
self._tx_cache.clear()
else:
raise IndexError(f"headers.connect() returned negative number ({added})")
@@ -514,8 +513,10 @@ class Ledger(metaclass=LedgerRegistry):
)
return True
synced_txs = []
to_request = {}
pending_synced_history = {}
updated_cached_items = {}
already_synced = set()
already_synced_offset = 0
@@ -525,6 +526,10 @@ class Ledger(metaclass=LedgerRegistry):
already_synced.add((txid, remote_height))
already_synced_offset += 1
continue
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx_indexes = {}
@@ -532,16 +537,29 @@ class Ledger(metaclass=LedgerRegistry):
tx_indexes[txid] = i
if (txid, remote_height) in already_synced:
continue
cache_item = self._tx_cache.get(txid)
cache_item.pending_verifications += 1
updated_cached_items[txid] = cache_item
assert cache_item is not None, 'cache item is none'
# tx = cache_item.tx
# if cache_item.tx is not None and \
# cache_item.tx.height >= remote_height and \
# (cache_item.tx.is_verified or remote_height < 1):
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
# continue
to_request[i] = (txid, remote_height)
log.debug(
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
len(remote_history), address
)
remote_history_txids = set(txid for txid, _ in remote_history)
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
if len(pending_synced_history) % 100 == 0:
synced_txs.append(tx)
if len(synced_txs) >= 100:
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
@@ -554,7 +572,19 @@ class Ledger(metaclass=LedgerRegistry):
if f"{txid}:{height}:" != pending_synced_history[i]:
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
synced_history += pending_synced_history[i]
await self.db.set_address_history(address, synced_history)
await self.db.save_transaction_io_batch(
synced_txs, address, self.address_to_hash160(address), synced_history
)
while synced_txs:
self._on_transaction_controller.add(TransactionEvent(address, synced_txs.pop()))
cache_size = self.config.get("tx_cache_size", 100_000)
for txid, cache_item in updated_cached_items.items():
cache_item.pending_verifications -= 1
if cache_item.pending_verifications < 0:
log.warning("config value tx cache size %i needs to be increased", cache_size)
cache_item.pending_verifications = 0
if address_manager is None:
address_manager = await self.get_address_manager_for_address(address)
@@ -562,6 +592,13 @@ class Ledger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
for txid, cache_item in updated_cached_items.items():
if self._tx_cache.get(txid) is not cache_item:
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
if reattempt_update:
return await self.update_history(address, remote_status, address_manager, False)
return False
local_status, local_history = \
await self.get_local_status_and_history(address, synced_history)
@@ -593,8 +630,48 @@ class Ledger(metaclass=LedgerRegistry):
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
return True
async def cache_transaction(self, txid, remote_height, check_local=True):
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = self._tx_cache[txid] = TransactionCacheItem()
elif cache_item.tx is not None and \
cache_item.tx.height >= remote_height and \
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
cache_item.pending_verifications += 1
try:
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
cache_item.tx = tx # make sure it's saved before caching it
tx.height = remote_height
if merkle and 0 < remote_height < len(self.headers):
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
finally:
cache_item.pending_verifications -= 1
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx.height = remote_height
cached = self._tx_cache.get(tx.id)
if not cached:
# cache txs looked up by transaction_show too
cached = TransactionCacheItem()
self._tx_cache[tx.id] = cached
cached.tx = tx
if 0 < remote_height < len(self.headers):
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
if not merkle:
@@ -605,65 +682,39 @@ class Ledger(metaclass=LedgerRegistry):
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], session=None):
batches = [[]]
remote_heights = {}
cache_hits = set()
for txid, height in sorted(to_request, key=lambda x: x[1]):
if cached:
cached_tx = self._tx_cache.get(txid)
if cached_tx is not None:
if cached_tx.tx is not None and cached_tx.tx.is_verified:
cache_hits.add(txid)
continue
else:
self._tx_cache[txid] = TransactionCacheItem()
remote_heights[txid] = height
if len(batches[-1]) == 100:
batches.append([])
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
if cached and cache_hits:
yield {txid: self._tx_cache[txid].tx for txid in cache_hits}
for batch in batches:
txs = await self._single_batch(batch, remote_heights)
if cached:
for txid, tx in txs.items():
self._tx_cache[txid].tx = tx
yield txs
async for tx in self._single_batch(batch, remote_heights, session):
yield tx
async def request_synced_transactions(self, to_request, remote_history, address):
async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())):
for tx in txs.values():
yield tx
await self._sync_and_save_batch(address, remote_history, txs)
pending_sync = []
async for tx in self.request_transactions(((txid, height) for txid, height in to_request.values())):
pending_sync.append(asyncio.ensure_future(self._sync(tx, remote_history)))
yield tx
await asyncio.gather(*pending_sync)
async def _single_batch(self, batch, remote_heights):
heights = {remote_heights[txid] for txid in batch}
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
txs = {}
async def _single_batch(self, batch, remote_heights, session):
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, session)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
tx = Transaction(unhexlify(raw), height=remote_height)
txs[tx.id] = tx
await self.maybe_verify_transaction(tx, remote_height, merkle)
return txs
yield tx
async def _sync_and_save_batch(self, address, remote_history, pending_txs):
await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values()))
await self.db.save_transaction_io_batch(
pending_txs.values(), address, self.address_to_hash160(address), ""
)
while pending_txs:
self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1]))
async def _sync(self, tx, remote_history, pending_txs):
async def _sync(self, tx, remote_history):
check_db_for_txos = {}
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
@@ -671,8 +722,9 @@ class Ledger(metaclass=LedgerRegistry):
wanted_txid = txi.txo_ref.tx_ref.id
if wanted_txid not in remote_history:
continue
if wanted_txid in pending_txs:
txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref
if wanted_txid in self._tx_cache:
await self._tx_cache[wanted_txid].has_tx.wait()
txi.txo_ref = self._tx_cache[wanted_txid].tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos[txi] = txi.txo_ref.id
@@ -686,11 +738,7 @@ class Ledger(metaclass=LedgerRegistry):
if txi.txo_ref.id in referenced_txos:
txi.txo_ref = referenced_txos[txi.txo_ref.id].ref
else:
tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id)
if tx_from_db is None:
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
else:
txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
return tx
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
@@ -756,13 +804,18 @@ class Ledger(metaclass=LedgerRegistry):
include_is_my_output=False,
include_sent_supports=False,
include_sent_tips=False,
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
include_received_tips=False,
session_override=None) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs: List[Transaction] = []
txs = []
if len(outputs.txs) > 0:
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
txs.extend(tx.values())
txs: List[Transaction] = []
if session_override:
async for tx in self.request_transactions(tuple(outputs.txs), session_override):
txs.append(tx)
else:
txs.extend((await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))))
_txos, blocked = outputs.inflate(txs)
@@ -837,17 +890,25 @@ class Ledger(metaclass=LedgerRegistry):
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
txos = []
urls_copy = list(urls)
if new_sdk_server:
resolve = partial(self.network.new_resolve, new_sdk_server)
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, **kwargs
))[0]
)
else:
resolve = partial(self.network.retriable_call, self.network.resolve)
while urls_copy:
batch, urls_copy = urls_copy[:100], urls_copy[100:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, **kwargs
))[0]
)
async with self.network.single_call_context(self.network.resolve) as (resolve, session):
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, session_override=session, **kwargs
))[0]
)
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
@@ -869,13 +930,17 @@ class Ledger(metaclass=LedgerRegistry):
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
if new_sdk_server:
claim_search = partial(self.network.new_claim_search, new_sdk_server)
else:
claim_search = self.network.claim_search
return await self._inflate_outputs(
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output,
)
return await self._inflate_outputs(
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output,
)
async with self.network.single_call_context(self.network.claim_search) as (claim_search, session):
return await self._inflate_outputs(
claim_search(**kwargs), accounts, session_override=session,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output,
)
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
@@ -980,7 +1045,7 @@ class Ledger(metaclass=LedgerRegistry):
return self.db.get_channel_count(**constraints)
async def resolve_collection(self, collection, offset=0, page_size=1):
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
claim_ids = collection.claim.collection.claims.ids[offset:page_size+offset]
try:
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception as err:
@@ -1029,7 +1094,7 @@ class Ledger(metaclass=LedgerRegistry):
'txid': tx.id,
'timestamp': ts,
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'confirmations': (headers.height + 1) - tx.height if tx.height > 0 else 0,
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
'claim_info': [],
'update_info': [],
'support_info': [],
@@ -1039,7 +1104,7 @@ class Ledger(metaclass=LedgerRegistry):
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
if is_my_inputs:
# fees only matter if we are the ones paying them
item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee)
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
item['fee'] = dewies_to_lbc(-tx.fee)
else:
# someone else paid the fees
@@ -1062,13 +1127,13 @@ class Ledger(metaclass=LedgerRegistry):
if txi.txo_ref.txo is not None:
other_txo = txi.txo_ref.txo
if (other_txo.is_claim or other_txo.script.is_support_claim) \
and other_txo.claim_id == txo.claim_id:
and other_txo.claim_id == txo.claim_id:
previous = other_txo
break
if previous is not None:
item['update_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(previous.amount - txo.amount),
'balance_delta': dewies_to_lbc(previous.amount-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
@@ -1146,7 +1211,7 @@ class Ledger(metaclass=LedgerRegistry):
for account in accounts:
balance = self._balance_cache.get(account.id)
if not balance:
balance = self._balance_cache[account.id] = \
balance = self._balance_cache[account.id] =\
await account.get_detailed_balance(confirmations, reserved_subtotals=True)
for key, value in balance.items():
if key == 'reserved_subtotals':
@@ -1156,6 +1221,7 @@ class Ledger(metaclass=LedgerRegistry):
result[key] += value
return result
class TestNetLedger(Ledger):
network_name = 'testnet'
pubkey_address_prefix = bytes((111,))
@@ -1164,6 +1230,7 @@ class TestNetLedger(Ledger):
extended_private_key_prefix = unhexlify('04358394')
checkpoints = {}
class RegTestLedger(Ledger):
network_name = 'regtest'
headers_class = UnvalidatedHeaders
+27 -2
View File
@@ -3,7 +3,10 @@ import asyncio
import json
from time import perf_counter
from operator import itemgetter
from contextlib import asynccontextmanager
from functools import partial
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
@@ -252,6 +255,28 @@ class Network:
pass
raise asyncio.CancelledError() # if we got here, we are shutting down
@asynccontextmanager
async def single_call_context(self, function, *args, **kwargs):
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
server = self.session_pool.fastest_session.server
session = ClientSession(network=self, server=server)
async def call_with_reconnect(*a, **kw):
while self.running:
if not session.available:
await session.create_connection()
try:
return await partial(function, *args, session_override=session, **kwargs)(*a, **kw)
except asyncio.TimeoutError:
log.warning("'%s' failed, retrying", function.__name__)
try:
yield (call_with_reconnect, session)
finally:
await session.close()
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
@@ -260,9 +285,9 @@ class Network:
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_batch(self, txids, restricted=True):
def get_transaction_batch(self, txids, restricted=True, session=None):
# use any server if its old, otherwise restrict to who gave us the history
return self.rpc('blockchain.transaction.get_batch', txids, restricted)
return self.rpc('blockchain.transaction.get_batch', txids, restricted, session)
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
+51 -72
View File
@@ -171,8 +171,7 @@ class BlockProcessor:
# Caches of unflushed items.
self.headers = []
self.block_hashes = []
self.block_txs = []
self.tx_hashes = []
self.undo_infos = []
# UTXO cache
@@ -216,7 +215,6 @@ class BlockProcessor:
for cache in self.search_cache.values():
cache.clear()
self.history_cache.clear()
self.notifications.notified_mempool_txs.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
self.block_count_metric.set(self.height)
@@ -248,7 +246,7 @@ class BlockProcessor:
self.logger.info('chain reorg detected')
else:
self.logger.info(f'faking a reorg of {count:,d} blocks')
await self.flush(True)
async def get_raw_blocks(last_height, hex_hashes):
heights = range(last_height, last_height - len(hex_hashes), -1)
@@ -265,28 +263,17 @@ class BlockProcessor:
self.touched.discard(None)
self.db.flush_backup(self.flush_data(), self.touched)
try:
await self.flush(True)
start, last, hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
self.logger.info("reorg %i block hashes", len(hashes))
for hex_hashes in chunks(hashes, 50):
raw_blocks = await get_raw_blocks(last, hex_hashes)
self.logger.info("got %i raw blocks", len(raw_blocks))
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
except:
self.logger.exception("boom")
raise
finally:
self.logger.info("done with reorg")
start, last, hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
for hex_hashes in chunks(hashes, 50):
raw_blocks = await get_raw_blocks(last, hex_hashes)
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
async def reorg_hashes(self, count):
"""Return a pair (start, last, hashes) of blocks to back up during a
@@ -349,8 +336,8 @@ class BlockProcessor:
def flush_data(self):
"""The data for a flush. The lock must be taken."""
assert self.state_lock.locked()
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
self.block_txs, self.undo_infos, self.utxo_cache,
return FlushData(self.height, self.tx_count, self.headers,
self.tx_hashes, self.undo_infos, self.utxo_cache,
self.db_deletes, self.tip)
async def flush(self, flush_utxos):
@@ -405,8 +392,7 @@ class BlockProcessor:
for block in blocks:
height += 1
undo_info = self.advance_txs(
height, block.transactions, self.coin.electrum_header(block.header, height),
self.coin.header_hash(block.header)
height, block.transactions, self.coin.electrum_header(block.header, height)
)
if height >= min_height:
self.undo_infos.append((undo_info, height))
@@ -417,26 +403,25 @@ class BlockProcessor:
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
def advance_txs(self, height, txs, header, block_hash):
self.block_hashes.append(block_hash)
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
undo_info = []
tx_num = self.tx_count
hashXs_by_tx = []
def advance_txs(self, height, txs, header):
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
# Use local vars for speed in the loops
undo_info = []
tx_num = self.tx_count
script_hashX = self.coin.hashX_from_script
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
update_touched = self.touched.update
append_hashX_by_tx = hashXs_by_tx.append
hashX_from_script = self.coin.hashX_from_script
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
for tx, tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
tx_numb = pack('<I', tx_num)
tx_numb = s_pack('<I', tx_num)
# Spend the inputs
for txin in tx.inputs:
@@ -449,17 +434,18 @@ class BlockProcessor:
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hashX. Ignore unspendable outputs
hashX = hashX_from_script(txout.pk_script)
hashX = script_hashX(txout.pk_script)
if hashX:
append_hashX(hashX)
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
put_utxo(tx_hash + s_pack('<H', idx),
hashX + tx_numb + s_pack('<Q', txout.value))
append_hashX_by_tx(hashXs)
append_hashXs(hashXs)
update_touched(hashXs)
self.db.total_transactions.append(tx_hash)
tx_num += 1
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
self.tx_count = tx_num
self.db.tx_counts.append(tx_num)
@@ -476,7 +462,6 @@ class BlockProcessor:
coin = self.coin
for raw_block in raw_blocks:
self.logger.info("backup block %i", self.height)
# Check and update self.tip
block = coin.block(raw_block, self.height)
header_hash = coin.header_hash(block.header)
@@ -502,16 +487,20 @@ class BlockProcessor:
# Use local vars for speed in the loops
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 12 + HASHX_LEN
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
hashX = self.coin.hashX_from_script(txout.pk_script)
hashX = script_hashX(txout.pk_script)
if hashX:
cache_value = self.spend_utxo(tx_hash, idx)
self.touched.add(cache_value[:-12])
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-12])
# Restore the inputs
for txin in reversed(tx.inputs):
@@ -519,10 +508,9 @@ class BlockProcessor:
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
self.touched.add(undo_item[:-12])
self.db.total_transactions.pop()
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
undo_item)
touched.add(undo_item[:-12])
assert n == 0
self.tx_count -= len(txs)
@@ -601,17 +589,13 @@ class BlockProcessor:
prefix = b'h' + tx_hash[:4] + idx_packed
candidates = {db_key: hashX for db_key, hashX
in self.db.utxo_db.iterator(prefix=prefix)}
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-4:]
if len(candidates) > 1:
if len(candidates) > 1:
tx_num, = unpack('<I', tx_num_packed)
try:
hash, height = self.db.fs_tx_hash(tx_num)
except IndexError:
self.logger.error("data integrity error for hashx history: %s missing tx #%s (%s:%s)",
hashX.hex(), tx_num, hash_to_hex_str(tx_hash), tx_idx)
continue
hash, height = self.db.fs_tx_hash(tx_num)
if hash != tx_hash:
assert hash is not None # Should always be found
continue
@@ -620,17 +604,12 @@ class BlockProcessor:
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-6:]
utxo_value_packed = self.db.utxo_db.get(udb_key)
if utxo_value_packed is None:
self.logger.warning(
"%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), tx_idx, hash_to_hex_str(hashX)
)
raise ChainError(f"{hash_to_hex_str(tx_hash)}:{tx_idx} is not found in UTXO db for {hash_to_hex_str(hashX)}")
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
self.logger.error('UTXO {hash_to_hex_str(tx_hash)} / {tx_idx} not found in "h" table')
raise ChainError('UTXO {} / {:,d} not found in "h" table'
.format(hash_to_hex_str(tx_hash), tx_idx))
@@ -643,7 +622,7 @@ class BlockProcessor:
self._caught_up_event.set()
await self.blocks_event.wait()
self.blocks_event.clear()
if self.reorg_count: # this could only happen by calling the reorg rpc
if self.reorg_count:
await self.reorg_chain(self.reorg_count)
self.reorg_count = 0
else:
@@ -791,9 +770,9 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
def advance_txs(self, height, txs, header, block_hash):
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
self.timer.show(height=height)
+6 -7
View File
@@ -3,10 +3,11 @@ import itertools
import json
import time
from functools import wraps
from pylru import lrucache
import aiohttp
from prometheus_client import Gauge, Histogram
from lbry.utils import LRUCache
from lbry.wallet.rpc.jsonrpc import RPCError
from lbry.wallet.server.util import hex_to_bytes, class_logger
from lbry.wallet.rpc import JSONRPC
@@ -54,8 +55,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = LRUCache(100000)
self._block_cache = LRUCache(2**16, metric_name='block', namespace=NAMESPACE)
self._block_hash_cache = lrucache(100000)
self._block_cache = lrucache(10000)
async def close(self):
if self.connector:
@@ -249,10 +250,8 @@ class Daemon:
async def deserialised_block(self, hex_hash):
"""Return the deserialised block with the given hex hash."""
if hex_hash not in self._block_cache:
block = await self._send_single('getblock', (hex_hash, True))
self._block_cache[hex_hash] = block
return block
if not self._block_cache.get(hex_hash):
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
return self._block_cache[hex_hash]
async def raw_blocks(self, hex_hashes):
+1 -1
View File
@@ -283,7 +283,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = {
unhexlify(cid)[::-1] for cid in channel_ids if cid
unhexlify(cid)[::-1] for cid in channel_ids
}
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
+1 -3
View File
@@ -37,9 +37,7 @@ class Env:
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = [
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
]
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
+4 -4
View File
@@ -67,17 +67,17 @@ def hash160(x):
return ripemd160(sha256(x))
def hash_to_hex_str(x: bytes) -> str:
def hash_to_hex_str(x):
"""Convert a big-endian binary hash to displayed hex string.
Display form of a binary hash is reversed and converted to hex.
"""
return x[::-1].hex()
return bytes(reversed(x)).hex()
def hex_str_to_hash(x: str) -> bytes:
def hex_str_to_hash(x):
"""Convert a displayed hex string to a binary hash."""
return hex_to_bytes(x)[::-1]
return bytes(reversed(hex_to_bytes(x)))
class Base58Error(Exception):
+14 -14
View File
@@ -173,20 +173,20 @@ class History:
self.logger.info(f'backing up removed {nremoves:,d} history entries')
# def get_txnums(self, hashX, limit=1000):
# """Generator that returns an unpruned, sorted list of tx_nums in the
# history of a hashX. Includes both spending and receiving
# transactions. By default yields at most 1000 entries. Set
# limit to None to get them all. """
# limit = util.resolve_limit(limit)
# for key, hist in self.db.iterator(prefix=hashX):
# a = array.array('I')
# a.frombytes(hist)
# for tx_num in a:
# if limit == 0:
# return
# yield tx_num
# limit -= 1
def get_txnums(self, hashX, limit=1000):
"""Generator that returns an unpruned, sorted list of tx_nums in the
history of a hashX. Includes both spending and receiving
transactions. By default yields at most 1000 entries. Set
limit to None to get them all. """
limit = util.resolve_limit(limit)
for key, hist in self.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
if limit == 0:
return
yield tx_num
limit -= 1
#
# History compaction
+89 -171
View File
@@ -14,17 +14,15 @@ import array
import ast
import os
import time
import zlib
import typing
from typing import Optional, List, Tuple, Iterable
from asyncio import sleep
from bisect import bisect_right
from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.utils import LRUCache
from lbry.wallet.server import util
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.merkle import Merkle, MerkleCache
@@ -37,10 +35,6 @@ UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
HEADER_PREFIX = b'H'
TX_COUNT_PREFIX = b'T'
TX_HASH_PREFIX = b'X'
TX_PREFIX = b'B'
TX_NUM_PREFIX = b'N'
BLOCK_HASH_PREFIX = b'C'
@attr.s(slots=True)
@@ -48,8 +42,7 @@ class FlushData:
height = attr.ib()
tx_count = attr.ib()
headers = attr.ib()
block_hashes = attr.ib()
block_txs = attr.ib()
block_tx_hashes = attr.ib()
# The following are flushed to the UTXO DB if undo_infos is not None
undo_infos = attr.ib()
adds = attr.ib()
@@ -81,7 +74,6 @@ class LevelDB:
self.history = History()
self.utxo_db = None
self.tx_counts = None
self.headers = None
self.last_flush = time.time()
self.logger.info(f'using {self.env.db_engine} for DB backend')
@@ -90,11 +82,9 @@ class LevelDB:
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.hashes_db = None
self.headers_db = None
self.tx_db = None
self._tx_and_merkle_cache = LRUCache(2**17, metric_name='tx_and_merkle', namespace="wallet_server")
self.total_transactions = None
self.tx_count_db = None
async def _read_tx_counts(self):
if self.tx_counts is not None:
@@ -105,7 +95,7 @@ class LevelDB:
def get_counts():
return tuple(
util.unpack_be_uint64(tx_count)
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
)
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
@@ -118,34 +108,9 @@ class LevelDB:
else:
assert self.db_tx_count == 0
async def _read_txids(self):
def get_txids():
return list(self.tx_db.iterator(prefix=TX_HASH_PREFIX, include_key=False))
start = time.perf_counter()
self.logger.info("loading txids")
txids = await asyncio.get_event_loop().run_in_executor(self.executor, get_txids)
assert len(txids) == len(self.tx_counts) == 0 or len(txids) == self.tx_counts[-1]
self.total_transactions = txids
ts = time.perf_counter() - start
self.logger.info("loaded %i txids in %ss", len(self.total_transactions), round(ts, 4))
async def _read_headers(self):
if self.headers is not None:
return
def get_headers():
return [
header for header in self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False)
]
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
self.headers = headers
async def _open_dbs(self, for_sync, compacting):
if self.executor is None:
self.executor = ThreadPoolExecutor(1)
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
coin_path = os.path.join(self.env.db_dir, 'COIN')
if not os.path.isfile(coin_path):
with util.open_file(coin_path, create=True) as f:
@@ -158,11 +123,17 @@ class LevelDB:
self.logger.info('created new headers db')
self.logger.info(f'opened headers DB (for sync: {for_sync})')
assert self.tx_db is None
self.tx_db = self.db_class('tx', for_sync)
if self.tx_db.is_new:
self.logger.info('created new tx db')
self.logger.info(f'opened tx DB (for sync: {for_sync})')
assert self.tx_count_db is None
self.tx_count_db = self.db_class('tx_count', for_sync)
if self.tx_count_db.is_new:
self.logger.info('created new tx count db')
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
assert self.hashes_db is None
self.hashes_db = self.db_class('hashes', for_sync)
if self.hashes_db.is_new:
self.logger.info('created new tx hashes db')
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
assert self.utxo_db is None
# First UTXO DB
@@ -180,15 +151,13 @@ class LevelDB:
# Read TX counts (requires meta directory)
await self._read_tx_counts()
if self.total_transactions is None:
await self._read_txids()
await self._read_headers()
def close(self):
self.utxo_db.close()
self.history.close_db()
self.headers_db.close()
self.tx_db.close()
self.tx_count_db.close()
self.hashes_db.close()
self.executor.shutdown(wait=True)
self.executor = None
@@ -218,9 +187,12 @@ class LevelDB:
if self.headers_db:
self.headers_db.close()
self.headers_db = None
if self.tx_db:
self.tx_db.close()
self.tx_db = None
if self.tx_count_db:
self.tx_count_db.close()
self.tx_count_db = None
if self.hashes_db:
self.hashes_db.close()
self.hashes_db = None
await self._open_dbs(False, False)
self.logger.info("opened for serving")
@@ -245,7 +217,7 @@ class LevelDB:
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.block_txs
assert not flush_data.block_tx_hashes
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
@@ -303,48 +275,33 @@ class LevelDB:
"""
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_txs) == len(flush_data.headers)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
assert len(
b''.join(hashes for hashes, _ in flush_data.block_txs)
) // 32 == flush_data.tx_count - prior_tx_count
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers
# Write the headers, tx counts, and tx hashes
start_time = time.perf_counter()
with self.headers_db.write_batch() as batch:
batch_put = batch.put
for i, header in enumerate(flush_data.headers):
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
self.headers.append(header)
flush_data.headers.clear()
height_start = self.fs_height + 1
tx_num = prior_tx_count
with self.tx_db.write_batch() as batch:
batch_put = batch.put
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
tx_count = self.tx_counts[height_start]
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
tx_num += 1
offset += 32
flush_data.block_txs.clear()
flush_data.block_hashes.clear()
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
tx_count = self.tx_counts[height_start]
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
tx_num += 1
offset += 32
flush_data.block_tx_hashes.clear()
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
flush_data.headers.clear()
elapsed = time.perf_counter() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
@@ -405,7 +362,7 @@ class LevelDB:
def flush_backup(self, flush_data, touched):
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
assert not flush_data.headers
assert not flush_data.block_txs
assert not flush_data.block_tx_hashes
assert flush_data.height < self.db_height
self.history.assert_flushed()
@@ -426,21 +383,19 @@ class LevelDB:
def backup_fs(self, height, tx_count):
"""Back up during a reorg. This just updates our pointers."""
while self.fs_height > height:
self.fs_height -= 1
self.headers.pop()
self.fs_height = height
self.fs_tx_count = tx_count
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(height + 1)
def raw_header(self, height):
async def raw_header(self, height):
"""Return the binary header at the given height."""
header, n = self.read_headers(height, 1)
header, n = await self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]:
async def read_headers(self, start_height, count):
"""Requires start_height >= 0, count >= 0. Reads as many headers as
are available starting at start_height up to count. This
would be zero if start_height is beyond self.db_height, for
@@ -449,15 +404,24 @@ class LevelDB:
Returns a (binary, n) pair where binary is the concatenated
binary headers, and n is the count of headers returned.
"""
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(self.headers[start_height:start_height + disk_count]), disk_count
return b'', 0
def read_headers():
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(
self.headers_db.iterator(
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
include_key=False
)
), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -465,60 +429,22 @@ class LevelDB:
If the tx_height is not on disk, returns (None, tx_height)."""
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
return None, tx_height
return self.total_transactions[tx_num], tx_height
def _fs_transactions(self, txids: Iterable[str]):
unpack_be_uint64 = util.unpack_be_uint64
tx_counts = self.tx_counts
tx_db_get = self.tx_db.get
tx_cache = self._tx_and_merkle_cache
tx_infos = {}
for tx_hash in txids:
cached_tx = tx_cache.get(tx_hash)
if cached_tx:
tx, merkle = cached_tx
else:
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
tx = None
tx_height = -1
if tx_num is not None:
tx_num = unpack_be_uint64(tx_num)
tx_height = bisect_right(tx_counts, tx_num)
if tx_height < self.db_height:
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
if tx_height == -1:
merkle = {
'block_height': -1
}
else:
tx_pos = tx_num - tx_counts[tx_height - 1]
branch, root = self.merkle.branch_and_root(
self.total_transactions[tx_counts[tx_height - 1]:tx_counts[tx_height]], tx_pos
)
merkle = {
'block_height': tx_height,
'merkle': [
hash_to_hex_str(hash)
for hash in branch
],
'pos': tx_pos
}
if tx_height + 10 < self.db_height:
tx_cache[tx_hash] = tx, merkle
tx_infos[tx_hash] = (None if not tx else tx.hex(), merkle)
return tx_infos
async def fs_transactions(self, txids):
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
return tx_hash, tx_height
async def fs_block_hashes(self, height, count):
if height + count > len(self.headers):
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
offset = 0
headers = []
for n in range(count):
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
offset += self.coin.BASIC_HEADER_SIZE
return [self.coin.header_hash(header) for header in headers]
async def limited_history(self, hashX, *, limit=1000):
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
@@ -527,33 +453,26 @@ class LevelDB:
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
"""
def read_history():
db_height = self.db_height
tx_counts = self.tx_counts
cnt = 0
txs = []
for hist in self.history.db.iterator(prefix=hashX, include_key=False):
hashx_history = []
for key, hist in self.history.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(tx_counts, tx_num)
if tx_height > db_height:
return
txs.append((tx_num, tx_height))
cnt += 1
if limit and cnt >= limit:
break
if limit and cnt >= limit:
break
return txs
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
hashx_history.append((tx_hash, tx_height))
if limit and len(hashx_history) >= limit:
return hashx_history
return hashx_history
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if history is not None:
return [(self.total_transactions[tx_num], tx_height) for (tx_num, tx_height) in history]
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
await sleep(0.25)
@@ -710,14 +629,13 @@ class LevelDB:
utxos = []
utxos_append = utxos.append
s_unpack = unpack
fs_tx_hash = self.fs_tx_hash
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
value, = unpack('<Q', db_value)
tx_hash, height = fs_tx_hash(tx_num)
tx_hash, height = self.fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
return utxos
+42 -37
View File
@@ -6,11 +6,13 @@
# and warranty status of this software.
"""Mempool handling."""
import os
import asyncio
import itertools
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
from prometheus_client import Histogram
import attr
@@ -72,7 +74,7 @@ class MemPoolAPI(ABC):
"""
@abstractmethod
async def on_mempool(self, touched, new_touched, height):
async def on_mempool(self, touched, height):
"""Called each time the mempool is synchronized. touched is a set of
hashXs touched since the previous call. height is the
daemon's height at the time the mempool was obtained."""
@@ -115,8 +117,8 @@ class MemPool:
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
self.mempool_process_time_metric = mempool_process_time_metric
self.notified_mempool_txs = set()
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
@@ -133,11 +135,15 @@ class MemPool:
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
while True:
await synchronized_event.wait()
async with self.lock:
self._update_histogram(100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
try:
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
finally:
self.executor.shutdown(wait=True)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
@@ -220,15 +226,10 @@ class MemPool:
continue
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
async with self.lock:
new_hashes = hashes.difference(self.notified_mempool_txs)
touched = await self._process_mempool(hashes)
self.notified_mempool_txs.update(new_hashes)
new_touched = {
touched_hashx for touched_hashx, txs in self.hashXs.items() if txs.intersection(new_hashes)
}
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(touched, new_touched, height)
await self.api.on_mempool(touched, height)
duration = time.perf_counter() - start
self.mempool_process_time_metric.observe(duration)
try:
@@ -242,8 +243,7 @@ class MemPool:
async def _process_mempool(self, all_hashes):
# Re-sync with the new set of hashes
txs = self.txs
hashXs = self.hashXs # hashX: [tx_hash, ...]
hashXs = self.hashXs
touched = set()
# First handle txs that have disappeared
@@ -274,8 +274,8 @@ class MemPool:
# FIXME: this is not particularly efficient
while tx_map and len(tx_map) != prior_count:
prior_count = len(tx_map)
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map, touched)
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map,
touched)
if tx_map:
self.logger.info(f'{len(tx_map)} txs dropped')
@@ -286,25 +286,30 @@ class MemPool:
hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes)
raw_txs = await self.api.raw_transactions(hex_hashes_iter)
to_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
def deserialize_txs(): # This function is pure
to_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
tx_map = {}
for hash, raw_tx in zip(hashes, raw_txs):
# The daemon may have evicted the tx from its
# mempool or it may have gotten in a block
if not raw_tx:
continue
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
# Convert the inputs and outputs into (hashX, value) pairs
# Drop generation-like inputs from MemPoolTx.prevouts
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
for txin in tx.inputs
if not txin.is_generation())
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
for txout in tx.outputs)
tx_map[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
0, tx_size)
txs = {}
for hash, raw_tx in zip(hashes, raw_txs):
# The daemon may have evicted the tx from its
# mempool or it may have gotten in a block
if not raw_tx:
continue
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
# Convert the inputs and outputs into (hashX, value) pairs
# Drop generation-like inputs from MemPoolTx.prevouts
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
for txin in tx.inputs
if not txin.is_generation())
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
for txout in tx.outputs)
txs[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
0, tx_size)
return txs
# Thread this potentially slow operation so as not to block
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
@@ -327,7 +332,7 @@ class MemPool:
"""Keep the mempool synchronized with the daemon."""
await asyncio.wait([
self._refresh_hashes(synchronized_event),
# self._refresh_histogram(synchronized_event),
self._refresh_histogram(synchronized_event),
self._logging(synchronized_event)
])
@@ -361,7 +366,7 @@ class MemPool:
result.update(tx.prevouts)
return result
def transaction_summaries(self, hashX):
async def transaction_summaries(self, hashX):
"""Return a list of MemPoolTxSummary objects for the hashX."""
result = []
for tx_hash in self.hashXs.get(hashX, ()):
+1 -1
View File
@@ -63,7 +63,7 @@ class Merkle:
raise TypeError('index must be an integer')
# This also asserts hashes is not empty
if not 0 <= index < len(hashes):
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
raise ValueError('index out of range')
natural_length = self.branch_length(len(hashes))
if length is None:
length = natural_length
+1 -1
View File
@@ -319,7 +319,7 @@ class PeerManager:
# Check prior header too in case of hard fork.
check_height = min(our_height, their_height)
raw_header = self.db.raw_header(check_height)
raw_header = await self.db.raw_header(check_height)
if ptuple >= (1, 4):
ours = raw_header.hex()
message = 'blockchain.block.header'
+7 -11
View File
@@ -25,10 +25,9 @@ class Notifications:
def __init__(self):
self._touched_mp = {}
self._touched_bp = {}
self.notified_mempool_txs = set()
self._highest_block = -1
async def _maybe_notify(self, new_touched):
async def _maybe_notify(self):
tmp, tbp = self._touched_mp, self._touched_bp
common = set(tmp).intersection(tbp)
if common:
@@ -45,24 +44,24 @@ class Notifications:
del tmp[old]
for old in [h for h in tbp if h <= height]:
touched.update(tbp.pop(old))
await self.notify(height, touched, new_touched)
await self.notify(height, touched)
async def notify(self, height, touched, new_touched):
async def notify(self, height, touched):
pass
async def start(self, height, notify_func):
self._highest_block = height
self.notify = notify_func
await self.notify(height, set(), set())
await self.notify(height, set())
async def on_mempool(self, touched, new_touched, height):
async def on_mempool(self, touched, height):
self._touched_mp[height] = touched
await self._maybe_notify(new_touched)
await self._maybe_notify()
async def on_block(self, touched, height):
self._touched_bp[height] = touched
self._highest_block = height
await self._maybe_notify(set())
await self._maybe_notify()
class Server:
@@ -85,12 +84,9 @@ class Server:
notifications.mempool_hashes = daemon.mempool_hashes
notifications.raw_transactions = daemon.getrawtransactions
notifications.lookup_utxos = db.lookup_utxos
MemPoolAPI.register(Notifications)
self.mempool = mempool = MemPool(env.coin, notifications)
notifications.notified_mempool_txs = self.mempool.notified_mempool_txs
self.session_mgr = env.coin.SESSION_MANAGER(
env, db, bp, daemon, mempool, self.shutdown_event
)
+160 -125
View File
@@ -4,6 +4,7 @@ import math
import time
import json
import zlib
import pylru
import base64
import codecs
import typing
@@ -16,12 +17,12 @@ from asyncio import Event, sleep
from collections import defaultdict
from functools import partial
from binascii import hexlify, unhexlify
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.utils import LRUCache
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
@@ -140,11 +141,7 @@ class SessionManager:
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
labelnames=("method", "version"))
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
@@ -189,9 +186,7 @@ class SessionManager:
self.shutdown_event = shutdown_event
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
self.sessions: typing.Dict[int, 'SessionBase'] = {}
self.hashx_subscriptions_by_session: typing.DefaultDict[str, typing.Set[int]] = defaultdict(set)
self.mempool_statuses = {}
self.sessions: typing.Set['SessionBase'] = set()
self.cur_group = SessionGroup(0)
self.txs_sent = 0
self.start_time = time.time()
@@ -277,12 +272,12 @@ class SessionManager:
def _group_map(self):
group_map = defaultdict(list)
for session in self.sessions.values():
for session in self.sessions:
group_map[session.group].append(session)
return group_map
def _sub_count(self) -> int:
return sum(s.sub_count() for s in self.sessions.values())
return sum(s.sub_count() for s in self.sessions)
def _lookup_session(self, session_id):
try:
@@ -290,7 +285,7 @@ class SessionManager:
except Exception:
pass
else:
for session in self.sessions.values():
for session in self.sessions:
if session.session_id == session_id:
return session
return None
@@ -314,7 +309,7 @@ class SessionManager:
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.perf_counter() - session_timeout
stale_sessions = [session for session in self.sessions.values()
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions:
text = ', '.join(str(session.session_id)
@@ -346,7 +341,7 @@ class SessionManager:
pending_requests = 0
closing = 0
for s in self.sessions.values():
for s in self.sessions:
error_count += s.errors
if s.log_me:
logged += 1
@@ -380,7 +375,7 @@ class SessionManager:
def _session_data(self, for_log):
"""Returned to the RPC 'sessions' call."""
now = time.time()
sessions = sorted(self.sessions.values(), key=lambda s: s.start_time)
sessions = sorted(self.sessions, key=lambda s: s.start_time)
return [(session.session_id,
session.flags(),
session.peer_address_str(for_log=for_log),
@@ -584,7 +579,7 @@ class SessionManager:
await self._close_servers(list(self.servers.keys()))
if self.sessions:
await asyncio.wait([
session.close(force_after=1) for session in self.sessions.values()
session.close(force_after=1) for session in self.sessions
])
await self.stop_other()
@@ -608,7 +603,7 @@ class SessionManager:
async def raw_header(self, height):
"""Return the binary header at the given height."""
try:
return self.db.raw_header(height)
return await self.db.raw_header(height)
except IndexError:
raise RPCError(BAD_REQUEST, f'height {height:,d} '
'out of range') from None
@@ -634,42 +629,18 @@ class SessionManager:
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
return self.history_cache[hashX]
async def _notify_sessions(self, height, touched, new_touched):
async def _notify_sessions(self, height, touched):
"""Notify sessions about height changes and touched addresses."""
height_changed = height != self.notified_height
if height_changed:
await self._refresh_hsub_results(height)
if not self.sessions:
return
if height_changed:
header_tasks = [
session.send_notification('blockchain.headers.subscribe', (self.hsub_results[session.subscribe_headers_raw], ))
for session in self.sessions.values() if session.subscribe_headers
]
if header_tasks:
self.logger.info(f'notify {len(header_tasks)} sessions of new header')
asyncio.create_task(asyncio.wait(header_tasks))
for hashX in touched.intersection(self.mempool_statuses.keys()):
self.mempool_statuses.pop(hashX, None)
touched.intersection_update(self.hashx_subscriptions_by_session.keys())
if touched or (height_changed and self.mempool_statuses):
notified_hashxs = 0
notified_sessions = 0
to_notify = touched if height_changed else new_touched
for hashX in to_notify:
for session_id in self.hashx_subscriptions_by_session[hashX]:
asyncio.create_task(self.sessions[session_id].send_history_notification(hashX))
notified_sessions += 1
notified_hashxs += 1
if notified_sessions:
self.logger.info(f'notified {notified_sessions} sessions/{notified_hashxs:,d} touched addresses')
if self.sessions:
await asyncio.wait([
session.notify(touched, height_changed) for session in self.sessions
])
def add_session(self, session):
self.sessions[id(session)] = session
self.sessions.add(session)
self.session_event.set()
gid = int(session.start_time - self.start_time) // 900
if self.cur_group.gid != gid:
@@ -678,13 +649,7 @@ class SessionManager:
def remove_session(self, session):
"""Remove a session from our sessions list if there."""
session_id = id(session)
for hashX in session.hashX_subs:
sessions = self.hashx_subscriptions_by_session[hashX]
sessions.remove(session_id)
if not sessions:
self.hashx_subscriptions_by_session.pop(hashX)
self.sessions.pop(session_id)
self.sessions.remove(session)
self.session_event.set()
@@ -719,6 +684,8 @@ class SessionBase(RPCSession):
self._receive_message_orig = self.connection.receive_message
self.connection.receive_message = self.receive_message
async def notify(self, touched, height_changed):
pass
def default_framer(self):
return NewlineFramer(self.env.max_receive)
@@ -810,8 +777,8 @@ class LBRYSessionManager(SessionManager):
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = LRUCache(2**14, metric_name='search', namespace=NAMESPACE)
self.search_cache['resolve'] = LRUCache(2**16, metric_name='resolve', namespace=NAMESPACE)
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
async def process_metrics(self):
while self.running:
@@ -915,6 +882,7 @@ class LBRYElectrumX(SessionBase):
self.connection.max_response_size = self.env.max_send
self.hashX_subs = {}
self.sv_seen = False
self.mempool_statuses = {}
self.protocol_tuple = self.PROTOCOL_MIN
self.daemon = self.session_mgr.daemon
@@ -959,22 +927,48 @@ class LBRYElectrumX(SessionBase):
def sub_count(self):
return len(self.hashX_subs)
async def send_history_notification(self, hashX):
start = time.perf_counter()
alias = self.hashX_subs[hashX]
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
async def notify(self, touched, height_changed):
"""Notify the client about changes to touched addresses (from mempool
updates or new blocks) and height.
"""
if height_changed and self.subscribe_headers:
args = (await self.subscribe_headers_result(), )
if not (await self.send_notification('blockchain.headers.subscribe', args)):
return
async def send_history_notification(alias, hashX):
start = time.perf_counter()
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
start = time.perf_counter()
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
touched = touched.intersection(self.hashX_subs)
if touched or (height_changed and self.mempool_statuses):
notified = set()
mempool_addrs = tuple(self.mempool_statuses.keys())
for hashX in touched:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
for hashX in mempool_addrs:
if hashX not in notified:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
if touched:
es = '' if len(touched) == 1 else 'es'
self.logger.info(f'notified {len(notified)} mempool/{len(touched):,d} touched address{es}')
def get_metrics_or_placeholder_for_api(self, query_name):
""" Do not hold on to a reference to the metrics
@@ -1051,12 +1045,7 @@ class LBRYElectrumX(SessionBase):
async def claimtrie_resolve(self, *urls):
if urls:
count = len(urls)
try:
self.session_mgr.urls_to_resolve_count_metric.inc(count)
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
finally:
self.session_mgr.resolved_url_count_metric.inc(count)
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
async def get_server_height(self):
return self.bp.height
@@ -1191,7 +1180,7 @@ class LBRYElectrumX(SessionBase):
# For mempool, height is -1 if it has unconfirmed inputs, otherwise 0
db_history = await self.session_mgr.limited_history(hashX)
mempool = self.mempool.transaction_summaries(hashX)
mempool = await self.mempool.transaction_summaries(hashX)
status = ''.join(f'{hash_to_hex_str(tx_hash)}:'
f'{height:d}:'
@@ -1205,9 +1194,9 @@ class LBRYElectrumX(SessionBase):
status = None
if mempool:
self.session_mgr.mempool_statuses[hashX] = status
self.mempool_statuses[hashX] = status
else:
self.session_mgr.mempool_statuses.pop(hashX, None)
self.mempool_statuses.pop(hashX, None)
return status
async def hashX_listunspent(self, hashX):
@@ -1226,14 +1215,13 @@ class LBRYElectrumX(SessionBase):
async def hashX_subscribe(self, hashX, alias):
self.hashX_subs[hashX] = alias
self.session_mgr.hashx_subscriptions_by_session[hashX].add(id(self))
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
sessions = self.session_mgr.hashx_subscriptions_by_session[hashX]
sessions.remove(id(self))
if not sessions:
self.hashX_subs.pop(hashX, None)
try:
del self.hashX_subs[hashX]
except ValueError:
pass
def address_to_hashX(self, address):
try:
@@ -1255,7 +1243,7 @@ class LBRYElectrumX(SessionBase):
async def address_get_mempool(self, address):
"""Return the mempool transactions touching an address."""
hashX = self.address_to_hashX(address)
return self.unconfirmed_history(hashX)
return await self.unconfirmed_history(hashX)
async def address_listunspent(self, address):
"""Return the list of UTXOs of an address."""
@@ -1291,20 +1279,20 @@ class LBRYElectrumX(SessionBase):
hashX = scripthash_to_hashX(scripthash)
return await self.get_balance(hashX)
def unconfirmed_history(self, hashX):
async def unconfirmed_history(self, hashX):
# Note unconfirmed history is unordered in electrum-server
# height is -1 if it has unconfirmed inputs, otherwise 0
return [{'tx_hash': hash_to_hex_str(tx.hash),
'height': -tx.has_unconfirmed_inputs,
'fee': tx.fee}
for tx in self.mempool.transaction_summaries(hashX)]
for tx in await self.mempool.transaction_summaries(hashX)]
async def confirmed_and_unconfirmed_history(self, hashX):
# Note history is ordered but unconfirmed is unordered in e-s
history = await self.session_mgr.limited_history(hashX)
conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height}
for tx_hash, height in history]
return conf + self.unconfirmed_history(hashX)
return conf + await self.unconfirmed_history(hashX)
async def scripthash_get_history(self, scripthash):
"""Return the confirmed and unconfirmed history of a scripthash."""
@@ -1314,7 +1302,7 @@ class LBRYElectrumX(SessionBase):
async def scripthash_get_mempool(self, scripthash):
"""Return the mempool transactions touching a scripthash."""
hashX = scripthash_to_hashX(scripthash)
return self.unconfirmed_history(hashX)
return await self.unconfirmed_history(hashX)
async def scripthash_listunspent(self, scripthash):
"""Return the list of UTXOs of a scripthash."""
@@ -1335,12 +1323,31 @@ class LBRYElectrumX(SessionBase):
f'require header height {height:,d} <= '
f'cp_height {cp_height:,d} <= '
f'chain height {max_height:,d}')
branch, root = await self.db.header_branch_and_root(cp_height + 1, height)
branch, root = await self.db.header_branch_and_root(cp_height + 1,
height)
return {
'branch': [hash_to_hex_str(elt) for elt in branch],
'root': hash_to_hex_str(root),
}
async def block_header(self, height, cp_height=0):
"""Return a raw block header as a hexadecimal string, or as a
dictionary with a merkle proof."""
height = non_negative_integer(height)
cp_height = non_negative_integer(cp_height)
raw_header_hex = (await self.session_mgr.raw_header(height)).hex()
if cp_height == 0:
return raw_header_hex
result = {'header': raw_header_hex}
result.update(await self._merkle_proof(cp_height, height))
return result
async def block_header_13(self, height):
"""Return a raw block header as a hexadecimal string.
height: the header's height"""
return await self.block_header(height)
async def block_headers(self, start_height, count, cp_height=0, b64=False):
"""Return count concatenated block headers as hex for the main chain;
starting at start_height.
@@ -1354,13 +1361,9 @@ class LBRYElectrumX(SessionBase):
max_size = self.MAX_CHUNK_SIZE
count = min(count, max_size)
headers, count = self.db.read_headers(start_height, count)
if b64:
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode()
else:
headers = headers.hex()
headers, count = await self.db.read_headers(start_height, count)
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
result = {
'base64' if b64 else 'hex': headers,
'count': count,
@@ -1378,7 +1381,7 @@ class LBRYElectrumX(SessionBase):
index = non_negative_integer(index)
size = self.coin.CHUNK_SIZE
start_height = index * size
headers, _ = self.db.read_headers(start_height, size)
headers, _ = await self.db.read_headers(start_height, size)
return headers.hex()
async def block_get_header(self, height):
@@ -1520,23 +1523,27 @@ class LBRYElectrumX(SessionBase):
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
return (await self.transaction_get_batch(tx_hash))[tx_hash]
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
async def transaction_get_batch(self, *tx_hashes):
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = await self.db.fs_transactions(tx_hashes)
needed_merkles = {}
batch_result = {}
for tx_hash in tx_hashes:
if tx_hash in batch_result and batch_result[tx_hash][0]:
continue
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
merkle = {}
if block_hash:
block = await self.daemon.deserialised_block(block_hash)
height = block['height']
@@ -1545,21 +1552,12 @@ class LBRYElectrumX(SessionBase):
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
merkle["pos"] = pos
else:
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
def threaded_get_merkle():
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
batch_result[tx_hash] = raw_tx, {
'merkle': self._get_merkle_branch(block_txs, pos),
'pos': pos,
'block_height': block_height
}
if needed_merkles:
await asyncio.get_running_loop().run_in_executor(self.db.executor, threaded_get_merkle)
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
height = -1
merkle['block_height'] = height
batch_result[tx_hash] = [raw_tx, merkle]
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
@@ -1574,6 +1572,19 @@ class LBRYElectrumX(SessionBase):
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def _block_hash_and_tx_hashes(self, height):
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
the given height.
block_hash is a hexadecimal string, and tx_hashes is an
ordered list of hexadecimal strings.
"""
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon.deserialised_block(block_hash)
return block_hash, block['tx']
def _get_merkle_branch(self, tx_hashes, tx_pos):
"""Return a merkle branch to a transaction.
@@ -1593,11 +1604,35 @@ class LBRYElectrumX(SessionBase):
height: the height of the block it is in
"""
assert_tx_hash(tx_hash)
result = await self.transaction_get_batch(tx_hash)
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block at height {height:,d}')
return result[tx_hash][1]
f'block {block_hash} at height {height:,d}')
branch = self._get_merkle_branch(tx_hashes, pos)
return {"block_height": height, "merkle": branch, "pos": pos}
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
"""Return the txid and optionally a merkle proof, given
a block height and position in the block.
"""
tx_pos = non_negative_integer(tx_pos)
if merkle not in (True, False):
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
tx_hash = tx_hashes[tx_pos]
except IndexError:
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
f'block {block_hash} at height {height:,d}')
if merkle:
branch = self._get_merkle_branch(tx_hashes, tx_pos)
return {"tx_hash": tx_hash, "merkle": branch}
else:
return tx_hash
class LocalRPC(SessionBase):
+6 -4
View File
@@ -77,16 +77,18 @@ class LevelDB(Storage):
import plyvel
cls.module = plyvel
def open(self, name, create, lru_cache_size=None):
mof = 10000
def open(self, name, create):
mof = 512 if self.for_sync else 128
path = os.path.join(self.db_dir, name)
# Use snappy compression (the default)
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
self.db = self.module.DB(path, create_if_missing=create,
max_open_files=mof)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
self.iterator = self.db.iterator
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
self.write_batch = partial(self.db.write_batch, transaction=True,
sync=True)
class RocksDB(Storage):
+1 -2
View File
@@ -337,8 +337,7 @@ class Examples(CommandTestCase):
'Use the parent_id param to make replies',
'comment', 'create',
'--comment="I have photographic evidence confirming Sasquatch exists"',
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}',
f'--claim_id={stream_id}'
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
)
await r(
-273
View File
@@ -1,273 +0,0 @@
#!/usr/bin/env python
import asyncio
import json
import logging
import pickle
import signal
import sqlite3
import time
from os import path
from pprint import pprint
from urllib.parse import unquote
from aioupnp import upnp, fault as upnpfault
from aiohttp import web
from lbry.wallet.network import ClientSession
from lbry.schema.result import Outputs
from lbry.wallet.transaction import Transaction
from binascii import hexlify, unhexlify
from lbry.dht import node, peer
log = logging.getLogger("lbry")
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
async def main():
data_dir = "/home/grin/code/lbry/sdk"
state_dir = data_dir + '/nodestate/'
loop = asyncio.get_event_loop()
try:
loop.add_signal_handler(signal.SIGINT, shutdown)
loop.add_signal_handler(signal.SIGTERM, shutdown)
except NotImplementedError:
pass # Not implemented on Windows
db = sqlite3.connect(data_dir + "/tracker.sqlite3")
db.execute('CREATE TABLE IF NOT EXISTS announce (local_id TEXT, hash TEXT, node_id TEXT, ip TEXT, port INT, timestamp INT)')
db.execute('CREATE UNIQUE INDEX IF NOT EXISTS node_id_hash_idx ON announce (node_id, hash)')
spv_host = 'spv13.lbry.com'
wallet_client = ClientSession(network=None, server=(spv_host, 50001))
await wallet_client.create_connection()
asyncio.create_task(run_web_api(loop, db, wallet_client))
num_nodes = 128
u = await upnp.UPnP.discover()
external_ip = await u.get_external_ip()
try:
nodes = await start_nodes(loop, num_nodes, external_ip, state_dir)
await asyncio.gather(*map(lambda n: n.started_listening.wait(), nodes), loop=loop)
print("joined")
queue = asyncio.Queue(maxsize=100*num_nodes)
for n in nodes:
asyncio.create_task(drain_events(n, queue))
while True:
(n, node_id, ip, method, args) = await queue.get()
local_node_id = bytes.hex(n.protocol.node_id)
if method != b'store':
# print(f"{local_node_id[:8]}: {method} from {bytes.hex(node_id)} ({ip})")
continue
if len(args)< 5:
print(f'malformed args to Store')
pprint(args)
continue
blob_hash, token, port, original_publisher_id, age = args[:5]
print(f"STORE to {local_node_id[:8]} from {bytes.hex(node_id)[:8]} ({ip}) for blob {bytes.hex(blob_hash)[:8]}")
try:
cur = db.cursor()
cur.execute(
'''
INSERT INTO announce (local_id, hash, node_id, ip, port, timestamp) VALUES (?,?,?,?,?,?)
ON CONFLICT (node_id, hash) DO UPDATE SET
local_id=excluded.local_id, ip=excluded.ip, port=excluded.port, timestamp=excluded.timestamp
''',
(local_node_id, bytes.hex(blob_hash), bytes.hex(node_id), ip, port, int(time.time()))
)
db.commit()
cur.close()
except sqlite3.Error as err:
print("failed sqlite insert", err)
finally:
print("shutting down")
for n in nodes:
node_id = bytes.hex(n.protocol.node_id)
n.stop()
# print(f'deleting upnp port mapping {n.protocol.udp_port}')
try:
await u.delete_port_mapping(n.protocol.udp_port, "UDP")
except upnpfault.UPnPError:
pass
state = n.get_state()
# keep existing rt if there is one
if len(state.routing_table_peers) == 0 and path.exists(state_dir + node_id):
with open(state_dir + node_id, 'rb') as f:
existing_state = pickle.load(f)
if len(existing_state.routing_table_peers) > 0:
state.routing_table_peers = existing_state.routing_table_peers
print(f'rt empty on save, but old rt was recovered ({len(state.routing_table_peers)} peers)')
with open(state_dir + node_id, 'wb') as f:
# pprint(state.routing_table_peers)
# pprint(state.datastore)
print(f'{node_id[:8]}: saved {len(state.routing_table_peers)} rt peers, {len(state.datastore)} in store')
pickle.dump(state, f)
await wallet_client.close()
db.close()
class ShutdownErr(BaseException):
pass
def shutdown():
print("got interrupt signal...")
raise ShutdownErr()
def make_node_id(i: int, n: int) -> str:
"""
split dht address space into N chunks and return the first id of the i'th chunk
make_node_id(0,n) returns 000...000 for any n
"""
if not 0 <= i < n:
raise ValueError("i must be between 0 (inclusive) and n (exclusive)")
bytes_in_id = 48
return "{0:0{1}x}".format(i * ((2**8)**bytes_in_id // n), bytes_in_id*2)
async def start_nodes(loop, num_nodes, external_ip, state_dir):
start_port = 4445
known_node_urls = [("lbrynet1.lbry.com", 4444), ("lbrynet2.lbry.com", 4444), ("lbrynet3.lbry.com", 4444)]
peer_manager = peer.PeerManager(loop)
nodes = []
for i in range(num_nodes):
node_id = make_node_id(i, num_nodes)
# pprint(node_id)
port = start_port + i
# await u.get_next_mapping(port, "UDP", "lbry dht tracker") # not necessary, i just opened ports in router
n = node.Node(loop, peer_manager, node_id=bytes.fromhex(node_id), external_ip=external_ip,
udp_port=port, internal_udp_port=port, peer_port=3333)
persisted_peers = []
if path.exists(state_dir + node_id):
with open(state_dir + node_id, 'rb') as f:
state = pickle.load(f)
# pprint(state.routing_table_peers)
# pprint(state.datastore)
print(f'{node_id[:8]}: loaded {len(state.routing_table_peers)} rt peers, {len(state.datastore)} in store')
n.load_state(state)
persisted_peers = state.routing_table_peers
# if len(persisted_peers) == 0 and len(state.datastore) > 0:
# peers_to_import = map(lambda p: (p[0], p[1], p[2], p[3]), n.get_state().datastore)
# persisted_peers.extend(peers_to_import)
# print(f'{node_id[:8]}: rt is empty but we recovered {len(state.datastore)} '
# f'peers from the datastore. {len(peers_to_import)} of those were recent enough to import')
n.start("0.0.0.0", known_node_urls, persisted_peers)
nodes.append(n)
return nodes
async def drain_events(n, q):
# print(f'drain started on {bytes.hex(n.protocol.node_id)[:8]}')
while True:
(node_id, ip, method, args) = await n.protocol.event_queue.get()
try:
q.put_nowait((n, node_id, ip, method, args))
except asyncio.QueueFull:
pass
async def run_web_api(loop, db, wallet_client):
try:
app = web.Application(loop=loop)
app['db'] = db
app['wallet_client'] = wallet_client
app.add_routes([
web.get('/', api_handler),
web.get('/seeds/hash/{hash}', seeds_handler),
web.get('/seeds/url/{url}', url_handler),
])
# server = web.Server(api_handler, loop=loop)
# runner = web.ServerRunner(server)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, 'localhost', 8080)
await site.start()
except Exception as err:
pprint(err)
await runner.cleanup()
async def seeds_handler(request):
blobhash = request.match_info['hash']
db = request.app['db']
try:
cur = db.cursor()
c = cur.execute("""
select count(distinct(node_id)) from announce where hash = ? and timestamp > strftime('%s','now','-1 day')
""", (blobhash,)).fetchone()[0]
cur.close()
return web.Response(text=json.dumps({'seeds': c})+"\n")
except Exception as err:
return web.Response(text=json.dumps({'error': err})+"\n")
async def url_handler(request):
url = unquote(request.match_info['url'])
log.warning(url)
db = request.app['db']
wallet_client = request.app['wallet_client']
try:
sd_hash = await get_sd_hash(wallet_client, url)
if sd_hash is None:
return web.Response(text=json.dumps({'error': 'Could not get sd hash for url', 'url': url})+"\n")
seeds = get_seeds(db, sd_hash)
return web.Response(text=json.dumps({'url': url, 'sd_hash': sd_hash, 'seeds': seeds})+"\n")
except Exception as err:
return web.Response(text=json.dumps({'error': err})+"\n")
def get_seeds(db, blobhash):
cur = db.cursor()
c = cur.execute(
"select count(distinct(node_id)) from announce where hash = ? and timestamp > strftime('%s','now','-1 day')",
(blobhash,)
).fetchone()[0]
cur.close()
return c
async def get_sd_hash(wallet_client, url):
try:
resolved_txos = Outputs.from_base64(await wallet_client.send_request('blockchain.claimtrie.resolve', [url]))
if not resolved_txos.txos:
return None
raw_txs = await wallet_client.send_request('blockchain.transaction.get_batch', [txid for (txid, height) in resolved_txos.txs])
txo_proto = resolved_txos.txos[0]
txid = txo_proto.tx_hash[::-1].hex()
raw_tx_hex, _ = raw_txs[txid]
txo = Transaction(bytes.fromhex(raw_tx_hex)).outputs[txo_proto.nout]
return txo.claim.stream.source.sd_hash
except Exception as err: # claim is not a stream, stream has no source, protobuf err, etc
if isinstance(err, asyncio.CancelledError):
raise err
log.exception("failed to get sd_hash")
return None
async def api_handler(request):
return web.Response(text="tracker OK")
if __name__ == "__main__":
try:
asyncio.run(main())
except ShutdownErr:
pass
@@ -9,32 +9,17 @@ class BlockchainReorganizationTests(CommandTestCase):
VERBOSITY = logging.WARN
async def assertBlockHash(self, height):
bp = self.conductor.spv_node.server.bp
def get_txids():
return [
bp.db.fs_tx_hash(tx_num)[0][::-1].hex()
for tx_num in range(bp.db.tx_counts[height - 1], bp.db.tx_counts[height])
]
block_hash = await self.blockchain.get_block_hash(height)
self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode())
self.assertEqual(block_hash, (await bp.db.fs_block_hashes(height, 1))[0][::-1].hex())
txids = await asyncio.get_event_loop().run_in_executor(bp.db.executor, get_txids)
txs = await bp.db.fs_transactions(txids)
block_txs = (await bp.daemon.deserialised_block(block_hash))['tx']
self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions')
self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order')
self.assertEqual(
(await self.ledger.headers.hash(height)).decode(),
await self.blockchain.get_block_hash(height)
)
async def test_reorg(self):
bp = self.conductor.spv_node.server.bp
bp.reorg_count_metric.set(0)
# invalidate current block, move forward 2
height = 206
self.assertEqual(self.ledger.headers.height, height)
await self.assertBlockHash(height)
self.assertEqual(self.ledger.headers.height, 206)
await self.assertBlockHash(206)
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.generate(2)
await self.ledger.on_header.where(lambda e: e.height == 207)
@@ -52,11 +37,6 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.assertBlockHash(207)
await self.assertBlockHash(208)
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
await self.blockchain.generate(3)
await self.ledger.on_header.where(lambda e: e.height == 211)
await self.assertBlockHash(209)
await self.assertBlockHash(210)
await self.assertBlockHash(211)
async def test_reorg_change_claim_height(self):
# sanity check
@@ -71,8 +51,6 @@ class BlockchainReorganizationTests(CommandTestCase):
# create a claim and verify it's returned by claim_search
self.assertEqual(self.ledger.headers.height, 207)
await self.assertBlockHash(207)
broadcast_tx = await self.daemon.jsonrpc_stream_create(
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
@@ -80,8 +58,6 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.generate(1)
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
self.assertEqual(self.ledger.headers.height, 208)
await self.assertBlockHash(208)
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
txo = txos[0]
@@ -99,18 +75,17 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.blockchain.clear_mempool()
await self.blockchain.generate(2)
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
# verify the claim was dropped from block 208 as returned by lbrycrdd
reorg_block_hash = await self.blockchain.get_block_hash(208)
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
block_207 = await self.blockchain.get_block(reorg_block_hash)
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
@@ -6,7 +6,6 @@ from binascii import unhexlify
from urllib.request import urlopen
from lbry.error import InsufficientFundsError
from lbry.extras.daemon.comment_client import verify
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
from lbry.testcase import CommandTestCase
@@ -1005,18 +1004,6 @@ class ChannelCommands(CommandTestCase):
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2)
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1)
async def test_sign_hex_encoded_data(self):
data_to_sign = "CAFEBABE"
# claim new name
await self.channel_create('@someotherchan')
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
await self.confirm_tx(channel_tx.id)
channel = channel_tx.outputs[0]
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
async def test_channel_export_import_before_sending_channel(self):
# export
tx = await self.channel_create('@foo', '1.0')
@@ -117,6 +117,11 @@ class ResolveCommand(BaseResolveTestCase):
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo*1', claim_id1)
await self.assertResolvesToClaimId('foo*2', claim_id2)
await self.assertResolvesToClaimId('foo*3', claim_id3)
await self.assertResolvesToClaimId('foo*4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
@@ -138,8 +138,8 @@ class BasicTransactionTests(IntegrationTestCase):
# evil trick: mempool is unsorted on real life, but same order between python instances. reproduce it
original_summary = self.conductor.spv_node.server.mempool.transaction_summaries
def random_summary(*args, **kwargs):
summary = original_summary(*args, **kwargs)
async def random_summary(*args, **kwargs):
summary = await original_summary(*args, **kwargs)
if summary and len(summary) > 2:
ordered = summary.copy()
while summary == ordered:
@@ -10,7 +10,7 @@ from lbry.wallet.dewies import dict_values_to_lbc
class WalletCommands(CommandTestCase):
async def test_wallet_create_and_add_subscribe(self):
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions.values()))
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions))
self.assertEqual(len(session.hashX_subs), 27)
wallet = await self.daemon.jsonrpc_wallet_create('foo', create_account=True, single_key=True)
self.assertEqual(len(session.hashX_subs), 28)
@@ -4,9 +4,9 @@ from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateM
class TestExchangeRateManager(AsyncioTestCase):
async def test_exchange_rate_manager(self):
# TODO: re-enable cryptonator.com
# TODO: this uses real exchange rate feeds... update to use mocks
manager = ExchangeRateManager(FEEDS)
manager.start()
self.addCleanup(manager.stop)
@@ -18,5 +18,5 @@ class TestExchangeRateManager(AsyncioTestCase):
self.assertTrue(feed.is_online)
self.assertIsInstance(feed.rate, ExchangeRate)
lbc = manager.convert_currency('USD', 'LBC', Decimal('0.01'))
self.assertGreaterEqual(lbc, 0.01)
self.assertGreaterEqual(lbc, 0.1)
self.assertLessEqual(lbc, 10.0)
@@ -36,14 +36,12 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
fixed_peers=[])
self.server_config.transaction_cache_size = 10000
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir,
wallet=self.client_wallet_dir, fixed_peers=[])
self.client_config.transaction_cache_size = 10000
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
+8 -2
View File
@@ -9,7 +9,7 @@ claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
class TestURLParsing(unittest.TestCase):
segments = 'stream', 'channel'
fields = 'name', 'claim_id', 'amount_order'
fields = 'name', 'claim_id', 'sequence', 'amount_order'
def _assert_url(self, url_string, strictly=True, **kwargs):
url = URL.parse(url_string)
@@ -43,22 +43,25 @@ class TestURLParsing(unittest.TestCase):
url = self._assert_url
# stream
url('test', stream_name='test')
url('test*1', stream_name='test*1')
url('test*1', stream_name='test', stream_sequence='1')
url('test$1', stream_name='test', stream_amount_order='1')
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False)
url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id)
# channel
url('@test', channel_name='@test')
url('@test*1', channel_name='@test', channel_sequence='1')
url('@test$1', channel_name='@test', channel_amount_order='1')
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False)
url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id)
# channel/stream
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
url('lbry://@test*1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False)
url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# combined legacy and new
url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
url('@test*1/stuff#2', channel_sequence='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\uD799')
@@ -108,8 +111,10 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://test@")
fail("lbry://tes@t")
fail(f"lbry://test:1#{claim_id}")
fail("lbry://test*0")
fail("lbry://test$0")
fail("lbry://test/path")
fail("lbry://@test1*1ab/fakepath")
fail("lbry://test:1:1:1")
fail("whatever/lbry://test")
fail("lbry://lbry://test")
@@ -117,4 +122,5 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://abc:0x123")
fail("lbry://abc:0x123/page")
fail("lbry://@test1#ABCDEF/fakepath")
fail("test*0001")
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
+11 -8
View File
@@ -93,7 +93,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
@@ -113,8 +114,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -136,8 +137,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash], sent)
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -159,7 +160,9 @@ class TestReflector(AsyncioTestCase):
await not_incoming.wait()
await incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
with self.assertRaises(asyncio.CancelledError):
await reflect_task
# self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
@@ -184,8 +187,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
+1 -3
View File
@@ -98,14 +98,12 @@ async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
wallet = Wallet()
ledger = Ledger({
'db': Database(os.path.join(wallet_dir, 'blockchain.db')),
'headers': FakeHeaders(514082),
'tx_cache_size': 10000
'headers': FakeHeaders(514082)
})
await ledger.db.open()
wallet.generate_account(ledger)
manager = WalletManager()
manager.config = Config()
manager.config.transaction_cache_size = 10000
manager.wallets.append(wallet)
manager.ledgers[Ledger] = ledger
manager.ledger.network.client = ClientSession(
+4 -2
View File
@@ -40,7 +40,7 @@ class MockNetwork:
merkle = await self.get_merkle(tx_hash, known_height)
return tx, merkle
async def get_transaction_batch(self, txids, restricted):
async def get_transaction_batch(self, txids):
return {
txid: await self.get_transaction_and_merkle(txid)
for txid in txids
@@ -126,7 +126,9 @@ class TestSynchronization(LedgerTestCase):
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
self.assertEqual(0, len(self.ledger._tx_cache))
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])