Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
715451b5fb | ||
|
|
87f1895405 | ||
|
|
923d817751 | ||
|
|
0728209b66 | ||
|
|
b8b9dcc2ee | ||
|
|
f35e879852 | ||
|
|
34f4f12eb9 | ||
|
|
fa63bf758d | ||
|
|
f6b396ae64 | ||
|
|
2c7fd58e34 | ||
|
|
982f2c9634 | ||
|
|
f2fd42b47a | ||
|
|
1b4ccad938 | ||
|
|
a9de1ce8e0 | ||
|
|
ac752d5ec2 | ||
|
|
632d8d02d2 | ||
|
|
48aeb26e02 | ||
|
|
1694af8b5e | ||
|
|
83bcab9cd2 | ||
|
|
bdc7f4b3f5 | ||
|
|
39202a3d79 | ||
|
|
912065a121 | ||
|
|
c8466afac2 |
No files matched your search
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.84.0"
|
||||
__version__ = "0.85.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+1
-8
@@ -6,7 +6,7 @@ from typing import NamedTuple, Tuple
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%*?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
@@ -23,7 +23,6 @@ def _create_url_regex():
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(r'\*' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
||||
) + '?'
|
||||
)
|
||||
@@ -50,7 +49,6 @@ def normalize_name(name):
|
||||
class PathSegment(NamedTuple):
|
||||
name: str
|
||||
claim_id: str = None
|
||||
sequence: int = None
|
||||
amount_order: int = None
|
||||
|
||||
@property
|
||||
@@ -61,8 +59,6 @@ class PathSegment(NamedTuple):
|
||||
q = {'name': self.name}
|
||||
if self.claim_id is not None:
|
||||
q['claim_id'] = self.claim_id
|
||||
if self.sequence is not None:
|
||||
q['sequence'] = self.sequence
|
||||
if self.amount_order is not None:
|
||||
q['amount_order'] = self.amount_order
|
||||
return q
|
||||
@@ -70,8 +66,6 @@ class PathSegment(NamedTuple):
|
||||
def __str__(self):
|
||||
if self.claim_id is not None:
|
||||
return f"{self.name}:{self.claim_id}"
|
||||
elif self.sequence is not None:
|
||||
return f"{self.name}*{self.sequence}"
|
||||
elif self.amount_order is not None:
|
||||
return f"{self.name}${self.amount_order}"
|
||||
return self.name
|
||||
@@ -118,7 +112,6 @@ class URL(NamedTuple):
|
||||
segments[segment] = PathSegment(
|
||||
parts[f'{segment}_name'],
|
||||
parts[f'{segment}_claim_id'],
|
||||
parts[f'{segment}_sequence'],
|
||||
parts[f'{segment}_amount_order']
|
||||
)
|
||||
|
||||
|
||||
@@ -121,7 +121,12 @@ class AIOSQLite:
|
||||
if self._closing:
|
||||
return
|
||||
self._closing = True
|
||||
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
|
||||
|
||||
def __checkpoint_and_close(conn: sqlite3.Connection):
|
||||
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
|
||||
conn.close()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, __checkpoint_and_close, self.writer_connection)
|
||||
self.writer_executor.shutdown(wait=True)
|
||||
self.reader_executor.shutdown(wait=True)
|
||||
self.read_ready.clear()
|
||||
|
||||
+22
-75
@@ -637,40 +637,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||
return True
|
||||
|
||||
async def cache_transaction(self, txid, remote_height, check_local=True):
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = self._tx_cache[txid] = TransactionCacheItem()
|
||||
elif cache_item.tx is not None and \
|
||||
cache_item.tx.height >= remote_height and \
|
||||
(cache_item.tx.is_verified or remote_height < 1):
|
||||
return cache_item.tx # cached tx is already up-to-date
|
||||
|
||||
cache_item.pending_verifications += 1
|
||||
try:
|
||||
async with cache_item.lock:
|
||||
tx = cache_item.tx
|
||||
if tx is None and check_local:
|
||||
# check local db
|
||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
||||
merkle = None
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw, merkle = await self.network.retriable_call(
|
||||
self.network.get_transaction_and_merkle, txid, remote_height
|
||||
)
|
||||
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
tx.height = remote_height
|
||||
if merkle and 0 < remote_height < len(self.headers):
|
||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||
header = await self.headers.get(remote_height)
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
return tx
|
||||
finally:
|
||||
cache_item.pending_verifications -= 1
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||
tx.height = remote_height
|
||||
cached = self._tx_cache.get(tx.id)
|
||||
@@ -688,7 +654,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
|
||||
async def request_transactions_for_inflate(self, to_request: Tuple[Tuple[str, int], ...], session_override=None):
|
||||
async def request_transactions_for_inflate(self, to_request: Tuple[Tuple[str, int], ...]):
|
||||
header_cache = {}
|
||||
batches = [[]]
|
||||
remote_heights = {}
|
||||
@@ -709,12 +675,9 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
batches.pop()
|
||||
|
||||
async def _single_batch(batch):
|
||||
if session_override:
|
||||
batch_result = await self.network.get_transaction_batch(
|
||||
batch, restricted=False, session=session_override
|
||||
)
|
||||
else:
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
|
||||
batch_result = await self.network.retriable_call(
|
||||
self.network.get_transaction_batch, batch, restricted=False
|
||||
)
|
||||
for txid, (raw, merkle) in batch_result.items():
|
||||
remote_height = remote_heights[txid]
|
||||
merkle_height = merkle['block_height']
|
||||
@@ -722,7 +685,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if cache_item is None:
|
||||
cache_item = TransactionCacheItem()
|
||||
self._tx_cache[txid] = cache_item
|
||||
tx = cache_item.tx or Transaction(unhexlify(raw), height=remote_height)
|
||||
tx = cache_item.tx or Transaction(bytes.fromhex(raw), height=remote_height)
|
||||
tx.height = remote_height
|
||||
cache_item.tx = tx
|
||||
if 'merkle' in merkle and remote_heights[txid] > 0:
|
||||
@@ -898,17 +861,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
include_is_my_output=False,
|
||||
include_sent_supports=False,
|
||||
include_sent_tips=False,
|
||||
include_received_tips=False,
|
||||
session_override=None) -> Tuple[List[Output], dict, int, int]:
|
||||
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs = []
|
||||
if len(outputs.txs) > 0:
|
||||
txs: List[Transaction] = []
|
||||
if session_override:
|
||||
txs.extend((await self.request_transactions_for_inflate(tuple(outputs.txs), session_override)))
|
||||
else:
|
||||
txs.extend((await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))))
|
||||
txs.extend((await self.request_transactions_for_inflate(tuple(outputs.txs))))
|
||||
|
||||
_txos, blocked = outputs.inflate(txs)
|
||||
|
||||
@@ -983,25 +942,17 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
|
||||
txos = []
|
||||
urls_copy = list(urls)
|
||||
|
||||
if new_sdk_server:
|
||||
resolve = partial(self.network.new_resolve, new_sdk_server)
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:500], urls_copy[500:]
|
||||
txos.extend(
|
||||
(await self._inflate_outputs(
|
||||
resolve(batch), accounts, **kwargs
|
||||
))[0]
|
||||
)
|
||||
else:
|
||||
async with self.network.single_call_context(self.network.resolve) as (resolve, session):
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:500], urls_copy[500:]
|
||||
txos.extend(
|
||||
(await self._inflate_outputs(
|
||||
resolve(batch), accounts, session_override=session, **kwargs
|
||||
))[0]
|
||||
)
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:100], urls_copy[100:]
|
||||
txos.extend(
|
||||
(await self._inflate_outputs(
|
||||
resolve(batch), accounts, **kwargs
|
||||
))[0]
|
||||
)
|
||||
|
||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||
result = {}
|
||||
@@ -1023,17 +974,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||
if new_sdk_server:
|
||||
claim_search = partial(self.network.new_claim_search, new_sdk_server)
|
||||
return await self._inflate_outputs(
|
||||
claim_search(**kwargs), accounts,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output,
|
||||
)
|
||||
async with self.network.single_call_context(self.network.claim_search) as (claim_search, session):
|
||||
return await self._inflate_outputs(
|
||||
claim_search(**kwargs), accounts, session_override=session,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output,
|
||||
)
|
||||
else:
|
||||
claim_search = self.network.claim_search
|
||||
return await self._inflate_outputs(
|
||||
claim_search(**kwargs), accounts,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output,
|
||||
)
|
||||
|
||||
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
|
||||
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
|
||||
|
||||
@@ -3,10 +3,7 @@ import asyncio
|
||||
import json
|
||||
from time import perf_counter
|
||||
from operator import itemgetter
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import partial
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
from lbry import __version__
|
||||
@@ -255,28 +252,6 @@ class Network:
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
@asynccontextmanager
|
||||
async def single_call_context(self, function, *args, **kwargs):
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
server = self.session_pool.fastest_session.server
|
||||
session = ClientSession(network=self, server=server)
|
||||
|
||||
async def call_with_reconnect(*a, **kw):
|
||||
while self.running:
|
||||
if not session.available:
|
||||
await session.create_connection()
|
||||
try:
|
||||
return await partial(function, *args, session_override=session, **kwargs)(*a, **kw)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("'%s' failed, retrying", function.__name__)
|
||||
try:
|
||||
yield (call_with_reconnect, session)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
|
||||
@@ -171,7 +171,8 @@ class BlockProcessor:
|
||||
|
||||
# Caches of unflushed items.
|
||||
self.headers = []
|
||||
self.tx_hashes = []
|
||||
self.block_hashes = []
|
||||
self.block_txs = []
|
||||
self.undo_infos = []
|
||||
|
||||
# UTXO cache
|
||||
@@ -336,8 +337,8 @@ class BlockProcessor:
|
||||
def flush_data(self):
|
||||
"""The data for a flush. The lock must be taken."""
|
||||
assert self.state_lock.locked()
|
||||
return FlushData(self.height, self.tx_count, self.headers,
|
||||
self.tx_hashes, self.undo_infos, self.utxo_cache,
|
||||
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
|
||||
self.block_txs, self.undo_infos, self.utxo_cache,
|
||||
self.db_deletes, self.tip)
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
@@ -392,7 +393,8 @@ class BlockProcessor:
|
||||
for block in blocks:
|
||||
height += 1
|
||||
undo_info = self.advance_txs(
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height)
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height),
|
||||
self.coin.header_hash(block.header)
|
||||
)
|
||||
if height >= min_height:
|
||||
self.undo_infos.append((undo_info, height))
|
||||
@@ -403,25 +405,26 @@ class BlockProcessor:
|
||||
self.headers.extend(headers)
|
||||
self.tip = self.coin.header_hash(headers[-1])
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
self.block_hashes.append(block_hash)
|
||||
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
undo_info = []
|
||||
tx_num = self.tx_count
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
s_pack = pack
|
||||
hashXs_by_tx = []
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
undo_info_append = undo_info.append
|
||||
update_touched = self.touched.update
|
||||
hashXs_by_tx = []
|
||||
append_hashXs = hashXs_by_tx.append
|
||||
append_hashX_by_tx = hashXs_by_tx.append
|
||||
hashX_from_script = self.coin.hashX_from_script
|
||||
|
||||
for tx, tx_hash in txs:
|
||||
hashXs = []
|
||||
append_hashX = hashXs.append
|
||||
tx_numb = s_pack('<I', tx_num)
|
||||
tx_numb = pack('<I', tx_num)
|
||||
|
||||
# Spend the inputs
|
||||
for txin in tx.inputs:
|
||||
@@ -434,18 +437,16 @@ class BlockProcessor:
|
||||
# Add the new UTXOs
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Get the hashX. Ignore unspendable outputs
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
hashX = hashX_from_script(txout.pk_script)
|
||||
if hashX:
|
||||
append_hashX(hashX)
|
||||
put_utxo(tx_hash + s_pack('<H', idx),
|
||||
hashX + tx_numb + s_pack('<Q', txout.value))
|
||||
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
|
||||
|
||||
append_hashXs(hashXs)
|
||||
append_hashX_by_tx(hashXs)
|
||||
update_touched(hashXs)
|
||||
tx_num += 1
|
||||
|
||||
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
|
||||
|
||||
self.tx_count = tx_num
|
||||
self.db.tx_counts.append(tx_num)
|
||||
|
||||
@@ -487,20 +488,16 @@ class BlockProcessor:
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
s_pack = pack
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
touched = self.touched
|
||||
undo_entry_len = 12 + HASHX_LEN
|
||||
|
||||
for tx, tx_hash in reversed(txs):
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Spend the TX outputs. Be careful with unspendable
|
||||
# outputs - we didn't save those in the first place.
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
hashX = self.coin.hashX_from_script(txout.pk_script)
|
||||
if hashX:
|
||||
cache_value = spend_utxo(tx_hash, idx)
|
||||
touched.add(cache_value[:-12])
|
||||
cache_value = self.spend_utxo(tx_hash, idx)
|
||||
self.touched.add(cache_value[:-12])
|
||||
|
||||
# Restore the inputs
|
||||
for txin in reversed(tx.inputs):
|
||||
@@ -508,9 +505,8 @@ class BlockProcessor:
|
||||
continue
|
||||
n -= undo_entry_len
|
||||
undo_item = undo_info[n:n + undo_entry_len]
|
||||
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
|
||||
undo_item)
|
||||
touched.add(undo_item[:-12])
|
||||
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
|
||||
self.touched.add(undo_item[:-12])
|
||||
|
||||
assert n == 0
|
||||
self.tx_count -= len(txs)
|
||||
@@ -770,9 +766,9 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
|
||||
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
|
||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
||||
self.timer.show(height=height)
|
||||
|
||||
@@ -37,7 +37,9 @@ class Env:
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
|
||||
self.trending_algorithms = [
|
||||
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
|
||||
]
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
|
||||
+173
-74
@@ -14,10 +14,11 @@ import array
|
||||
import ast
|
||||
import os
|
||||
import time
|
||||
import zlib
|
||||
import base64
|
||||
from asyncio import sleep
|
||||
from bisect import bisect_right
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
@@ -35,6 +36,10 @@ UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
|
||||
HEADER_PREFIX = b'H'
|
||||
TX_COUNT_PREFIX = b'T'
|
||||
TX_HASH_PREFIX = b'X'
|
||||
TX_PREFIX = b'B'
|
||||
TX_NUM_PREFIX = b'N'
|
||||
BLOCK_HASH_PREFIX = b'C'
|
||||
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
@@ -42,7 +47,8 @@ class FlushData:
|
||||
height = attr.ib()
|
||||
tx_count = attr.ib()
|
||||
headers = attr.ib()
|
||||
block_tx_hashes = attr.ib()
|
||||
block_hashes = attr.ib()
|
||||
block_txs = attr.ib()
|
||||
# The following are flushed to the UTXO DB if undo_infos is not None
|
||||
undo_infos = attr.ib()
|
||||
adds = attr.ib()
|
||||
@@ -74,6 +80,7 @@ class LevelDB:
|
||||
self.history = History()
|
||||
self.utxo_db = None
|
||||
self.tx_counts = None
|
||||
self.headers = None
|
||||
self.last_flush = time.time()
|
||||
|
||||
self.logger.info(f'using {self.env.db_engine} for DB backend')
|
||||
@@ -82,9 +89,8 @@ class LevelDB:
|
||||
self.merkle = Merkle()
|
||||
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
|
||||
|
||||
self.hashes_db = None
|
||||
self.headers_db = None
|
||||
self.tx_count_db = None
|
||||
self.tx_db = None
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
if self.tx_counts is not None:
|
||||
@@ -95,7 +101,7 @@ class LevelDB:
|
||||
def get_counts():
|
||||
return tuple(
|
||||
util.unpack_be_uint64(tx_count)
|
||||
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
|
||||
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
|
||||
)
|
||||
|
||||
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
|
||||
@@ -108,6 +114,19 @@ class LevelDB:
|
||||
else:
|
||||
assert self.db_tx_count == 0
|
||||
|
||||
async def _read_headers(self):
|
||||
if self.headers is not None:
|
||||
return
|
||||
|
||||
def get_headers():
|
||||
return [
|
||||
header for header in self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False)
|
||||
]
|
||||
|
||||
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
|
||||
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
|
||||
self.headers = headers
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
if self.executor is None:
|
||||
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
|
||||
@@ -123,17 +142,11 @@ class LevelDB:
|
||||
self.logger.info('created new headers db')
|
||||
self.logger.info(f'opened headers DB (for sync: {for_sync})')
|
||||
|
||||
assert self.tx_count_db is None
|
||||
self.tx_count_db = self.db_class('tx_count', for_sync)
|
||||
if self.tx_count_db.is_new:
|
||||
self.logger.info('created new tx count db')
|
||||
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
|
||||
|
||||
assert self.hashes_db is None
|
||||
self.hashes_db = self.db_class('hashes', for_sync)
|
||||
if self.hashes_db.is_new:
|
||||
self.logger.info('created new tx hashes db')
|
||||
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
|
||||
assert self.tx_db is None
|
||||
self.tx_db = self.db_class('tx', for_sync)
|
||||
if self.tx_db.is_new:
|
||||
self.logger.info('created new tx db')
|
||||
self.logger.info(f'opened tx DB (for sync: {for_sync})')
|
||||
|
||||
assert self.utxo_db is None
|
||||
# First UTXO DB
|
||||
@@ -151,13 +164,13 @@ class LevelDB:
|
||||
|
||||
# Read TX counts (requires meta directory)
|
||||
await self._read_tx_counts()
|
||||
await self._read_headers()
|
||||
|
||||
def close(self):
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.headers_db.close()
|
||||
self.tx_count_db.close()
|
||||
self.hashes_db.close()
|
||||
self.tx_db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
self.executor = None
|
||||
|
||||
@@ -187,12 +200,9 @@ class LevelDB:
|
||||
if self.headers_db:
|
||||
self.headers_db.close()
|
||||
self.headers_db = None
|
||||
if self.tx_count_db:
|
||||
self.tx_count_db.close()
|
||||
self.tx_count_db = None
|
||||
if self.hashes_db:
|
||||
self.hashes_db.close()
|
||||
self.hashes_db = None
|
||||
if self.tx_db:
|
||||
self.tx_db.close()
|
||||
self.tx_db = None
|
||||
|
||||
await self._open_dbs(False, False)
|
||||
self.logger.info("opened for serving")
|
||||
@@ -217,7 +227,7 @@ class LevelDB:
|
||||
assert flush_data.height == self.fs_height == self.db_height
|
||||
assert flush_data.tip == self.db_tip
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert not flush_data.block_txs
|
||||
assert not flush_data.adds
|
||||
assert not flush_data.deletes
|
||||
assert not flush_data.undo_infos
|
||||
@@ -275,33 +285,48 @@ class LevelDB:
|
||||
"""
|
||||
prior_tx_count = (self.tx_counts[self.fs_height]
|
||||
if self.fs_height >= 0 else 0)
|
||||
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
|
||||
assert len(flush_data.block_txs) == len(flush_data.headers)
|
||||
assert flush_data.height == self.fs_height + len(flush_data.headers)
|
||||
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
|
||||
else 0)
|
||||
assert len(self.tx_counts) == flush_data.height + 1
|
||||
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
|
||||
assert len(
|
||||
b''.join(hashes for hashes, _ in flush_data.block_txs)
|
||||
) // 32 == flush_data.tx_count - prior_tx_count
|
||||
|
||||
# Write the headers, tx counts, and tx hashes
|
||||
# Write the headers
|
||||
start_time = time.perf_counter()
|
||||
|
||||
with self.headers_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for i, header in enumerate(flush_data.headers):
|
||||
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
|
||||
self.headers.append(header)
|
||||
flush_data.headers.clear()
|
||||
|
||||
height_start = self.fs_height + 1
|
||||
tx_num = prior_tx_count
|
||||
|
||||
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
|
||||
tx_count = self.tx_counts[height_start]
|
||||
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
|
||||
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
|
||||
height_start += 1
|
||||
offset = 0
|
||||
while offset < len(tx_hashes):
|
||||
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
|
||||
tx_num += 1
|
||||
offset += 32
|
||||
with self.tx_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
|
||||
tx_count = self.tx_counts[height_start]
|
||||
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
|
||||
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
|
||||
height_start += 1
|
||||
offset = 0
|
||||
while offset < len(tx_hashes):
|
||||
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
|
||||
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
|
||||
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
|
||||
tx_num += 1
|
||||
offset += 32
|
||||
|
||||
flush_data.block_txs.clear()
|
||||
flush_data.block_hashes.clear()
|
||||
|
||||
flush_data.block_tx_hashes.clear()
|
||||
self.fs_height = flush_data.height
|
||||
self.fs_tx_count = flush_data.tx_count
|
||||
flush_data.headers.clear()
|
||||
elapsed = time.perf_counter() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
|
||||
@@ -362,7 +387,7 @@ class LevelDB:
|
||||
def flush_backup(self, flush_data, touched):
|
||||
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert not flush_data.block_txs
|
||||
assert flush_data.height < self.db_height
|
||||
self.history.assert_flushed()
|
||||
|
||||
@@ -383,7 +408,9 @@ class LevelDB:
|
||||
|
||||
def backup_fs(self, height, tx_count):
|
||||
"""Back up during a reorg. This just updates our pointers."""
|
||||
self.fs_height = height
|
||||
while self.fs_height > height:
|
||||
self.fs_height -= 1
|
||||
self.headers.pop()
|
||||
self.fs_tx_count = tx_count
|
||||
# Truncate header_mc: header count is 1 more than the height.
|
||||
self.header_mc.truncate(height + 1)
|
||||
@@ -395,7 +422,7 @@ class LevelDB:
|
||||
raise IndexError(f'height {height:,d} out of range')
|
||||
return header
|
||||
|
||||
async def read_headers(self, start_height, count):
|
||||
async def read_headers(self, start_height, count, b16=False, b64=False):
|
||||
"""Requires start_height >= 0, count >= 0. Reads as many headers as
|
||||
are available starting at start_height up to count. This
|
||||
would be zero if start_height is beyond self.db_height, for
|
||||
@@ -404,6 +431,7 @@ class LevelDB:
|
||||
Returns a (binary, n) pair where binary is the concatenated
|
||||
binary headers, and n is the count of headers returned.
|
||||
"""
|
||||
|
||||
if start_height < 0 or count < 0:
|
||||
raise self.DBError(f'{count:,d} headers starting at '
|
||||
f'{start_height:,d} not on disk')
|
||||
@@ -412,15 +440,19 @@ class LevelDB:
|
||||
# Read some from disk
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
return b''.join(
|
||||
self.headers_db.iterator(
|
||||
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
|
||||
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
|
||||
include_key=False
|
||||
)
|
||||
), disk_count
|
||||
headers = b''.join(self.headers[start_height:start_height+disk_count])
|
||||
if b16:
|
||||
return headers.hex().encode(), disk_count
|
||||
elif b64:
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
return base64.b64encode(compressobj.compress(headers) + compressobj.flush()), disk_count
|
||||
return headers, disk_count
|
||||
return b'', 0
|
||||
|
||||
if not b16 and not b64:
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
return b''.join(header for header in self.headers[start_height:start_height + disk_count]), disk_count
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
|
||||
|
||||
def fs_tx_hash(self, tx_num):
|
||||
@@ -431,20 +463,57 @@ class LevelDB:
|
||||
if tx_height > self.db_height:
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
tx_hash = self.tx_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
return tx_hash, tx_height
|
||||
|
||||
async def fs_block_hashes(self, height, count):
|
||||
headers_concat, headers_count = await self.read_headers(height, count)
|
||||
if headers_count != count:
|
||||
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
offset = 0
|
||||
headers = []
|
||||
for n in range(count):
|
||||
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
|
||||
offset += self.coin.BASIC_HEADER_SIZE
|
||||
def _fs_transactions(self, txids):
|
||||
def _iter_transactions():
|
||||
block_txs = {}
|
||||
branch_and_root = self.merkle.branch_and_root
|
||||
tx_iterator = self.tx_db.iterator
|
||||
tx_counts = self.tx_counts
|
||||
tx_db_get = self.tx_db.get
|
||||
unpack_be_uint64 = util.unpack_be_uint64
|
||||
pack_be_uint64 = util.pack_be_uint64
|
||||
|
||||
return [self.coin.header_hash(header) for header in headers]
|
||||
for tx_hash in txids:
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
|
||||
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
|
||||
if tx_num is not None:
|
||||
tx_num = unpack_be_uint64(tx_num)
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
else:
|
||||
yield tx_hash, (None, {'block_height': -1})
|
||||
continue
|
||||
if tx_height >= self.db_height:
|
||||
yield tx_hash, (None, {'block_height': -1})
|
||||
continue
|
||||
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
|
||||
if tx_height not in block_txs:
|
||||
block_txs[tx_height] = list(tx_iterator(
|
||||
start=TX_HASH_PREFIX + pack_be_uint64(tx_counts[tx_height - 1]),
|
||||
stop=None if tx_height + 1 == len(tx_counts) else
|
||||
TX_HASH_PREFIX + pack_be_uint64(tx_counts[tx_height] + 1), include_key=False
|
||||
))
|
||||
tx_pos = tx_num - tx_counts[tx_height - 1]
|
||||
branch, root = branch_and_root(block_txs[tx_height], tx_pos)
|
||||
merkle = {
|
||||
'block_height': tx_height,
|
||||
'merkle': [hash_to_hex_str(hash) for hash in branch],
|
||||
'pos': tx_pos
|
||||
}
|
||||
yield tx_hash, (None if not tx else tx.hex(), merkle)
|
||||
return {
|
||||
_tx_hash: _val for (_tx_hash, _val) in _iter_transactions()
|
||||
}
|
||||
|
||||
async def fs_transactions(self, txids):
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
|
||||
|
||||
async def fs_block_hashes(self, height, count):
|
||||
if height + count > len(self.headers):
|
||||
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
|
||||
|
||||
async def limited_history(self, hashX, *, limit=1000):
|
||||
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
|
||||
@@ -453,25 +522,54 @@ class LevelDB:
|
||||
transactions. By default returns at most 1000 entries. Set
|
||||
limit to None to get them all.
|
||||
"""
|
||||
def read_history():
|
||||
hashx_history = []
|
||||
# def read_history():
|
||||
# hashx_history = []
|
||||
# for key, hist in self.history.db.iterator(prefix=hashX):
|
||||
# a = array.array('I')
|
||||
# a.frombytes(hist)
|
||||
# for tx_num in a:
|
||||
# tx_height = bisect_right(self.tx_counts, tx_num)
|
||||
# if tx_height > self.db_height:
|
||||
# tx_hash = None
|
||||
# else:
|
||||
# tx_hash = self.tx_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
#
|
||||
# hashx_history.append((tx_hash, tx_height))
|
||||
# if limit and len(hashx_history) >= limit:
|
||||
# return hashx_history
|
||||
# return hashx_history
|
||||
|
||||
def iter_tx_heights():
|
||||
db_height = self.db_height
|
||||
tx_counts = self.tx_counts
|
||||
tx_db_get = self.tx_db.get
|
||||
pack_be_uint64 = util.pack_be_uint64
|
||||
|
||||
cnt = 0
|
||||
|
||||
for key, hist in self.history.db.iterator(prefix=hashX):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
tx_height = bisect_right(self.tx_counts, tx_num)
|
||||
if tx_height > self.db_height:
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
hashx_history.append((tx_hash, tx_height))
|
||||
if limit and len(hashx_history) >= limit:
|
||||
return hashx_history
|
||||
return hashx_history
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
if tx_height > db_height:
|
||||
yield None, tx_height
|
||||
return
|
||||
yield tx_db_get(TX_HASH_PREFIX + pack_be_uint64(tx_num)), tx_height
|
||||
cnt += 1
|
||||
if limit and cnt >= limit:
|
||||
return
|
||||
if limit and cnt >= limit:
|
||||
return
|
||||
|
||||
def read_history():
|
||||
return [
|
||||
(tx_num, tx_height) for (tx_num, tx_height) in iter_tx_heights()
|
||||
]
|
||||
|
||||
while True:
|
||||
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
|
||||
if all(hash is not None for hash, height in history):
|
||||
if not history or history[-1][0] is not None:
|
||||
return history
|
||||
self.logger.warning(f'limited_history: tx hash '
|
||||
f'not found (reorg?), retrying...')
|
||||
@@ -629,13 +727,14 @@ class LevelDB:
|
||||
utxos = []
|
||||
utxos_append = utxos.append
|
||||
s_unpack = unpack
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
# Key: b'u' + address_hashX + tx_idx + tx_num
|
||||
# Value: the UTXO value as a 64-bit unsigned integer
|
||||
prefix = b'u' + hashX
|
||||
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
|
||||
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
|
||||
value, = unpack('<Q', db_value)
|
||||
tx_hash, height = self.fs_tx_hash(tx_num)
|
||||
tx_hash, height = fs_tx_hash(tx_num)
|
||||
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
|
||||
return utxos
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ class Merkle:
|
||||
raise TypeError('index must be an integer')
|
||||
# This also asserts hashes is not empty
|
||||
if not 0 <= index < len(hashes):
|
||||
raise ValueError('index out of range')
|
||||
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
|
||||
natural_length = self.branch_length(len(hashes))
|
||||
if length is None:
|
||||
length = natural_length
|
||||
|
||||
@@ -17,7 +17,7 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from binascii import hexlify, unhexlify
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
@@ -141,7 +141,11 @@ class SessionManager:
|
||||
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version",))
|
||||
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
labelnames=("method", "version"))
|
||||
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
|
||||
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
|
||||
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
|
||||
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
|
||||
|
||||
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
db_operational_error_metric = Counter(
|
||||
@@ -1045,7 +1049,12 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
|
||||
count = len(urls)
|
||||
try:
|
||||
self.session_mgr.urls_to_resolve_count_metric.inc(count)
|
||||
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
|
||||
finally:
|
||||
self.session_mgr.resolved_url_count_metric.inc(count)
|
||||
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
@@ -1218,10 +1227,7 @@ class LBRYElectrumX(SessionBase):
|
||||
return await self.address_status(hashX)
|
||||
|
||||
async def hashX_unsubscribe(self, hashX, alias):
|
||||
try:
|
||||
del self.hashX_subs[hashX]
|
||||
except ValueError:
|
||||
pass
|
||||
self.hashX_subs.pop(hashX, None)
|
||||
|
||||
def address_to_hashX(self, address):
|
||||
try:
|
||||
@@ -1361,11 +1367,9 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
max_size = self.MAX_CHUNK_SIZE
|
||||
count = min(count, max_size)
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
headers, count = await self.db.read_headers(start_height, count, b16=not b64, b64=b64)
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'base64' if b64 else 'hex': headers.decode(),
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
@@ -1523,27 +1527,22 @@ class LBRYElectrumX(SessionBase):
|
||||
f'network rules.\n\n{message}\n[{raw_tx}]')
|
||||
|
||||
async def transaction_info(self, tx_hash: str):
|
||||
assert_tx_hash(tx_hash)
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
block_hash = tx_info.get('blockhash')
|
||||
if not block_hash:
|
||||
return raw_tx, {'block_height': -1}
|
||||
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
|
||||
merkle = await self.transaction_merkle(tx_hash, merkle_height)
|
||||
return raw_tx, merkle
|
||||
return (await self.transaction_get_batch(tx_hash))[tx_hash]
|
||||
|
||||
async def transaction_get_batch(self, *tx_hashes):
|
||||
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
|
||||
if len(tx_hashes) > 100:
|
||||
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
|
||||
for tx_hash in tx_hashes:
|
||||
assert_tx_hash(tx_hash)
|
||||
batch_result = {}
|
||||
batch_result = await self.db.fs_transactions(tx_hashes)
|
||||
needed_merkles = {}
|
||||
for tx_hash in tx_hashes:
|
||||
if tx_hash in batch_result and batch_result[tx_hash][0]:
|
||||
continue
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
block_hash = tx_info.get('blockhash')
|
||||
merkle = {}
|
||||
if block_hash:
|
||||
block = await self.daemon.deserialised_block(block_hash)
|
||||
height = block['height']
|
||||
@@ -1552,12 +1551,21 @@ class LBRYElectrumX(SessionBase):
|
||||
except ValueError:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
|
||||
merkle["pos"] = pos
|
||||
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
|
||||
else:
|
||||
height = -1
|
||||
merkle['block_height'] = height
|
||||
batch_result[tx_hash] = [raw_tx, merkle]
|
||||
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
|
||||
|
||||
def threaded_get_merkle():
|
||||
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
|
||||
batch_result[tx_hash] = raw_tx, {
|
||||
'merkle': self._get_merkle_branch(block_txs, pos),
|
||||
'pos': pos,
|
||||
'block_height': block_height
|
||||
}
|
||||
if needed_merkles:
|
||||
await asyncio.get_running_loop().run_in_executor(self.db.executor, threaded_get_merkle)
|
||||
|
||||
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
|
||||
return batch_result
|
||||
|
||||
async def transaction_get(self, tx_hash, verbose=False):
|
||||
@@ -1572,19 +1580,6 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
|
||||
|
||||
async def _block_hash_and_tx_hashes(self, height):
|
||||
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
|
||||
the given height.
|
||||
|
||||
block_hash is a hexadecimal string, and tx_hashes is an
|
||||
ordered list of hexadecimal strings.
|
||||
"""
|
||||
height = non_negative_integer(height)
|
||||
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
|
||||
block_hash = hex_hashes[0]
|
||||
block = await self.daemon.deserialised_block(block_hash)
|
||||
return block_hash, block['tx']
|
||||
|
||||
def _get_merkle_branch(self, tx_hashes, tx_pos):
|
||||
"""Return a merkle branch to a transaction.
|
||||
|
||||
@@ -1604,35 +1599,11 @@ class LBRYElectrumX(SessionBase):
|
||||
height: the height of the block it is in
|
||||
"""
|
||||
assert_tx_hash(tx_hash)
|
||||
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
|
||||
try:
|
||||
pos = tx_hashes.index(tx_hash)
|
||||
except ValueError:
|
||||
result = await self.transaction_get_batch(tx_hash)
|
||||
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
branch = self._get_merkle_branch(tx_hashes, pos)
|
||||
return {"block_height": height, "merkle": branch, "pos": pos}
|
||||
|
||||
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
|
||||
"""Return the txid and optionally a merkle proof, given
|
||||
a block height and position in the block.
|
||||
"""
|
||||
tx_pos = non_negative_integer(tx_pos)
|
||||
if merkle not in (True, False):
|
||||
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
|
||||
|
||||
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
|
||||
try:
|
||||
tx_hash = tx_hashes[tx_pos]
|
||||
except IndexError:
|
||||
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
|
||||
if merkle:
|
||||
branch = self._get_merkle_branch(tx_hashes, tx_pos)
|
||||
return {"tx_hash": tx_hash, "merkle": branch}
|
||||
else:
|
||||
return tx_hash
|
||||
f'block at height {height:,d}')
|
||||
return result[tx_hash][1]
|
||||
|
||||
|
||||
class LocalRPC(SessionBase):
|
||||
|
||||
@@ -77,18 +77,16 @@ class LevelDB(Storage):
|
||||
import plyvel
|
||||
cls.module = plyvel
|
||||
|
||||
def open(self, name, create):
|
||||
mof = 512 if self.for_sync else 128
|
||||
def open(self, name, create, lru_cache_size=None):
|
||||
mof = 10000
|
||||
path = os.path.join(self.db_dir, name)
|
||||
# Use snappy compression (the default)
|
||||
self.db = self.module.DB(path, create_if_missing=create,
|
||||
max_open_files=mof)
|
||||
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
|
||||
self.close = self.db.close
|
||||
self.get = self.db.get
|
||||
self.put = self.db.put
|
||||
self.iterator = self.db.iterator
|
||||
self.write_batch = partial(self.db.write_batch, transaction=True,
|
||||
sync=True)
|
||||
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
|
||||
|
||||
|
||||
class RocksDB(Storage):
|
||||
|
||||
@@ -9,17 +9,32 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def assertBlockHash(self, height):
|
||||
self.assertEqual(
|
||||
(await self.ledger.headers.hash(height)).decode(),
|
||||
await self.blockchain.get_block_hash(height)
|
||||
)
|
||||
bp = self.conductor.spv_node.server.bp
|
||||
|
||||
def get_txids():
|
||||
return [
|
||||
bp.db.fs_tx_hash(tx_num)[0][::-1].hex()
|
||||
for tx_num in range(bp.db.tx_counts[height - 1], bp.db.tx_counts[height])
|
||||
]
|
||||
|
||||
block_hash = await self.blockchain.get_block_hash(height)
|
||||
|
||||
self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode())
|
||||
self.assertEqual(block_hash, (await bp.db.fs_block_hashes(height, 1))[0][::-1].hex())
|
||||
|
||||
txids = await asyncio.get_event_loop().run_in_executor(bp.db.executor, get_txids)
|
||||
txs = await bp.db.fs_transactions(txids)
|
||||
block_txs = (await bp.daemon.deserialised_block(block_hash))['tx']
|
||||
self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions')
|
||||
self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order')
|
||||
|
||||
async def test_reorg(self):
|
||||
bp = self.conductor.spv_node.server.bp
|
||||
bp.reorg_count_metric.set(0)
|
||||
# invalidate current block, move forward 2
|
||||
self.assertEqual(self.ledger.headers.height, 206)
|
||||
await self.assertBlockHash(206)
|
||||
height = 206
|
||||
self.assertEqual(self.ledger.headers.height, height)
|
||||
await self.assertBlockHash(height)
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
await self.blockchain.generate(2)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 207)
|
||||
@@ -37,6 +52,11 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
|
||||
await self.blockchain.generate(3)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 211)
|
||||
await self.assertBlockHash(209)
|
||||
await self.assertBlockHash(210)
|
||||
await self.assertBlockHash(211)
|
||||
|
||||
async def test_reorg_change_claim_height(self):
|
||||
# sanity check
|
||||
@@ -51,6 +71,8 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
|
||||
# create a claim and verify it's returned by claim_search
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
await self.assertBlockHash(207)
|
||||
|
||||
broadcast_tx = await self.daemon.jsonrpc_stream_create(
|
||||
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
|
||||
)
|
||||
@@ -58,6 +80,8 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
|
||||
self.assertEqual(self.ledger.headers.height, 208)
|
||||
await self.assertBlockHash(208)
|
||||
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertEqual(1, len(txos))
|
||||
txo = txos[0]
|
||||
@@ -75,17 +99,18 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
await asyncio.wait_for(self.on_header(209), 3.0)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
await self.assertBlockHash(209)
|
||||
|
||||
# verify the claim was dropped from block 208 as returned by lbrycrdd
|
||||
reorg_block_hash = await self.blockchain.get_block_hash(208)
|
||||
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
|
||||
block_207 = await self.blockchain.get_block(reorg_block_hash)
|
||||
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
await asyncio.wait_for(self.on_header(209), 3.0)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
await self.assertBlockHash(209)
|
||||
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
|
||||
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
|
||||
|
||||
|
||||
@@ -117,11 +117,6 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
|
||||
# plain winning claim
|
||||
await self.assertResolvesToClaimId('foo', claim_id3)
|
||||
# sequence resolution
|
||||
await self.assertResolvesToClaimId('foo*1', claim_id1)
|
||||
await self.assertResolvesToClaimId('foo*2', claim_id2)
|
||||
await self.assertResolvesToClaimId('foo*3', claim_id3)
|
||||
await self.assertResolvesToClaimId('foo*4', None)
|
||||
# amount order resolution
|
||||
await self.assertResolvesToClaimId('foo$1', claim_id3)
|
||||
await self.assertResolvesToClaimId('foo$2', claim_id2)
|
||||
|
||||
@@ -9,7 +9,7 @@ claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
|
||||
class TestURLParsing(unittest.TestCase):
|
||||
|
||||
segments = 'stream', 'channel'
|
||||
fields = 'name', 'claim_id', 'sequence', 'amount_order'
|
||||
fields = 'name', 'claim_id', 'amount_order'
|
||||
|
||||
def _assert_url(self, url_string, strictly=True, **kwargs):
|
||||
url = URL.parse(url_string)
|
||||
@@ -43,25 +43,22 @@ class TestURLParsing(unittest.TestCase):
|
||||
url = self._assert_url
|
||||
# stream
|
||||
url('test', stream_name='test')
|
||||
url('test*1', stream_name='test', stream_sequence='1')
|
||||
url('test*1', stream_name='test*1')
|
||||
url('test$1', stream_name='test', stream_amount_order='1')
|
||||
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False)
|
||||
url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id)
|
||||
# channel
|
||||
url('@test', channel_name='@test')
|
||||
url('@test*1', channel_name='@test', channel_sequence='1')
|
||||
url('@test$1', channel_name='@test', channel_amount_order='1')
|
||||
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False)
|
||||
url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id)
|
||||
# channel/stream
|
||||
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
|
||||
url('lbry://@test*1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
|
||||
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
|
||||
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False)
|
||||
url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
|
||||
# combined legacy and new
|
||||
url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
|
||||
url('@test*1/stuff#2', channel_sequence='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
|
||||
# unicode regex edges
|
||||
_url = lambda name: url(name, stream_name=name)
|
||||
_url('\uD799')
|
||||
@@ -111,10 +108,8 @@ class TestURLParsing(unittest.TestCase):
|
||||
fail("lbry://test@")
|
||||
fail("lbry://tes@t")
|
||||
fail(f"lbry://test:1#{claim_id}")
|
||||
fail("lbry://test*0")
|
||||
fail("lbry://test$0")
|
||||
fail("lbry://test/path")
|
||||
fail("lbry://@test1*1ab/fakepath")
|
||||
fail("lbry://test:1:1:1")
|
||||
fail("whatever/lbry://test")
|
||||
fail("lbry://lbry://test")
|
||||
@@ -122,5 +117,4 @@ class TestURLParsing(unittest.TestCase):
|
||||
fail("lbry://abc:0x123")
|
||||
fail("lbry://abc:0x123/page")
|
||||
fail("lbry://@test1#ABCDEF/fakepath")
|
||||
fail("test*0001")
|
||||
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
|
||||
Reference in new issue
Block a user