Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eb2a4aebba |
+24
-22
@@ -155,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._on_ready_controller = StreamController()
|
||||
self.on_ready = self._on_ready_controller.stream
|
||||
|
||||
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 10_000))
|
||||
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 1024))
|
||||
self._update_tasks = TaskGroup()
|
||||
self._other_tasks = TaskGroup() # that we dont need to start
|
||||
self._utxo_reservation_lock = asyncio.Lock()
|
||||
@@ -513,6 +513,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
)
|
||||
return True
|
||||
|
||||
synced_txs = []
|
||||
to_request = {}
|
||||
pending_synced_history = {}
|
||||
already_synced = set()
|
||||
@@ -524,6 +525,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
already_synced.add((txid, remote_height))
|
||||
already_synced_offset += 1
|
||||
continue
|
||||
|
||||
tx_indexes = {}
|
||||
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
@@ -533,14 +535,21 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
to_request[i] = (txid, remote_height)
|
||||
|
||||
log.debug(
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
|
||||
len(remote_history), address
|
||||
)
|
||||
remote_history_txids = set(txid for txid, _ in remote_history)
|
||||
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
synced_txs.append(tx)
|
||||
if len(synced_txs) >= 100:
|
||||
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), ""
|
||||
)
|
||||
while synced_txs:
|
||||
tx = synced_txs.pop()
|
||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
|
||||
assert len(pending_synced_history) == len(remote_history), \
|
||||
@@ -552,7 +561,11 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if f"{txid}:{height}:" != pending_synced_history[i]:
|
||||
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
|
||||
synced_history += pending_synced_history[i]
|
||||
await self.db.set_address_history(address, synced_history)
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), synced_history
|
||||
)
|
||||
while synced_txs:
|
||||
self._on_transaction_controller.add(TransactionEvent(address, synced_txs.pop()))
|
||||
|
||||
if address_manager is None:
|
||||
address_manager = await self.get_address_manager_for_address(address)
|
||||
@@ -616,9 +629,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
|
||||
for txid, height in sorted(to_request, key=lambda x: x[1]):
|
||||
if txid in self._tx_cache:
|
||||
tx = self._tx_cache[txid].tx
|
||||
if tx is not None and tx.is_verified:
|
||||
yield tx
|
||||
if self._tx_cache[txid].tx is not None and self._tx_cache[txid].tx.is_verified:
|
||||
yield self._tx_cache[txid].tx
|
||||
continue
|
||||
else:
|
||||
self._tx_cache[txid] = TransactionCacheItem()
|
||||
@@ -629,36 +641,26 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if not batches[-1]:
|
||||
batches.pop()
|
||||
|
||||
for routine in asyncio.as_completed([self._single_batch(batch, remote_heights) for batch in batches]):
|
||||
for tx in await routine:
|
||||
for batch in batches:
|
||||
async for tx in self._single_batch(batch, remote_heights):
|
||||
yield tx
|
||||
|
||||
async def request_synced_transactions(self, to_request, remote_history, address):
|
||||
pending_sync = []
|
||||
async for tx in self.request_transactions(((txid, height) for txid, height in to_request.values())):
|
||||
pending_sync.append(asyncio.ensure_future(self._sync(tx, remote_history)))
|
||||
log.info("%d/%d", len(pending_sync), len(to_request))
|
||||
yield tx
|
||||
pending_sync = await asyncio.gather(*pending_sync)
|
||||
await self.db.save_transaction_io_batch(
|
||||
pending_sync, address, self.address_to_hash160(address), ""
|
||||
)
|
||||
while pending_sync:
|
||||
self._on_transaction_controller.add(TransactionEvent(address, pending_sync.pop()))
|
||||
await asyncio.gather(*pending_sync)
|
||||
|
||||
async def _single_batch(self, batch, remote_heights):
|
||||
heights = {remote_heights[txid] for txid in batch}
|
||||
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
|
||||
txs = []
|
||||
pending_verification = []
|
||||
for txid, (raw, merkle) in batch_result.items():
|
||||
remote_height = remote_heights[txid]
|
||||
tx = Transaction(unhexlify(raw), height=remote_height)
|
||||
pending_verification.append(self.maybe_verify_transaction(tx, remote_height, merkle))
|
||||
txs.append(tx)
|
||||
await asyncio.gather(*pending_verification)
|
||||
return txs
|
||||
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||
yield tx
|
||||
|
||||
async def _sync(self, tx, remote_history):
|
||||
check_db_for_txos = {}
|
||||
|
||||
Reference in new issue
Block a user