Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5671224fe0 | ||
|
|
f3c1dcef81 | ||
|
|
a2c4608bc8 | ||
|
|
5562e84722 |
No files matched your search
@@ -1,11 +1,11 @@
|
||||
from typing import Dict
|
||||
|
||||
|
||||
def split_range_into_10k_batches(start, end):
|
||||
def split_range_into_batches(start, end, batch_size=100_000):
|
||||
batch = [start, end]
|
||||
batches = [batch]
|
||||
for block in range(start, end+1):
|
||||
if 0 < block != batch[0] and block % 10_000 == 0:
|
||||
if 0 < block != batch[0] and block % batch_size == 0:
|
||||
batch = [block, block]
|
||||
batches.append(batch)
|
||||
else:
|
||||
@@ -50,6 +50,7 @@ class FilterBuilder:
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.group_filters = [
|
||||
GroupFilter(start, end, 5),
|
||||
GroupFilter(start, end, 4),
|
||||
GroupFilter(start, end, 3),
|
||||
GroupFilter(start, end, 2),
|
||||
|
||||
@@ -16,7 +16,7 @@ from lbry.error import LbrycrdEventSubscriptionError
|
||||
|
||||
from . import blocks as block_phase, claims as claim_phase, supports as support_phase
|
||||
from .context import uninitialize
|
||||
from .filter_builder import split_range_into_10k_batches
|
||||
from .filter_builder import split_range_into_batches
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -187,7 +187,7 @@ class BlockchainSync(Sync):
|
||||
else:
|
||||
blocks = await self.db.run(block_phase.get_block_range_without_filters)
|
||||
if blocks != (-1, -1):
|
||||
batches = split_range_into_10k_batches(*blocks)
|
||||
batches = split_range_into_batches(*blocks)
|
||||
p.step()
|
||||
else:
|
||||
p.step()
|
||||
|
||||
+2
-2
@@ -269,8 +269,8 @@ class Database:
|
||||
async def insert_tx_filters(self, filters):
|
||||
return await self.run(q.insert_tx_filters, filters)
|
||||
|
||||
async def insert_transaction(self, block_hash, tx):
|
||||
return await self.run(q.insert_transaction, block_hash, tx)
|
||||
async def insert_transactions(self, txs):
|
||||
return await self.run(q.insert_transactions, txs)
|
||||
|
||||
async def update_address_used_times(self, addresses):
|
||||
return await self.run(q.update_address_used_times, addresses)
|
||||
|
||||
+54
-29
@@ -26,17 +26,29 @@ class DatabaseAddressIterator:
|
||||
self.chain = chain
|
||||
self.n = -1
|
||||
|
||||
@staticmethod
|
||||
def get_sql(account_id, chain):
|
||||
return (
|
||||
select(
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.n
|
||||
).where(
|
||||
(AccountAddress.c.account == account_id) &
|
||||
(AccountAddress.c.chain == chain)
|
||||
).order_by(AccountAddress.c.n)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_address_hash_bytes(account_id, chain):
|
||||
return [
|
||||
bytearray(hash160(row['pubkey'])) for row in context().fetchall(
|
||||
DatabaseAddressIterator.get_sql(account_id, chain)
|
||||
)
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
|
||||
with context().connect_streaming() as c:
|
||||
sql = (
|
||||
select(
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.n
|
||||
).where(
|
||||
(AccountAddress.c.account == self.account_id) &
|
||||
(AccountAddress.c.chain == self.chain)
|
||||
).order_by(AccountAddress.c.n)
|
||||
)
|
||||
sql = self.get_sql(self.account_id, self.chain)
|
||||
for row in c.execute(sql):
|
||||
self.n = row['n']
|
||||
yield hash160(row['pubkey']), self.n, False
|
||||
@@ -82,46 +94,59 @@ class PersistingAddressIterator(DatabaseAddressIterator):
|
||||
yield hash160(pubkey_child.pubkey_bytes), self.n, True
|
||||
|
||||
|
||||
class BatchAddressIterator:
|
||||
|
||||
def __init__(self, iterator: PersistingAddressIterator, size):
|
||||
self.iterator = iterator
|
||||
self.size = size
|
||||
|
||||
def __iter__(self) -> Iterator[bytearray]:
|
||||
i = iter(self.iterator)
|
||||
while True:
|
||||
yield [bytearray(next(i)[0]) for _ in range(self.size)]
|
||||
|
||||
|
||||
def generate_addresses_using_filters(best_height, allowed_gap, address_manager) -> Set:
|
||||
need, have = set(), set()
|
||||
matchers = get_filter_matchers(best_height)
|
||||
with PersistingAddressIterator(*address_manager) as addresses:
|
||||
gap = 0
|
||||
for address_hash, n, is_new in addresses: # pylint: disable=unused-variable
|
||||
gap += 1
|
||||
address_bytes = bytearray(address_hash)
|
||||
with PersistingAddressIterator(*address_manager) as address_iterator:
|
||||
for addresses in BatchAddressIterator(address_iterator, allowed_gap):
|
||||
has_match = False
|
||||
for matcher, filter_range in matchers:
|
||||
if matcher.Match(address_bytes):
|
||||
gap = 0
|
||||
if matcher.MatchAny(addresses):
|
||||
has_match = True
|
||||
if filter_range not in need and filter_range not in have:
|
||||
if has_filter_range(*filter_range):
|
||||
have.add(filter_range)
|
||||
else:
|
||||
need.add(filter_range)
|
||||
if gap >= allowed_gap:
|
||||
if not has_match:
|
||||
break
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_sub_filters_for_addresses(granularity, address_manager):
|
||||
need = set()
|
||||
for matcher, filter_range in get_filter_matchers_at_granularity(granularity):
|
||||
for address_hash, _, _ in DatabaseAddressIterator(*address_manager):
|
||||
address_bytes = bytearray(address_hash)
|
||||
if matcher.Match(address_bytes) and not has_filter_range(*filter_range):
|
||||
need.add(filter_range)
|
||||
break
|
||||
filters = get_filter_matchers_at_granularity(granularity)
|
||||
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
|
||||
for matcher, filter_range in filters:
|
||||
if matcher.MatchAny(addresses) and not has_filter_range(*filter_range):
|
||||
need.add(filter_range)
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_tx_for_addresses(address_manager):
|
||||
need = set()
|
||||
for tx_hash, matcher in get_tx_matchers_for_missing_txs():
|
||||
for address_hash, _, _ in DatabaseAddressIterator(*address_manager):
|
||||
address_bytes = bytearray(address_hash)
|
||||
if matcher.Match(address_bytes):
|
||||
need.add(tx_hash)
|
||||
break
|
||||
filters = get_tx_matchers_for_missing_txs()
|
||||
print(f' loaded tx filters ({len(filters)})')
|
||||
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
|
||||
print(f' loaded addresses ({len(addresses)})')
|
||||
print(' matching...')
|
||||
for i, (tx_hash, matcher) in enumerate(filters):
|
||||
if i > 0 and i % 1000 == 0:
|
||||
print(f' {i} of {len(filters)} processed')
|
||||
if matcher.MatchAny(addresses):
|
||||
need.add(tx_hash)
|
||||
return need
|
||||
|
||||
|
||||
|
||||
@@ -64,8 +64,11 @@ def get_block_headers(first, last=None):
|
||||
return rows
|
||||
|
||||
|
||||
def insert_transaction(block_hash, tx):
|
||||
context().get_bulk_loader().add_transaction(block_hash, tx).flush(TX)
|
||||
def insert_transactions(txs):
|
||||
loader = context().get_bulk_loader()
|
||||
for block_hash, tx in txs:
|
||||
loader.add_transaction(block_hash, tx)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def check_version_and_create_tables():
|
||||
|
||||
@@ -46,7 +46,7 @@ def get_filters(start_height, end_height=None, granularity=0):
|
||||
.order_by(TXFilter.c.height)
|
||||
)
|
||||
else:
|
||||
factor = granularity if granularity <= 4 else log10(granularity)
|
||||
factor = granularity if granularity < 100 else log10(granularity)
|
||||
if end_height is None:
|
||||
height_condition = (BlockFilter.c.height == start_height)
|
||||
elif end_height == -1:
|
||||
@@ -64,8 +64,11 @@ def get_filters(start_height, end_height=None, granularity=0):
|
||||
|
||||
def get_minimal_required_filter_ranges(height) -> Dict[int, Tuple[int, int]]:
|
||||
minimal = {}
|
||||
if height >= 100_000:
|
||||
minimal[5] = (0, ((height // 100_000)-1) * 100_000)
|
||||
if height >= 10_000:
|
||||
minimal[4] = (0, ((height // 10_000)-1) * 10_000)
|
||||
start = height - height % 100_000
|
||||
minimal[4] = (start, start + (((height - start) // 10_000) - 1) * 10_000)
|
||||
if height >= 1_000:
|
||||
start = height - height % 10_000
|
||||
minimal[3] = (start, start+(((height-start) // 1_000)-1) * 1_000)
|
||||
@@ -92,6 +95,9 @@ def get_maximum_known_filters() -> Dict[str, Optional[int]]:
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 4)
|
||||
.scalar_subquery().label('4'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 5)
|
||||
.scalar_subquery().label('5'),
|
||||
)
|
||||
return context().fetchone(query)
|
||||
|
||||
@@ -101,7 +107,7 @@ def get_missing_required_filters(height) -> Set[Tuple[int, int, int]]:
|
||||
missing_filters = set()
|
||||
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
|
||||
known_height = known_filters.get(str(granularity))
|
||||
if known_height is not None and known_height > start:
|
||||
if known_height is not None and known_height >= start:
|
||||
if granularity == 1:
|
||||
adjusted_height = known_height + 1
|
||||
else:
|
||||
|
||||
+1
-1
@@ -3447,7 +3447,7 @@ class Client(API):
|
||||
|
||||
async def connect(self):
|
||||
self.session = ClientSession()
|
||||
self.ws = await self.session.ws_connect(self.url, max_msg_size=20*1024*1024)
|
||||
self.ws = await self.session.ws_connect(self.url, max_msg_size=50*1024*1024)
|
||||
self.receive_messages_task = asyncio.create_task(self.receive_messages())
|
||||
|
||||
async def disconnect(self):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Dict
|
||||
@@ -135,13 +136,17 @@ class FilterManager:
|
||||
async def download_and_save_txs(self, tx_hashes):
|
||||
if not tx_hashes:
|
||||
return
|
||||
txids = [hexlify(tx_hash[::-1]).decode() for tx_hash in tx_hashes]
|
||||
print(f'=> transaction_search(len(txids): {len(txids)})')
|
||||
txs = await self.client.first.transaction_search(txids=txids, raw=True)
|
||||
print(f' @ transaction_search(len(txids): {len(txids)})')
|
||||
for raw_tx in txs.values():
|
||||
await self.db.insert_transaction(None, Transaction(unhexlify(raw_tx)))
|
||||
print(f' # transaction_search(len(txids): {len(txids)})')
|
||||
all_txids = [hexlify(tx_hash[::-1]).decode() for tx_hash in tx_hashes]
|
||||
chunk_size = 10
|
||||
for i in range(0, len(all_txids), chunk_size):
|
||||
txids = all_txids[i:i + chunk_size]
|
||||
print(f' => transaction_search(len(txids): {len(txids)})')
|
||||
txs = await self.client.first.transaction_search(txids=txids, raw=True)
|
||||
print(f' <= transaction_search(len(txids): {len(txids)})')
|
||||
await self.db.insert_transactions([
|
||||
(None, Transaction(unhexlify(raw_tx))) for raw_tx in txs.values()
|
||||
])
|
||||
print(f' saved {len(txids)}) transactions')
|
||||
|
||||
async def download_initial_filters(self, best_height):
|
||||
missing = await self.db.get_missing_required_filters(best_height)
|
||||
@@ -155,6 +160,7 @@ class FilterManager:
|
||||
f" matching addresses for account {account.id} "
|
||||
f"address group {address_manager.chain_number}..."
|
||||
)
|
||||
start = time.perf_counter()
|
||||
missing = await self.db.generate_addresses_using_filters(
|
||||
best_height, address_manager.gap, (
|
||||
account.id,
|
||||
@@ -164,6 +170,7 @@ class FilterManager:
|
||||
address_manager.public_key.depth
|
||||
)
|
||||
)
|
||||
print(f" {time.perf_counter()-start}s")
|
||||
if missing:
|
||||
print("downloading level 3 filters")
|
||||
await self.download_and_save_filters(missing)
|
||||
@@ -193,6 +200,8 @@ class FilterManager:
|
||||
await self.download_initial_filters(best_height)
|
||||
print('generating addresses...')
|
||||
await self.generate_addresses(best_height, wallets)
|
||||
print("downloading level 3 filters...")
|
||||
await self.download_sub_filters(4, wallets)
|
||||
print("downloading level 2 filters...")
|
||||
await self.download_sub_filters(3, wallets)
|
||||
print("downloading level 1 filters...")
|
||||
|
||||
+1
-1
@@ -260,7 +260,7 @@ class AsyncUnitDBTestCase(AsyncioTestCase):
|
||||
self.outputs.extend(tx.outputs)
|
||||
return block_or_tx
|
||||
elif isinstance(block_or_tx, Transaction):
|
||||
await self.db.insert_transaction(block_hash, block_or_tx)
|
||||
await self.db.insert_transactions([(block_hash, block_or_tx)])
|
||||
self.outputs.extend(block_or_tx.outputs)
|
||||
return block_or_tx.outputs[0]
|
||||
else:
|
||||
|
||||
@@ -12,7 +12,7 @@ class LightClientTests(IntegrationTestCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
await self.chain.generate(200)
|
||||
self.full_node_daemon = await self.make_full_node_daemon()
|
||||
self.full_node_daemon = await self.make_full_node_daemon(workers=2)
|
||||
self.full_node: FullNode = self.full_node_daemon.service
|
||||
self.light_client_daemon = await self.make_light_client_daemon(self.full_node_daemon, start=False)
|
||||
self.light_client: LightClient = self.light_client_daemon.service
|
||||
@@ -25,6 +25,7 @@ class LightClientTests(IntegrationTestCase):
|
||||
await self.light_client.wallets.open()
|
||||
await self.light_client.client.start_event_streams()
|
||||
self.db = self.light_client.db
|
||||
self.api = self.light_client_daemon.api
|
||||
self.sync = self.light_client.sync
|
||||
self.client = self.light_client.client
|
||||
self.account = self.light_client.wallets.default.accounts.default
|
||||
@@ -44,3 +45,9 @@ class LightClientTests(IntegrationTestCase):
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.sync.on_synced.first
|
||||
await self.assertBalance(self.account, '5.0')
|
||||
|
||||
await self.api.channel_create('@foo', '1.0')
|
||||
await self.chain.generate(1)
|
||||
await self.sync.on_synced.first
|
||||
channels = await self.api.channel_list()
|
||||
self.assertEqual(len(channels), 1)
|
||||
@@ -1,12 +1,13 @@
|
||||
from unittest import TestCase
|
||||
from lbry.blockchain.sync.filter_builder import (
|
||||
FilterBuilder as FB, GroupFilter as GF, split_range_into_10k_batches as split
|
||||
FilterBuilder as FB, GroupFilter as GF, split_range_into_batches
|
||||
)
|
||||
|
||||
|
||||
class TestFilterGenerationComponents(TestCase):
|
||||
|
||||
def test_split_range_into_10k_batches(self):
|
||||
def split(a, b): return split_range_into_batches(a, b, 10_000)
|
||||
# single block (same start-end)
|
||||
self.assertEqual(split(901_123, 901_123), [[901_123, 901_123]])
|
||||
# spans a 10k split
|
||||
@@ -70,37 +71,42 @@ class TestFilterGenerationComponents(TestCase):
|
||||
self.assertEqual(FB(819_913, 819_999).query_heights, (810_000, 819_999))
|
||||
|
||||
def test_filter_builder_add(self):
|
||||
fb = FB(818_813, 819_999)
|
||||
self.assertEqual(fb.query_heights, (810_000, 819_999))
|
||||
self.assertEqual(fb.group_filters[0].coverage, [810_000])
|
||||
self.assertEqual(fb.group_filters[1].coverage, [818_000, 819_000])
|
||||
self.assertEqual(fb.group_filters[2].coverage, [
|
||||
818_800, 818_900, 819_000, 819_100, 819_200, 819_300,
|
||||
819_400, 819_500, 819_600, 819_700, 819_800, 819_900
|
||||
])
|
||||
fb.add(b'beef0', 810_000, ['a'])
|
||||
fb.add(b'beef1', 815_001, ['b'])
|
||||
fb.add(b'beef2', 818_412, ['c'])
|
||||
fb.add(b'beef3', 818_812, ['d'])
|
||||
fb.add(b'beef4', 818_813, ['e'])
|
||||
fb.add(b'beef5', 819_000, ['f'])
|
||||
fb.add(b'beef6', 819_999, ['g'])
|
||||
fb.add(b'beef7', 819_999, ['h'])
|
||||
fb = FB(798_813, 809_999)
|
||||
self.assertEqual(fb.query_heights, (700_000, 809_999))
|
||||
self.assertEqual(fb.group_filters[0].coverage, [700_000])
|
||||
self.assertEqual(fb.group_filters[1].coverage, [790_000, 800_000])
|
||||
self.assertEqual(fb.group_filters[2].coverage, list(range(798_000, 809_000+1, 1_000)))
|
||||
self.assertEqual(fb.group_filters[3].coverage, list(range(798_800, 809_900+1, 100)))
|
||||
fb.add(b'beef0', 787_111, ['a'])
|
||||
fb.add(b'beef1', 798_222, ['b'])
|
||||
fb.add(b'beef2', 798_812, ['c'])
|
||||
fb.add(b'beef3', 798_813, ['d'])
|
||||
fb.add(b'beef4', 798_814, ['e'])
|
||||
fb.add(b'beef5', 809_000, ['f'])
|
||||
fb.add(b'beef6', 809_999, ['g'])
|
||||
fb.add(b'beef7', 809_999, ['h'])
|
||||
fb.add(b'beef8', 820_000, ['i'])
|
||||
self.assertEqual(fb.group_filters[0].groups, {
|
||||
810_000: {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}
|
||||
700_000: {'a', 'b', 'c', 'd', 'e'}
|
||||
})
|
||||
self.assertEqual(fb.group_filters[1].groups, {
|
||||
818_000: {'c', 'd', 'e'},
|
||||
819_000: {'f', 'g', 'h'}
|
||||
790_000: {'b', 'c', 'd', 'e'},
|
||||
800_000: {'f', 'g', 'h'}
|
||||
})
|
||||
self.assertEqual(fb.group_filters[2].groups[818_800], {'d', 'e'})
|
||||
self.assertEqual(fb.group_filters[2].groups[819_000], {'f'})
|
||||
self.assertEqual(fb.group_filters[2].groups[819_900], {'g', 'h'})
|
||||
self.assertEqual(fb.block_filters, {818813: {'e'}, 819000: {'f'}, 819999: {'g', 'h'}})
|
||||
self.assertEqual(fb.group_filters[2].groups, {
|
||||
798_000: {'b', 'c', 'd', 'e'}, 799_000: set(),
|
||||
800_000: set(), 801_000: set(), 802_000: set(), 803_000: set(), 804_000: set(),
|
||||
805_000: set(), 806_000: set(), 807_000: set(), 808_000: set(),
|
||||
809_000: {'f', 'g', 'h'}
|
||||
})
|
||||
self.assertEqual(fb.group_filters[3].groups[798_800], {'c', 'd', 'e'})
|
||||
self.assertEqual(fb.group_filters[3].groups[809_000], {'f'})
|
||||
self.assertEqual(fb.group_filters[3].groups[809_900], {'g', 'h'})
|
||||
self.assertEqual(fb.block_filters, {798813: {'d'}, 798814: {'e'}, 809000: {'f'}, 809999: {'h', 'g'}})
|
||||
self.assertEqual(fb.tx_filters, [
|
||||
(b'beef4', 818813, ['e']),
|
||||
(b'beef5', 819000, ['f']),
|
||||
(b'beef6', 819999, ['g']),
|
||||
(b'beef7', 819999, ['h'])
|
||||
(b'beef3', 798813, ['d']),
|
||||
(b'beef4', 798814, ['e']),
|
||||
(b'beef5', 809000, ['f']),
|
||||
(b'beef6', 809999, ['g']),
|
||||
(b'beef7', 809999, ['h'])
|
||||
])
|
||||
+108
-103
@@ -17,49 +17,55 @@ class TestMissingRequiredFiltersCalculation(UnitDBTestCase):
|
||||
self.assertEqual(q.get_missing_required_filters(100), {(2, 0, 0)})
|
||||
self.assertEqual(q.get_missing_required_filters(199), {(2, 0, 0), (1, 100, 199)})
|
||||
self.assertEqual(q.get_missing_required_filters(201), {(2, 0, 100), (1, 200, 201)})
|
||||
|
||||
# all filters missing
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), {
|
||||
(4, 0, 120_000),
|
||||
(3, 130_000, 133_000),
|
||||
(2, 134_000, 134_400),
|
||||
(1, 134_500, 134_567)
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), {
|
||||
(5, 0, 100_000),
|
||||
(4, 200_000, 220_000),
|
||||
(3, 230_000, 233_000),
|
||||
(2, 234_000, 234_400),
|
||||
(1, 234_500, 234_567)
|
||||
})
|
||||
|
||||
q.insert_block_filters([(110_000, 4, b'beef')])
|
||||
q.insert_block_filters([(129_000, 3, b'beef')])
|
||||
q.insert_block_filters([(133_900, 2, b'beef')])
|
||||
q.insert_block_filters([(134_499, 1, b'beef')])
|
||||
# we we have some filters, but not recent enough (all except 10k are adjusted)
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), {
|
||||
(4, 120_000, 120_000), # 0 -> 120_000
|
||||
(3, 130_000, 133_000),
|
||||
(2, 134_000, 134_400),
|
||||
(1, 134_500, 134_567)
|
||||
q.insert_block_filters([(0, 5, b'beef')])
|
||||
q.insert_block_filters([(190_000, 4, b'beef')])
|
||||
q.insert_block_filters([(229_000, 3, b'beef')])
|
||||
q.insert_block_filters([(233_900, 2, b'beef')])
|
||||
q.insert_block_filters([(234_499, 1, b'beef')])
|
||||
# we have some old filters but none useable as initial required (except one 100k filter)
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), {
|
||||
(5, 100_000, 100_000),
|
||||
(4, 200_000, 220_000),
|
||||
(3, 230_000, 233_000),
|
||||
(2, 234_000, 234_400),
|
||||
(1, 234_500, 234_567)
|
||||
})
|
||||
|
||||
q.insert_block_filters([(132_000, 3, b'beef')])
|
||||
q.insert_block_filters([(134_300, 2, b'beef')])
|
||||
q.insert_block_filters([(134_550, 1, b'beef')])
|
||||
# all filters get adjusted because we have recent of each
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), {
|
||||
(4, 120_000, 120_000), # 0 -> 120_000
|
||||
(3, 133_000, 133_000), # 130_000 -> 133_000
|
||||
(2, 134_400, 134_400), # 134_000 -> 134_400
|
||||
(1, 134_551, 134_567) # 134_500 -> 134_551
|
||||
q.insert_block_filters([(100_000, 5, b'beef')])
|
||||
q.insert_block_filters([(210_000, 4, b'beef')])
|
||||
q.insert_block_filters([(232_000, 3, b'beef')])
|
||||
q.insert_block_filters([(234_300, 2, b'beef')])
|
||||
q.insert_block_filters([(234_550, 1, b'beef')])
|
||||
# we have some useable initial filters, but not all
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), {
|
||||
(4, 220_000, 220_000),
|
||||
(3, 233_000, 233_000),
|
||||
(2, 234_400, 234_400),
|
||||
(1, 234_551, 234_567)
|
||||
})
|
||||
|
||||
q.insert_block_filters([(120_000, 4, b'beef')])
|
||||
q.insert_block_filters([(133_000, 3, b'beef')])
|
||||
q.insert_block_filters([(134_400, 2, b'beef')])
|
||||
q.insert_block_filters([(134_566, 1, b'beef')])
|
||||
q.insert_block_filters([(220_000, 4, b'beef')])
|
||||
q.insert_block_filters([(233_000, 3, b'beef')])
|
||||
q.insert_block_filters([(234_400, 2, b'beef')])
|
||||
q.insert_block_filters([(234_566, 1, b'beef')])
|
||||
# we have latest filters for all except latest single block
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), {
|
||||
(1, 134_567, 134_567) # 134_551 -> 134_567
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), {
|
||||
(1, 234_567, 234_567)
|
||||
})
|
||||
|
||||
q.insert_block_filters([(134_567, 1, b'beef')])
|
||||
q.insert_block_filters([(234_567, 1, b'beef')])
|
||||
# we have all latest filters
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), set())
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), set())
|
||||
|
||||
|
||||
class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
@@ -120,52 +126,54 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
q.insert_tx_filters([(hexlify(f'tx{height}'.encode()), height, create_address_filter(addresses))])
|
||||
|
||||
def test_generate_from_filters_and_download_txs(self):
|
||||
# 15 addresses will get generated, 9 due to filters and 6 due to gap
|
||||
pubkeys = [self.receiving_pubkey.child(n) for n in range(15)]
|
||||
# 18 addresses will get generated
|
||||
pubkeys = [self.receiving_pubkey.child(n) for n in range(18)]
|
||||
hashes = [hash160(key.pubkey_bytes) for key in pubkeys]
|
||||
|
||||
# create all required filters (include 9 of the addresses in the filters)
|
||||
|
||||
q.insert_block_filters([(0, 4, create_address_filter(hashes[0:1]))])
|
||||
q.insert_block_filters([(100_000, 4, create_address_filter(hashes[1:2]))])
|
||||
q.insert_block_filters([(110_000, 4, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(120_000, 4, create_address_filter(hashes[2:3]))])
|
||||
q.insert_block_filters([(0, 5, create_address_filter(hashes[0:1]))])
|
||||
q.insert_block_filters([(100_000, 5, create_address_filter([b'beef']))])
|
||||
|
||||
q.insert_block_filters([(130_000, 3, create_address_filter(hashes[3:4]))])
|
||||
q.insert_block_filters([(131_000, 3, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(133_000, 3, create_address_filter(hashes[4:5]))])
|
||||
q.insert_block_filters([(200_000, 4, create_address_filter(hashes[1:2]))])
|
||||
q.insert_block_filters([(210_000, 4, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(220_000, 4, create_address_filter(hashes[2:3]))])
|
||||
|
||||
q.insert_block_filters([(134_000, 2, create_address_filter(hashes[5:6]))])
|
||||
q.insert_block_filters([(134_200, 2, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(134_400, 2, create_address_filter(hashes[6:7]))])
|
||||
q.insert_block_filters([(230_000, 3, create_address_filter(hashes[3:4]))])
|
||||
q.insert_block_filters([(231_000, 3, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(233_000, 3, create_address_filter(hashes[4:5]))])
|
||||
|
||||
q.insert_block_filters([(134_500, 1, create_address_filter(hashes[7:8]))])
|
||||
q.insert_block_filters([(134_566, 1, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(134_567, 1, create_address_filter(hashes[8:9]))])
|
||||
q.insert_block_filters([(234_000, 2, create_address_filter(hashes[5:6]))])
|
||||
q.insert_block_filters([(234_200, 2, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(234_400, 2, create_address_filter(hashes[6:7]))])
|
||||
|
||||
q.insert_block_filters([(234_500, 1, create_address_filter(hashes[7:8]))])
|
||||
q.insert_block_filters([(234_566, 1, create_address_filter([b'beef']))])
|
||||
q.insert_block_filters([(234_567, 1, create_address_filter(hashes[8:9]))])
|
||||
|
||||
# check that all required filters did get created
|
||||
self.assertEqual(q.get_missing_required_filters(134_567), set())
|
||||
self.assertEqual(q.get_missing_required_filters(234_567), set())
|
||||
|
||||
# no addresses
|
||||
self.assertEqual([], self.get_ordered_addresses())
|
||||
|
||||
# generate addresses with 6 address gap, returns new sub filters needed
|
||||
self.assertEqual(
|
||||
q.generate_addresses_using_filters(134_567, 6, (
|
||||
q.generate_addresses_using_filters(234_567, 6, (
|
||||
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
||||
self.receiving_pubkey.pubkey_bytes,
|
||||
self.receiving_pubkey.chain_code,
|
||||
self.receiving_pubkey.depth
|
||||
)), {
|
||||
(0, 134500, 134500),
|
||||
(0, 134567, 134567),
|
||||
(1, 134000, 134099),
|
||||
(1, 134400, 134499),
|
||||
(2, 130000, 130900),
|
||||
(2, 133000, 133900),
|
||||
(3, 0, 9000),
|
||||
(3, 100000, 109000),
|
||||
(3, 120000, 129000)
|
||||
(0, 234500, 234500),
|
||||
(0, 234567, 234567),
|
||||
(1, 234000, 234099),
|
||||
(1, 234400, 234499),
|
||||
(2, 230000, 230900),
|
||||
(2, 233000, 233900),
|
||||
(3, 200000, 209000),
|
||||
(3, 220000, 229000),
|
||||
(4, 0, 90000),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -173,19 +181,19 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
self.assertEqual([key.address for key in pubkeys], self.get_ordered_addresses())
|
||||
|
||||
# "download" missing sub filters
|
||||
self.insert_sub_filters(4, hashes[0:1], 0)
|
||||
self.insert_sub_filters(4, hashes[1:2], 100_000)
|
||||
self.insert_sub_filters(4, hashes[2:3], 120_000)
|
||||
self.insert_sub_filters(3, hashes[3:4], 130_000)
|
||||
self.insert_sub_filters(3, hashes[4:5], 133_000)
|
||||
self.insert_sub_filters(2, hashes[5:6], 134_000)
|
||||
self.insert_sub_filters(2, hashes[6:7], 134_400)
|
||||
self.insert_sub_filters(1, hashes[7:8], 134_500)
|
||||
self.insert_sub_filters(1, hashes[8:9], 134_567)
|
||||
self.insert_sub_filters(5, hashes[0:1], 0)
|
||||
self.insert_sub_filters(4, hashes[1:2], 200_000)
|
||||
self.insert_sub_filters(4, hashes[2:3], 220_000)
|
||||
self.insert_sub_filters(3, hashes[3:4], 230_000)
|
||||
self.insert_sub_filters(3, hashes[4:5], 233_000)
|
||||
self.insert_sub_filters(2, hashes[5:6], 234_000)
|
||||
self.insert_sub_filters(2, hashes[6:7], 234_400)
|
||||
self.insert_sub_filters(1, hashes[7:8], 234_500)
|
||||
self.insert_sub_filters(1, hashes[8:9], 234_567)
|
||||
|
||||
# no sub filters needed to be downloaded now when re-checking all addresses
|
||||
self.assertEqual(
|
||||
q.generate_addresses_using_filters(134_567, 6, (
|
||||
q.generate_addresses_using_filters(234_567, 6, (
|
||||
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
||||
self.receiving_pubkey.pubkey_bytes,
|
||||
self.receiving_pubkey.chain_code,
|
||||
@@ -200,15 +208,13 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
q.get_missing_sub_filters_for_addresses(3, (
|
||||
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
||||
)), {
|
||||
(2, 3000, 3900),
|
||||
(2, 103000, 103900),
|
||||
(2, 123000, 123900),
|
||||
(2, 203000, 203900),
|
||||
(2, 223000, 223900),
|
||||
}
|
||||
)
|
||||
# "download" missing 1,000 sub filters
|
||||
self.insert_sub_filters(3, hashes[0:1], 3000)
|
||||
self.insert_sub_filters(3, hashes[1:2], 103_000)
|
||||
self.insert_sub_filters(3, hashes[2:3], 123_000)
|
||||
self.insert_sub_filters(3, hashes[1:2], 203_000)
|
||||
self.insert_sub_filters(3, hashes[2:3], 223_000)
|
||||
# no more missing sub filters at 1,000
|
||||
self.assertEqual(
|
||||
q.get_missing_sub_filters_for_addresses(3, (
|
||||
@@ -221,19 +227,18 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
q.get_missing_sub_filters_for_addresses(2, (
|
||||
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
||||
)), {
|
||||
(1, 3300, 3399),
|
||||
(1, 103300, 103399),
|
||||
(1, 123300, 123399),
|
||||
(1, 130300, 130399),
|
||||
(1, 133300, 133399),
|
||||
(1, 203300, 203399),
|
||||
(1, 223300, 223399),
|
||||
(1, 230300, 230399),
|
||||
(1, 233300, 233399),
|
||||
}
|
||||
)
|
||||
# "download" missing 100 sub filters
|
||||
self.insert_sub_filters(2, hashes[0:1], 3300)
|
||||
self.insert_sub_filters(2, hashes[1:2], 103_300)
|
||||
self.insert_sub_filters(2, hashes[2:3], 123_300)
|
||||
self.insert_sub_filters(2, hashes[3:4], 130_300)
|
||||
self.insert_sub_filters(2, hashes[4:5], 133_300)
|
||||
self.insert_sub_filters(2, hashes[1:2], 203_300)
|
||||
self.insert_sub_filters(2, hashes[2:3], 223_300)
|
||||
self.insert_sub_filters(2, hashes[3:4], 230_300)
|
||||
self.insert_sub_filters(2, hashes[4:5], 233_300)
|
||||
# no more missing sub filters at 100
|
||||
self.assertEqual(
|
||||
q.get_missing_sub_filters_for_addresses(2, (
|
||||
@@ -247,22 +252,22 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
self.root_pubkey.address, self.RECEIVING_KEY_N,
|
||||
)), {
|
||||
(0, 3303, 3303),
|
||||
(0, 103303, 103303),
|
||||
(0, 123303, 123303),
|
||||
(0, 130303, 130303),
|
||||
(0, 133303, 133303),
|
||||
(0, 134003, 134003),
|
||||
(0, 134403, 134403),
|
||||
(0, 203303, 203303),
|
||||
(0, 223303, 223303),
|
||||
(0, 230303, 230303),
|
||||
(0, 233303, 233303),
|
||||
(0, 234003, 234003),
|
||||
(0, 234403, 234403),
|
||||
}
|
||||
)
|
||||
# "download" missing tx filters
|
||||
self.insert_sub_filters(1, hashes[0:1], 3303)
|
||||
self.insert_sub_filters(1, hashes[1:2], 103_303)
|
||||
self.insert_sub_filters(1, hashes[2:3], 123_303)
|
||||
self.insert_sub_filters(1, hashes[3:4], 130_303)
|
||||
self.insert_sub_filters(1, hashes[4:5], 133_303)
|
||||
self.insert_sub_filters(1, hashes[5:6], 134_003)
|
||||
self.insert_sub_filters(1, hashes[6:7], 134_403)
|
||||
self.insert_sub_filters(1, hashes[1:2], 203_303)
|
||||
self.insert_sub_filters(1, hashes[2:3], 223_303)
|
||||
self.insert_sub_filters(1, hashes[3:4], 230_303)
|
||||
self.insert_sub_filters(1, hashes[4:5], 233_303)
|
||||
self.insert_sub_filters(1, hashes[5:6], 234_003)
|
||||
self.insert_sub_filters(1, hashes[6:7], 234_403)
|
||||
# no more missing tx filters
|
||||
self.assertEqual(
|
||||
q.get_missing_sub_filters_for_addresses(1, (
|
||||
@@ -272,14 +277,14 @@ class TestAddressGenerationAndTXSync(UnitDBTestCase):
|
||||
|
||||
# find TXs we need to download
|
||||
missing_txs = {
|
||||
b'7478313033333033',
|
||||
b'7478313233333033',
|
||||
b'7478313330333033',
|
||||
b'7478313333333033',
|
||||
b'7478313334303033',
|
||||
b'7478313334343033',
|
||||
b'7478313334353030',
|
||||
b'7478313334353637',
|
||||
b'7478323033333033',
|
||||
b'7478323233333033',
|
||||
b'7478323330333033',
|
||||
b'7478323333333033',
|
||||
b'7478323334303033',
|
||||
b'7478323334343033',
|
||||
b'7478323334353030',
|
||||
b'7478323334353637',
|
||||
b'747833333033'
|
||||
}
|
||||
self.assertEqual(
|
||||
|
||||
@@ -255,7 +255,7 @@ class TransactionIOBalancing(WalletTestCase):
|
||||
.add_inputs([self.txi(self.txo(sum(amounts)+0.1))]) \
|
||||
.add_outputs(utxos)
|
||||
|
||||
await self.db.insert_transaction(b'beef', self.funding_tx)
|
||||
await self.db.insert_transactions([(b'beef', self.funding_tx)])
|
||||
|
||||
return utxos
|
||||
|
||||
|
||||
Reference in new issue
Block a user