Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
95017b5fc8 | ||
|
|
1374d337d8 | ||
|
|
35e7cc685f | ||
|
|
2ac4883bda | ||
|
|
5a7212aebb | ||
|
|
3f39cc2955 | ||
|
|
8a5bfe884a | ||
|
|
046dd53913 | ||
|
|
d2f174635c | ||
|
|
01a760d8f0 | ||
|
|
9215d2c9f9 | ||
|
|
0bb09ae91e | ||
|
|
afeff61268 | ||
|
|
6f5a595975 | ||
|
|
8a69ec062e | ||
|
|
45fb673e49 | ||
|
|
93b5f1809e |
No files matched your search
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.38.3"
|
||||
__version__ = "0.38.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -187,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
@@ -220,7 +220,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: typing.Optional['AbstractBlob'], address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
@@ -242,7 +242,9 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', addres
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
connected_transport = protocol.transport
|
||||
if blob is None or blob.get_is_verified() or not blob.is_writeable():
|
||||
# blob is None happens when we are just opening a connection
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
|
||||
@@ -30,17 +30,19 @@ class BlobDownloader:
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
|
||||
if len(self.active_connections) >= max_probes:
|
||||
return False
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0,
|
||||
just_probe: bool = False):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
@@ -91,16 +93,18 @@ class BlobDownloader:
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
"running, %d peers, %d ignored, %d active, %s connections",
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored or peer in self.active_connections:
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
just_probe = len(self.connections) == 0
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
|
||||
self.storage = None
|
||||
|
||||
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
HEADER_SIZE = 112
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
|
||||
} if progress is not None and progress < 100 else {}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.headers.bytes_size
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if final_size_after_download % self.headers.header_size != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
buffer, header_size = b'', self.headers.header_size
|
||||
async for chunk in response.content.iter_any():
|
||||
chunk = buffer + chunk
|
||||
remaining = len(chunk) % header_size
|
||||
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
|
||||
if not chunk:
|
||||
continue
|
||||
if not await self.headers.connect(len(self.headers), chunk):
|
||||
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
|
||||
return
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
self.headers.bytes_size, final_size_after_download
|
||||
)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while not response.content.at_eof():
|
||||
local_header_size += fd.write(await response.content.readany())
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
local_header_size, final_size_after_download
|
||||
)
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
if os.path.isfile(self.headers_file):
|
||||
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
try:
|
||||
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return response.content_length // HEADER_SIZE
|
||||
except OSError:
|
||||
log.exception("Failed to download headers using https.")
|
||||
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return -1
|
||||
return response.content_length // self.headers.header_size
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.headers.height
|
||||
local_height = self.local_header_file_size() // self.headers.header_size
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
|
||||
return True
|
||||
return False
|
||||
|
||||
def verify_checkpoint(self):
|
||||
expected_hash, at_height = self.CHECKPOINT
|
||||
if self.local_header_file_size() // self.headers.header_size < at_height:
|
||||
return False
|
||||
hash = hashlib.sha256()
|
||||
chunk_size = self.headers.header_size * 1000
|
||||
with open(self.headers_file, 'rb') as header_file:
|
||||
data = header_file.read(chunk_size)
|
||||
while data and header_file.tell() <= at_height * self.headers.header_size:
|
||||
hash.update(data)
|
||||
data = header_file.read(chunk_size)
|
||||
return hash.hexdigest() == expected_hash
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
if await self.should_download_headers_from_s3():
|
||||
try:
|
||||
|
||||
try:
|
||||
if await self.should_download_headers_from_s3():
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
await self.headers.close()
|
||||
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
# fixme: workaround, this should happen before download but happens after because headers.connect fail
|
||||
if not self.verify_checkpoint():
|
||||
log.info("Checkpoint failed, verifying headers using slower method.")
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
await self.headers.close()
|
||||
else:
|
||||
log.info("Header checkpoint verified.")
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(new_txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
|
||||
@@ -214,9 +214,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim)
|
||||
height = self.ledger.headers.height
|
||||
await self.generate(1)
|
||||
await self.on_header(height + 1)
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -3,11 +3,10 @@ import logging
|
||||
from binascii import unhexlify
|
||||
from typing import Tuple, List, Dict
|
||||
|
||||
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from torba.client.baseaccount import SingleKey
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.resolve import Resolver
|
||||
from lbry.wallet.account import Account
|
||||
@@ -73,50 +72,7 @@ class MainNetLedger(BaseLedger):
|
||||
result[url] = txo
|
||||
else:
|
||||
result[url] = {'error': f'{url} did not resolve to a claim'}
|
||||
return await self.validate_resolutions(result)
|
||||
|
||||
async def validate_resolutions(self, resolutions: dict):
|
||||
provables = {}
|
||||
for url, txo in resolutions.items():
|
||||
if isinstance(txo, dict) and 'error' in txo:
|
||||
continue
|
||||
parsed_url = URL.parse(url)
|
||||
# cases we check: names without sequence, claim id or any modifier
|
||||
# except: @channel/name as the signature is enough, instead we check '@channel'
|
||||
if parsed_url.has_channel:
|
||||
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
|
||||
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
|
||||
if not provables:
|
||||
return resolutions
|
||||
root_hash = self.headers.claim_trie_root.decode()
|
||||
names = list(provables.keys())
|
||||
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
|
||||
for proof, name in zip(proofs, names):
|
||||
all_failed = True
|
||||
for url, txid in provables[name]:
|
||||
if not proof.get('txhash'):
|
||||
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
|
||||
continue
|
||||
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
|
||||
if txid != expected_txid:
|
||||
resolutions[url] = {
|
||||
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
|
||||
}
|
||||
else:
|
||||
all_failed = False
|
||||
if not all_failed:
|
||||
try:
|
||||
verify_proof(proof, root_hash, name)
|
||||
continue
|
||||
except InvalidProofError as error:
|
||||
for url, _ in provables[name]:
|
||||
resolution = resolutions[url]
|
||||
if isinstance(resolution, dict) and 'error' in resolution:
|
||||
continue
|
||||
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
|
||||
return resolutions
|
||||
return result
|
||||
|
||||
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
|
||||
return await self._inflate_outputs(self.network.claim_search(**kwargs))
|
||||
|
||||
@@ -9,8 +9,5 @@ class Network(BaseNetwork):
|
||||
def resolve(self, urls):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
||||
|
||||
def get_name_proofs(self, block_hash, *names):
|
||||
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
|
||||
|
||||
def claim_search(self, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
||||
@@ -121,6 +121,7 @@ class SQLDB:
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
@@ -134,8 +135,6 @@ class SQLDB:
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
@@ -104,7 +104,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
'blockchain.transaction.get_height': self.transaction_get_height,
|
||||
'blockchain.claimtrie.search': self.claimtrie_search,
|
||||
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
|
||||
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
|
||||
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
|
||||
'blockchain.block.get_server_height': self.get_server_height,
|
||||
}
|
||||
@@ -166,9 +165,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
|
||||
def claimtrie_getnameproofs(self, block_hash, *names):
|
||||
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
|
||||
|
||||
async def transaction_get_height(self, tx_hash):
|
||||
self.assert_tx_hash(tx_hash)
|
||||
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import os, asyncio, aiohttp, json, slack, sqlparse
|
||||
|
||||
|
||||
async def listen(slack_client, url):
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
|
||||
print(f"connecting to {url}")
|
||||
try:
|
||||
ws = await session.ws_connect(url)
|
||||
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
|
||||
print(f"failed to connect to {url}")
|
||||
return
|
||||
print(f"connected to {url}")
|
||||
|
||||
async for msg in ws:
|
||||
r = json.loads(msg.data)
|
||||
try:
|
||||
queries = r["api"]["search"]["interrupted_queries"]
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
for q in queries:
|
||||
# clean = re.sub(r"\s+", " ", q)
|
||||
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
|
||||
print(f'{url}: {clean}')
|
||||
response = await slack_client.chat_postMessage(
|
||||
username=url,
|
||||
icon_emoji=":hourglass_flowing_sand:",
|
||||
channel='#clubhouse-de-obscure',
|
||||
text="*Query timed out:* " + clean
|
||||
)
|
||||
if not response["ok"]:
|
||||
print("SLACK ERROR:\n", response)
|
||||
print()
|
||||
|
||||
|
||||
async def main():
|
||||
try:
|
||||
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
|
||||
except KeyError:
|
||||
print("Error: SLACK_TOKEN env var required")
|
||||
return
|
||||
|
||||
num_servers = 5
|
||||
tasks = []
|
||||
for i in range(1, num_servers+1):
|
||||
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,9 +1,11 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.orchstr8 import Conductor
|
||||
from torba.testcase import IntegrationTestCase
|
||||
import lbry.wallet
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.extras.daemon.Components import HeadersComponent
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
|
||||
class TestSessionBloat(IntegrationTestCase):
|
||||
@@ -29,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertTrue(session.is_closing())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
|
||||
|
||||
|
||||
class TestHeadersComponent(CommandTestCase):
|
||||
|
||||
LEDGER = lbry.wallet
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.component_manager = self.daemon.component_manager
|
||||
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
|
||||
self.headers_component = HeadersComponent(self.component_manager)
|
||||
|
||||
async def test_cant_reach_host(self):
|
||||
HeadersComponent.HEADERS_URL = 'notthere/'
|
||||
os.unlink(self.headers_component.headers.path)
|
||||
# test is that this doesnt raise
|
||||
await self.headers_component.start()
|
||||
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
|
||||
self.assertEqual(await self.headers_component.get_status(), {})
|
||||
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
# corrupt by appending
|
||||
headers.io.seek(block_bytes(len(headers)))
|
||||
headers.io.write(b"appending")
|
||||
headers._size = None
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 1499)
|
||||
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
|
||||
|
||||
@@ -115,7 +115,7 @@ class BaseHeaders:
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
self._size = self.io.tell() // self.header_size
|
||||
added += written
|
||||
if bail:
|
||||
break
|
||||
@@ -170,10 +170,12 @@ class BaseHeaders:
|
||||
|
||||
async def repair(self):
|
||||
previous_header_hash = fail = None
|
||||
self.io.seek(0)
|
||||
batch_size = 10000
|
||||
batch_size = 36
|
||||
for start_height in range(0, self.height, batch_size):
|
||||
self.io.seek(self.header_size * start_height)
|
||||
headers = self.io.read(self.header_size*batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
||||
height = header['block_height']
|
||||
if height:
|
||||
@@ -184,7 +186,7 @@ class BaseHeaders:
|
||||
fail = True
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
@@ -205,7 +207,7 @@ class BaseHeaders:
|
||||
end = min(len(headers), end + self.chunk_size * self.header_size)
|
||||
|
||||
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
|
||||
assert len(headers) % self.header_size == 0
|
||||
assert len(headers) % self.header_size == 0, len(headers)
|
||||
for idx in range(len(headers) // self.header_size):
|
||||
start, end = idx * self.header_size, (idx + 1) * self.header_size
|
||||
header = headers[start:end]
|
||||
|
||||
@@ -266,7 +266,6 @@ class BlockchainNode:
|
||||
self.data_path = None
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.blockchain_address = None
|
||||
self._block_expected = 0
|
||||
self.hostname = 'localhost'
|
||||
self.peerport = 9246 + 2 # avoid conflict with default peer port
|
||||
@@ -330,7 +329,7 @@ class BlockchainNode:
|
||||
self.daemon_bin,
|
||||
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
|
||||
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
|
||||
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
|
||||
f'-port={self.peerport}'
|
||||
)
|
||||
self.log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
@@ -365,10 +364,9 @@ class BlockchainNode:
|
||||
self.log.info(out.decode().strip())
|
||||
return out.decode().strip()
|
||||
|
||||
async def generate(self, blocks, address=None):
|
||||
def generate(self, blocks):
|
||||
self._block_expected += blocks
|
||||
address = await self.get_raw_change_address()
|
||||
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
|
||||
return self._cli_cmnd('generate', str(blocks))
|
||||
|
||||
def invalidate_block(self, blockhash):
|
||||
return self._cli_cmnd('invalidateblock', blockhash)
|
||||
@@ -377,7 +375,7 @@ class BlockchainNode:
|
||||
return self._cli_cmnd('getblockhash', str(block))
|
||||
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress', 'legacy')
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
|
||||
async def get_balance(self):
|
||||
return float(await self._cli_cmnd('getbalance'))
|
||||
|
||||
Reference in new issue
Block a user