Compare commits

..
Author SHA1 Message Date
Victor Shyba 7885f34221 upgrading latest lbrycrd wip 2019-07-23 12:42:05 -03:00
14 changed files with 115 additions and 140 deletions

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.4"
__version__ = "0.38.3"
version = tuple(__version__.split('.'))
+40 -50
View File
@@ -1,4 +1,3 @@
import hashlib
import os
import asyncio
import logging
@@ -97,10 +96,12 @@ class DatabaseComponent(Component):
self.storage = None
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
HEADER_SIZE = 112
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -134,26 +135,31 @@ class HeadersComponent(Component):
} if progress is not None and progress < 100 else {}
async def fetch_headers_from_s3(self):
local_header_size = self.local_header_file_size()
local_header_size = self.headers.bytes_size
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
log.warning("s3 is more out of date than we are")
return
final_size_after_download = response.content_length + local_header_size
if final_size_after_download % self.headers.header_size != 0:
if response.content_length % HEADER_SIZE != 0:
log.warning("s3 appears to have corrupted header")
return
write_mode = "wb"
final_size_after_download = response.content_length + local_header_size
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while not response.content.at_eof():
local_header_size += fd.write(await response.content.readany())
self._headers_progress_percent = self._round_progress(
local_header_size, final_size_after_download
)
buffer, header_size = b'', self.headers.header_size
async for chunk in response.content.iter_any():
chunk = buffer + chunk
remaining = len(chunk) % header_size
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
if not chunk:
continue
if not await self.headers.connect(len(self.headers), chunk):
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
return
self._headers_progress_percent = self._round_progress(
self.headers.bytes_size, final_size_after_download
)
def local_header_file_size(self) -> int:
if os.path.isfile(self.headers_file):
@@ -161,11 +167,13 @@ class HeadersComponent(Component):
return 0
async def get_downloadable_header_height(self) -> typing.Optional[int]:
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return -1
return response.content_length // self.headers.header_size
try:
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return response.content_length // HEADER_SIZE
except OSError:
log.exception("Failed to download headers using https.")
async def should_download_headers_from_s3(self) -> bool:
if self.conf.blockchain_name != "lbrycrd_main":
@@ -174,7 +182,7 @@ class HeadersComponent(Component):
if not s3_headers_depth:
return False
local_height = self.local_header_file_size() // self.headers.header_size
local_height = self.headers.height
remote_height = await self.get_downloadable_header_height()
if remote_height is not None:
log.info("remote height: %i, local height: %i", remote_height, local_height)
@@ -182,42 +190,24 @@ class HeadersComponent(Component):
return True
return False
def verify_checkpoint(self):
expected_hash, at_height = self.CHECKPOINT
if self.local_header_file_size() // self.headers.header_size < at_height:
return False
hash = hashlib.sha256()
chunk_size = self.headers.header_size * 1000
with open(self.headers_file, 'rb') as header_file:
data = header_file.read(chunk_size)
while data and header_file.tell() <= at_height * self.headers.header_size:
hash.update(data)
data = header_file.read(chunk_size)
return hash.hexdigest() == expected_hash
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
try:
if await self.should_download_headers_from_s3():
await self.headers.open()
await self.headers.repair()
if await self.should_download_headers_from_s3():
try:
self.is_downloading_headers = True
await self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
# fixme: workaround, this should happen before download but happens after because headers.connect fail
if not self.verify_checkpoint():
log.info("Checkpoint failed, verifying headers using slower method.")
await self.headers.open()
await self.headers.repair()
await self.headers.close()
else:
log.info("Header checkpoint verified.")
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
await self.headers.close()
async def stop(self):
pass
+2 -2
View File
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(new_txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
+2
View File
@@ -214,7 +214,9 @@ class CommandTestCase(IntegrationTestCase):
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
height = self.ledger.headers.height
await self.generate(1)
await self.on_header(height + 1)
await self.on_transaction_dict(claim)
return claim
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+46 -2
View File
@@ -3,10 +3,11 @@ import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL
from lbry.schema.url import URL, normalize_name
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
@@ -72,7 +73,50 @@ class MainNetLedger(BaseLedger):
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return result
return await self.validate_resolutions(result)
async def validate_resolutions(self, resolutions: dict):
provables = {}
for url, txo in resolutions.items():
if isinstance(txo, dict) and 'error' in txo:
continue
parsed_url = URL.parse(url)
# cases we check: names without sequence, claim id or any modifier
# except: @channel/name as the signature is enough, instead we check '@channel'
if parsed_url.has_channel:
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
if not provables:
return resolutions
root_hash = self.headers.claim_trie_root.decode()
names = list(provables.keys())
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
for proof, name in zip(proofs, names):
all_failed = True
for url, txid in provables[name]:
if not proof.get('txhash'):
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
continue
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
if txid != expected_txid:
resolutions[url] = {
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
}
else:
all_failed = False
if not all_failed:
try:
verify_proof(proof, root_hash, name)
continue
except InvalidProofError as error:
for url, _ in provables[name]:
resolution = resolutions[url]
if isinstance(resolution, dict) and 'error' in resolution:
continue
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
return resolutions
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
+3
View File
@@ -9,5 +9,8 @@ class Network(BaseNetwork):
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def get_name_proofs(self, block_hash, *names):
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
+2 -1
View File
@@ -121,7 +121,6 @@ class SQLDB:
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
@@ -135,6 +134,8 @@ class SQLDB:
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
+4
View File
@@ -104,6 +104,7 @@ class LBRYElectrumX(ElectrumX):
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': self.get_server_height,
}
@@ -165,6 +166,9 @@ class LBRYElectrumX(ElectrumX):
async def get_server_height(self):
return self.bp.height
def claimtrie_getnameproofs(self, block_hash, *names):
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
async def transaction_get_height(self, tx_hash):
self.assert_tx_hash(tx_hash)
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
-49
View File
@@ -1,49 +0,0 @@
import os, asyncio, aiohttp, json, slack, sqlparse
async def listen(slack_client, url):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
print(f"connecting to {url}")
try:
ws = await session.ws_connect(url)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
print(f"failed to connect to {url}")
return
print(f"connected to {url}")
async for msg in ws:
r = json.loads(msg.data)
try:
queries = r["api"]["search"]["interrupted_queries"]
except KeyError:
continue
for q in queries:
# clean = re.sub(r"\s+", " ", q)
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
print(f'{url}: {clean}')
response = await slack_client.chat_postMessage(
username=url,
icon_emoji=":hourglass_flowing_sand:",
channel='#clubhouse-de-obscure',
text="*Query timed out:* " + clean
)
if not response["ok"]:
print("SLACK ERROR:\n", response)
print()
async def main():
try:
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
except KeyError:
print("Error: SLACK_TOKEN env var required")
return
num_servers = 5
tasks = []
for i in range(1, num_servers+1):
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
await asyncio.gather(*tasks)
asyncio.run(main())
@@ -1,11 +1,9 @@
import asyncio
import os
import lbry.wallet
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.Components import HeadersComponent
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import lbry.wallet
class TestSessionBloat(IntegrationTestCase):
@@ -31,22 +29,3 @@ class TestSessionBloat(IntegrationTestCase):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
class TestHeadersComponent(CommandTestCase):
LEDGER = lbry.wallet
async def asyncSetUp(self):
await super().asyncSetUp()
self.component_manager = self.daemon.component_manager
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
self.headers_component = HeadersComponent(self.component_manager)
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesnt raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -109,6 +109,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
# corrupt by appending
headers.io.seek(block_bytes(len(headers)))
headers.io.write(b"appending")
headers._size = None
await headers.repair()
self.assertEqual(headers.height, 1499)
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
+5 -7
View File
@@ -115,7 +115,7 @@ class BaseHeaders:
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = self.io.tell() // self.header_size
self._size = None
added += written
if bail:
break
@@ -170,12 +170,10 @@ class BaseHeaders:
async def repair(self):
previous_header_hash = fail = None
batch_size = 36
self.io.seek(0)
batch_size = 10000
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
@@ -186,7 +184,7 @@ class BaseHeaders:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
@@ -207,7 +205,7 @@ class BaseHeaders:
end = min(len(headers), end + self.chunk_size * self.header_size)
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
assert len(headers) % self.header_size == 0, len(headers)
assert len(headers) % self.header_size == 0
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]
+6 -4
View File
@@ -266,6 +266,7 @@ class BlockchainNode:
self.data_path = None
self.protocol = None
self.transport = None
self.blockchain_address = None
self._block_expected = 0
self.hostname = 'localhost'
self.peerport = 9246 + 2 # avoid conflict with default peer port
@@ -329,7 +330,7 @@ class BlockchainNode:
self.daemon_bin,
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
f'-port={self.peerport}'
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
)
self.log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(
@@ -364,9 +365,10 @@ class BlockchainNode:
self.log.info(out.decode().strip())
return out.decode().strip()
def generate(self, blocks):
async def generate(self, blocks, address=None):
self._block_expected += blocks
return self._cli_cmnd('generate', str(blocks))
address = await self.get_raw_change_address()
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
def invalidate_block(self, blockhash):
return self._cli_cmnd('invalidateblock', blockhash)
@@ -375,7 +377,7 @@ class BlockchainNode:
return self._cli_cmnd('getblockhash', str(block))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress')
return self._cli_cmnd('getrawchangeaddress', 'legacy')
async def get_balance(self):
return float(await self._cli_cmnd('getbalance'))