Compare commits

...
12 Commits
8 changed files with 134 additions and 54 deletions

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.3"
__version__ = "0.38.4"
version = tuple(__version__.split('.'))
+50 -40
View File
@@ -1,3 +1,4 @@
import hashlib
import os
import asyncio
import logging
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
self.storage = None
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
HEADER_SIZE = 112
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
} if progress is not None and progress < 100 else {}
async def fetch_headers_from_s3(self):
local_header_size = self.headers.bytes_size
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
log.warning("s3 is more out of date than we are")
return
if response.content_length % HEADER_SIZE != 0:
final_size_after_download = response.content_length + local_header_size
if final_size_after_download % self.headers.header_size != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
buffer, header_size = b'', self.headers.header_size
async for chunk in response.content.iter_any():
chunk = buffer + chunk
remaining = len(chunk) % header_size
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
if not chunk:
continue
if not await self.headers.connect(len(self.headers), chunk):
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
return
self._headers_progress_percent = self._round_progress(
self.headers.bytes_size, final_size_after_download
)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while not response.content.at_eof():
local_header_size += fd.write(await response.content.readany())
self._headers_progress_percent = self._round_progress(
local_header_size, final_size_after_download
)
def local_header_file_size(self) -> int:
if os.path.isfile(self.headers_file):
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
return 0
async def get_downloadable_header_height(self) -> typing.Optional[int]:
try:
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return response.content_length // HEADER_SIZE
except OSError:
log.exception("Failed to download headers using https.")
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return -1
return response.content_length // self.headers.header_size
async def should_download_headers_from_s3(self) -> bool:
if self.conf.blockchain_name != "lbrycrd_main":
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
if not s3_headers_depth:
return False
local_height = self.headers.height
local_height = self.local_header_file_size() // self.headers.header_size
remote_height = await self.get_downloadable_header_height()
if remote_height is not None:
log.info("remote height: %i, local height: %i", remote_height, local_height)
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
return True
return False
def verify_checkpoint(self):
expected_hash, at_height = self.CHECKPOINT
if self.local_header_file_size() // self.headers.header_size < at_height:
return False
hash = hashlib.sha256()
chunk_size = self.headers.header_size * 1000
with open(self.headers_file, 'rb') as header_file:
data = header_file.read(chunk_size)
while data and header_file.tell() <= at_height * self.headers.header_size:
hash.update(data)
data = header_file.read(chunk_size)
return hash.hexdigest() == expected_hash
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
await self.headers.open()
await self.headers.repair()
if await self.should_download_headers_from_s3():
try:
try:
if await self.should_download_headers_from_s3():
self.is_downloading_headers = True
await self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
await self.headers.close()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
# fixme: workaround, this should happen before download but happens after because headers.connect fail
if not self.verify_checkpoint():
log.info("Checkpoint failed, verifying headers using slower method.")
await self.headers.open()
await self.headers.repair()
await self.headers.close()
else:
log.info("Header checkpoint verified.")
async def stop(self):
pass
+2 -2
View File
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(new_txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
+1 -2
View File
@@ -121,6 +121,7 @@ class SQLDB:
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
@@ -134,8 +135,6 @@ class SQLDB:
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
+49
View File
@@ -0,0 +1,49 @@
import os, asyncio, aiohttp, json, slack, sqlparse
async def listen(slack_client, url):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
print(f"connecting to {url}")
try:
ws = await session.ws_connect(url)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
print(f"failed to connect to {url}")
return
print(f"connected to {url}")
async for msg in ws:
r = json.loads(msg.data)
try:
queries = r["api"]["search"]["interrupted_queries"]
except KeyError:
continue
for q in queries:
# clean = re.sub(r"\s+", " ", q)
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
print(f'{url}: {clean}')
response = await slack_client.chat_postMessage(
username=url,
icon_emoji=":hourglass_flowing_sand:",
channel='#clubhouse-de-obscure',
text="*Query timed out:* " + clean
)
if not response["ok"]:
print("SLACK ERROR:\n", response)
print()
async def main():
try:
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
except KeyError:
print("Error: SLACK_TOKEN env var required")
return
num_servers = 5
tasks = []
for i in range(1, num_servers+1):
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
await asyncio.gather(*tasks)
asyncio.run(main())
@@ -1,9 +1,11 @@
import asyncio
import os
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import lbry.wallet
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.Components import HeadersComponent
from torba.client.basenetwork import ClientSession
from torba.testcase import IntegrationTestCase
class TestSessionBloat(IntegrationTestCase):
@@ -29,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
class TestHeadersComponent(CommandTestCase):
LEDGER = lbry.wallet
async def asyncSetUp(self):
await super().asyncSetUp()
self.component_manager = self.daemon.component_manager
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
self.headers_component = HeadersComponent(self.component_manager)
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesnt raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
# corrupt by appending
headers.io.seek(block_bytes(len(headers)))
headers.io.write(b"appending")
headers._size = None
await headers.repair()
self.assertEqual(headers.height, 1499)
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
+7 -5
View File
@@ -115,7 +115,7 @@ class BaseHeaders:
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = None
self._size = self.io.tell() // self.header_size
added += written
if bail:
break
@@ -170,10 +170,12 @@ class BaseHeaders:
async def repair(self):
previous_header_hash = fail = None
self.io.seek(0)
batch_size = 10000
batch_size = 36
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
@@ -184,7 +186,7 @@ class BaseHeaders:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
@@ -205,7 +207,7 @@ class BaseHeaders:
end = min(len(headers), end + self.chunk_size * self.header_size)
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
assert len(headers) % self.header_size == 0
assert len(headers) % self.header_size == 0, len(headers)
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]