Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f39cc2955 | ||
|
|
8a5bfe884a | ||
|
|
046dd53913 | ||
|
|
d2f174635c | ||
|
|
01a760d8f0 | ||
|
|
9215d2c9f9 | ||
|
|
0bb09ae91e | ||
|
|
afeff61268 | ||
|
|
6f5a595975 | ||
|
|
8a69ec062e | ||
|
|
45fb673e49 | ||
|
|
93b5f1809e |
No files matched your search
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.38.3"
|
||||
__version__ = "0.38.4"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
|
||||
self.storage = None
|
||||
|
||||
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
HEADER_SIZE = 112
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
|
||||
} if progress is not None and progress < 100 else {}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.headers.bytes_size
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if final_size_after_download % self.headers.header_size != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
buffer, header_size = b'', self.headers.header_size
|
||||
async for chunk in response.content.iter_any():
|
||||
chunk = buffer + chunk
|
||||
remaining = len(chunk) % header_size
|
||||
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
|
||||
if not chunk:
|
||||
continue
|
||||
if not await self.headers.connect(len(self.headers), chunk):
|
||||
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
|
||||
return
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
self.headers.bytes_size, final_size_after_download
|
||||
)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while not response.content.at_eof():
|
||||
local_header_size += fd.write(await response.content.readany())
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
local_header_size, final_size_after_download
|
||||
)
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
if os.path.isfile(self.headers_file):
|
||||
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
try:
|
||||
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return response.content_length // HEADER_SIZE
|
||||
except OSError:
|
||||
log.exception("Failed to download headers using https.")
|
||||
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return -1
|
||||
return response.content_length // self.headers.header_size
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.headers.height
|
||||
local_height = self.local_header_file_size() // self.headers.header_size
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
|
||||
return True
|
||||
return False
|
||||
|
||||
def verify_checkpoint(self):
|
||||
expected_hash, at_height = self.CHECKPOINT
|
||||
if self.local_header_file_size() // self.headers.header_size < at_height:
|
||||
return False
|
||||
hash = hashlib.sha256()
|
||||
chunk_size = self.headers.header_size * 1000
|
||||
with open(self.headers_file, 'rb') as header_file:
|
||||
data = header_file.read(chunk_size)
|
||||
while data and header_file.tell() <= at_height * self.headers.header_size:
|
||||
hash.update(data)
|
||||
data = header_file.read(chunk_size)
|
||||
return hash.hexdigest() == expected_hash
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
if await self.should_download_headers_from_s3():
|
||||
try:
|
||||
|
||||
try:
|
||||
if await self.should_download_headers_from_s3():
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
await self.headers.close()
|
||||
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
# fixme: workaround, this should happen before download but happens after because headers.connect fail
|
||||
if not self.verify_checkpoint():
|
||||
log.info("Checkpoint failed, verifying headers using slower method.")
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
await self.headers.close()
|
||||
else:
|
||||
log.info("Header checkpoint verified.")
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(new_txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
|
||||
@@ -121,6 +121,7 @@ class SQLDB:
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
@@ -134,8 +135,6 @@ class SQLDB:
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import os, asyncio, aiohttp, json, slack, sqlparse
|
||||
|
||||
|
||||
async def listen(slack_client, url):
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
|
||||
print(f"connecting to {url}")
|
||||
try:
|
||||
ws = await session.ws_connect(url)
|
||||
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
|
||||
print(f"failed to connect to {url}")
|
||||
return
|
||||
print(f"connected to {url}")
|
||||
|
||||
async for msg in ws:
|
||||
r = json.loads(msg.data)
|
||||
try:
|
||||
queries = r["api"]["search"]["interrupted_queries"]
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
for q in queries:
|
||||
# clean = re.sub(r"\s+", " ", q)
|
||||
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
|
||||
print(f'{url}: {clean}')
|
||||
response = await slack_client.chat_postMessage(
|
||||
username=url,
|
||||
icon_emoji=":hourglass_flowing_sand:",
|
||||
channel='#clubhouse-de-obscure',
|
||||
text="*Query timed out:* " + clean
|
||||
)
|
||||
if not response["ok"]:
|
||||
print("SLACK ERROR:\n", response)
|
||||
print()
|
||||
|
||||
|
||||
async def main():
|
||||
try:
|
||||
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
|
||||
except KeyError:
|
||||
print("Error: SLACK_TOKEN env var required")
|
||||
return
|
||||
|
||||
num_servers = 5
|
||||
tasks = []
|
||||
for i in range(1, num_servers+1):
|
||||
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,9 +1,11 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.orchstr8 import Conductor
|
||||
from torba.testcase import IntegrationTestCase
|
||||
import lbry.wallet
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.extras.daemon.Components import HeadersComponent
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
|
||||
class TestSessionBloat(IntegrationTestCase):
|
||||
@@ -29,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertTrue(session.is_closing())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
|
||||
|
||||
|
||||
class TestHeadersComponent(CommandTestCase):
|
||||
|
||||
LEDGER = lbry.wallet
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.component_manager = self.daemon.component_manager
|
||||
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
|
||||
self.headers_component = HeadersComponent(self.component_manager)
|
||||
|
||||
async def test_cant_reach_host(self):
|
||||
HeadersComponent.HEADERS_URL = 'notthere/'
|
||||
os.unlink(self.headers_component.headers.path)
|
||||
# test is that this doesnt raise
|
||||
await self.headers_component.start()
|
||||
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
|
||||
self.assertEqual(await self.headers_component.get_status(), {})
|
||||
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
# corrupt by appending
|
||||
headers.io.seek(block_bytes(len(headers)))
|
||||
headers.io.write(b"appending")
|
||||
headers._size = None
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 1499)
|
||||
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
|
||||
|
||||
@@ -115,7 +115,7 @@ class BaseHeaders:
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
self._size = self.io.tell() // self.header_size
|
||||
added += written
|
||||
if bail:
|
||||
break
|
||||
@@ -170,10 +170,12 @@ class BaseHeaders:
|
||||
|
||||
async def repair(self):
|
||||
previous_header_hash = fail = None
|
||||
self.io.seek(0)
|
||||
batch_size = 10000
|
||||
batch_size = 36
|
||||
for start_height in range(0, self.height, batch_size):
|
||||
self.io.seek(self.header_size * start_height)
|
||||
headers = self.io.read(self.header_size*batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
||||
height = header['block_height']
|
||||
if height:
|
||||
@@ -184,7 +186,7 @@ class BaseHeaders:
|
||||
fail = True
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
@@ -205,7 +207,7 @@ class BaseHeaders:
|
||||
end = min(len(headers), end + self.chunk_size * self.header_size)
|
||||
|
||||
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
|
||||
assert len(headers) % self.header_size == 0
|
||||
assert len(headers) % self.header_size == 0, len(headers)
|
||||
for idx in range(len(headers) // self.header_size):
|
||||
start, end = idx * self.header_size, (idx + 1) * self.header_size
|
||||
header = headers[start:end]
|
||||
|
||||
Reference in new issue
Block a user