Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
11617cefa7 | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 |
No files matched your search
@@ -8,7 +8,7 @@ install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.70.0"
|
||||
__version__ = "0.71.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+23
-9
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -198,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -319,8 +327,14 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
+4
-3
@@ -691,9 +691,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
@@ -2237,7 +2237,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
|
||||
if 'is_spent' not in kwargs:
|
||||
if not kwargs.get('is_spent', False):
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(**kwargs)
|
||||
|
||||
@@ -2971,6 +2971,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
|
||||
start = time.perf_counter()
|
||||
log.info("got publish request for '%s'", name)
|
||||
self.valid_stream_name_or_error(name)
|
||||
wallet = self.wallet_manager.get_wallet_or_default(kwargs.get('wallet_id'))
|
||||
if kwargs.get('account_id'):
|
||||
@@ -2980,6 +2983,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
claims = await self.ledger.get_claims(
|
||||
wallet=wallet, accounts=accounts, claim_name=name
|
||||
)
|
||||
log.info("got claim list for '%s' in %fs", name, time.perf_counter()-start)
|
||||
if len(claims) == 0:
|
||||
if 'bid' not in kwargs:
|
||||
raise Exception("'bid' is a required argument for new publishes.")
|
||||
@@ -3172,14 +3176,19 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
|
||||
self.valid_stream_name_or_error(name)
|
||||
account = wallet.get_account_or_default(account_id)
|
||||
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
|
||||
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
|
||||
log.info("got channel for '%s' in %fs", name, time.perf_counter()-start)
|
||||
start = time.perf_counter()
|
||||
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
|
||||
claim_address = await self.get_receiving_address(claim_address, account)
|
||||
log.info("got claim address for '%s' in %fs", name, time.perf_counter()-start)
|
||||
start = time.perf_counter()
|
||||
kwargs['fee_address'] = self.get_fee_address(kwargs, claim_address)
|
||||
|
||||
claims = await account.get_claims(claim_name=name)
|
||||
@@ -3189,35 +3198,47 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"You already have a stream claim published under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
|
||||
log.info("checked claims for '%s' in %fs", name, time.perf_counter()-start)
|
||||
start = time.perf_counter()
|
||||
file_path, spec = await self._video_file_analyzer.verify_or_repair(
|
||||
validate_file, optimize_file, file_path, ignore_non_video=True
|
||||
)
|
||||
log.info("ran ffmpeg for '%s' in %fs", name, time.perf_counter()-start)
|
||||
kwargs.update(spec)
|
||||
|
||||
claim = Claim()
|
||||
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
|
||||
start = time.perf_counter()
|
||||
tx = await Transaction.claim_create(
|
||||
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
|
||||
)
|
||||
new_txo = tx.outputs[0]
|
||||
|
||||
log.info("created tx for '%s' in %fs", name, time.perf_counter()-start)
|
||||
file_stream = None
|
||||
if not preview:
|
||||
start = time.perf_counter()
|
||||
file_stream = await self.stream_manager.create_stream(file_path)
|
||||
claim.stream.source.sd_hash = file_stream.sd_hash
|
||||
new_txo.script.generate()
|
||||
log.info("created stream for '%s' in %fs", name, time.perf_counter() - start)
|
||||
|
||||
if channel:
|
||||
new_txo.sign(channel)
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
log.info("broadcast and release")
|
||||
start = time.perf_counter()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
log.info("broadcast '%s' in %fs", name, time.perf_counter() - start)
|
||||
start = time.perf_counter()
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
log.info("saved claims '%s' in %fs", name, time.perf_counter() - start)
|
||||
start = time.perf_counter()
|
||||
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
|
||||
log.info("saved content claim '%s' in %fs", name, time.perf_counter() - start)
|
||||
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish'))
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
+17
-10
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def file_reader(file_path: str):
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -245,7 +252,7 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
for blob_bytes in file_reader(file_path):
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
|
||||
@@ -343,9 +343,10 @@ class ManagedStream:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -355,12 +356,12 @@ class ManagedStream:
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
open(output_path, 'wb').close()
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
|
||||
@@ -740,100 +740,4 @@ HASHES = {
|
||||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
|
||||
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
|
||||
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
|
||||
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
|
||||
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
|
||||
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
|
||||
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
|
||||
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
|
||||
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
|
||||
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
|
||||
}
|
||||
CHUNK_TIMES = [
|
||||
1446058291, 1466796062, 1466957160, 1467112807, 1467272478, 1467432743, 1467593826, 1467753971, 1467911127,
|
||||
1468070719, 1468224507, 1468384171, 1468543503, 1468703272, 1468863520, 1469023822, 1469183471, 1469342714,
|
||||
1469501778, 1469667089, 1469827045, 1469989357, 1470149385, 1470308706, 1470467480, 1470628399, 1470789513,
|
||||
1470949409, 1471110369, 1471271233, 1471432048, 1471589330, 1471749739, 1471910365, 1472071525, 1472235982,
|
||||
1472398890, 1472558207, 1472721074, 1472885421, 1473046344, 1473205609, 1473368205, 1473533441, 1473692268,
|
||||
1473853245, 1474016588, 1474178224, 1474340711, 1474502952, 1474662243, 1474821799, 1474981569, 1475145919,
|
||||
1475307424, 1475467443, 1475630023, 1475791989, 1475952599, 1476111994, 1476271960, 1476433228, 1476595751,
|
||||
1476756365, 1476918643, 1477080426, 1477244266, 1477404425, 1477564644, 1477729818, 1477890614, 1478052281,
|
||||
1478213943, 1478372873, 1478531715, 1478694298, 1478856902, 1479021042, 1479181793, 1479344484, 1479506486,
|
||||
1479663432, 1479825937, 1479985117, 1480148599, 1480310388, 1480474658, 1480639077, 1480800300, 1480960405,
|
||||
1481122737, 1481282632, 1481445451, 1481605639, 1481767227, 1481926963, 1482091620, 1482252291, 1482411821,
|
||||
1482573703, 1482736912, 1482897678, 1483058633, 1483218595, 1483378674, 1483541215, 1483702499, 1483863896,
|
||||
1484022728, 1484180459, 1484342882, 1484507416, 1484670823, 1484829895, 1484993458, 1485156047, 1485317583,
|
||||
1485481218, 1485643245, 1485805928, 1485967452, 1486133206, 1486293958, 1486456274, 1486617735, 1486778872,
|
||||
1486940299, 1487107013, 1487273582, 1487438229, 1487599911, 1487759331, 1487921967, 1488080929, 1488241128,
|
||||
1488404512, 1488565344, 1488725701, 1488887005, 1489047968, 1489207675, 1489371326, 1489535725, 1489703338,
|
||||
1489866064, 1490029222, 1490191595, 1490351274, 1490512577, 1490671939, 1490829835, 1490994376, 1491154639,
|
||||
1491317939, 1491481515, 1491643279, 1491804437, 1491964639, 1492124263, 1492284273, 1492445688, 1492605884,
|
||||
1492765009, 1492927220, 1493088404, 1493251395, 1493412278, 1493573461, 1493732746, 1493893060, 1494053987,
|
||||
1494216326, 1494376404, 1494540073, 1494702026, 1494862527, 1495024065, 1495185322, 1495344568, 1495506617,
|
||||
1495666173, 1495826079, 1495987151, 1496148961, 1496309855, 1496473442, 1496634935, 1496796598, 1496961204,
|
||||
1497121769, 1497283066, 1497443548, 1497603288, 1497767555, 1497933761, 1498094921, 1498256760, 1498420931,
|
||||
1498584545, 1498749559, 1498913792, 1499077566, 1499239970, 1499404091, 1499566804, 1499731157, 1499895613,
|
||||
1500059235, 1500222621, 1500388811, 1500554678, 1500721542, 1500884471, 1501044844, 1501209905, 1501371165,
|
||||
1501530890, 1501694185, 1501857382, 1502018129, 1502184292, 1502350341, 1502511801, 1502676243, 1502838811,
|
||||
1503002137, 1503163638, 1503325023, 1503486684, 1503650746, 1503812386, 1503973244, 1504133666, 1504297628,
|
||||
1504459991, 1504623358, 1504784832, 1504946961, 1505110359, 1505273975, 1505437560, 1505598954, 1505757098,
|
||||
1505917242, 1506077971, 1506237591, 1506398935, 1506559037, 1506720796, 1506885747, 1507050316, 1507212971,
|
||||
1507375409, 1507537815, 1507702332, 1507864531, 1508025612, 1508185476, 1508345744, 1508508500, 1508670141,
|
||||
1508832224, 1508992717, 1509152807, 1509310842, 1509473481, 1509633634, 1509796495, 1509954849, 1510116079,
|
||||
1510277697, 1510439798, 1510600524, 1510761373, 1510921787, 1511081683, 1511243190, 1511403276, 1511563575,
|
||||
1511723540, 1511884418, 1512045789, 1512207376, 1512368713, 1512530546, 1512691400, 1512852021, 1513012694,
|
||||
1513173368, 1513334988, 1513495509, 1513657679, 1513817134, 1513978820, 1514142957, 1514304599, 1514465234,
|
||||
1514624591, 1514785874, 1514945051, 1515108188, 1515268796, 1515430685, 1515590234, 1515751217, 1515910476,
|
||||
1516068662, 1516228676, 1516387357, 1516547616, 1516707229, 1516867094, 1517026140, 1517186251, 1517345155,
|
||||
1517505393, 1517664993, 1517824788, 1517983415, 1518144196, 1518303260, 1518464979, 1518625788, 1518785521,
|
||||
1518945093, 1519103570, 1519263344, 1519423481, 1519584426, 1519746582, 1519907472, 1520066218, 1520226493,
|
||||
1520385895, 1520543541, 1520702453, 1520861359, 1521018892, 1521177734, 1521341622, 1521502375, 1521662214,
|
||||
1521822716, 1521982142, 1522142729, 1522301856, 1522459931, 1522620631, 1522780689, 1522940183, 1523099791,
|
||||
1523258399, 1523420479, 1523579443, 1523739555, 1523899500, 1524059916, 1524219172, 1524378005, 1524537757,
|
||||
1524701031, 1524861748, 1525021162, 1525180528, 1525341200, 1525501213, 1525662981, 1525823008, 1525982484,
|
||||
1526142364, 1526300429, 1526459717, 1526620687, 1526781127, 1526940119, 1527099136, 1527259200, 1527418437,
|
||||
1527579937, 1527739330, 1527898324, 1528059125, 1528218736, 1528379347, 1528540251, 1528700525, 1528859406,
|
||||
1529018377, 1529178147, 1529339191, 1529498669, 1529656834, 1529815415, 1529977050, 1530135713, 1530295787,
|
||||
1530456752, 1530616682, 1530776198, 1530937477, 1531097196, 1531259398, 1531419197, 1531579213, 1531739816,
|
||||
1531899928, 1532060554, 1532220789, 1532378546, 1532537193, 1532695877, 1532852995, 1533013229, 1533170401,
|
||||
1533329105, 1533488406, 1533646944, 1533807289, 1533965434, 1534125093, 1534284340, 1534442620, 1534600547,
|
||||
1534759292, 1534920427, 1535082093, 1535241742, 1535404226, 1535566494, 1535726910, 1535884899, 1536042729,
|
||||
1536201423, 1536361908, 1536520607, 1536679327, 1536840381, 1536999216, 1537157660, 1537318284, 1537477213,
|
||||
1537635506, 1537793639, 1537953780, 1538117759, 1538278913, 1538439538, 1538597977, 1538756382, 1538915421,
|
||||
1539077166, 1539240317, 1539400708, 1539562594, 1539722205, 1539881920, 1540042540, 1540202519, 1540363464,
|
||||
1540521096, 1540681043, 1540838897, 1541000630, 1541159754, 1541320837, 1541479768, 1541640101, 1541798960,
|
||||
1541957879, 1542114653, 1542274838, 1542433802, 1542592336, 1542752146, 1542913026, 1543072942, 1543231105,
|
||||
1543390483, 1543548911, 1543705884, 1543864964, 1544025570, 1544185854, 1544343817, 1544503407, 1544661411,
|
||||
1544819366, 1544980849, 1545142380, 1545303819, 1545464301, 1545624146, 1545783061, 1545942620, 1546102174,
|
||||
1546262550, 1546421421, 1546580475, 1546741261, 1546901871, 1547061606, 1547221600, 1547379723, 1547538785,
|
||||
1547697419, 1547857490, 1548017070, 1548176553, 1548335542, 1548495290, 1548656463, 1548815906, 1548976481,
|
||||
1549134901, 1549293055, 1549453525, 1549615159, 1549775541, 1549934636, 1550095275, 1550255002, 1550416229,
|
||||
1550577697, 1550739302, 1550897867, 1551057080, 1551218892, 1551379080, 1551538880, 1551700931, 1551860435,
|
||||
1552018109, 1552178077, 1552337707, 1552497354, 1552659923, 1552821230, 1552980036, 1553138910, 1553298180,
|
||||
1553457714, 1553616756, 1553777086, 1553937586, 1554098177, 1554257525, 1554415929, 1554574675, 1554734734,
|
||||
1554895337, 1555054185, 1555214104, 1555375518, 1555532785, 1555694268, 1555851788, 1556013303, 1556174215,
|
||||
1556332761, 1556494320, 1556655155, 1556816169, 1556975478, 1557132393, 1557293583, 1557451564, 1557612602,
|
||||
1557772604, 1557931689, 1558091515, 1558250159, 1558409668, 1558568765, 1558726947, 1558886991, 1559046885,
|
||||
1559205383, 1559364094, 1559524528, 1559686227, 1559844138, 1560003768, 1560163656, 1560324916, 1560484561,
|
||||
1560645681, 1560808735, 1560971999, 1561135851, 1561297483, 1561455206, 1561614235, 1561777714, 1561945864,
|
||||
1562118106, 1562278703, 1562437641, 1562603277, 1562767483, 1562927551, 1563087487, 1563248549, 1563410262,
|
||||
1563573257, 1563733848, 1563894208, 1564054018, 1564212252, 1564374334, 1564532357, 1564693415, 1564854450,
|
||||
1565015965, 1565175795, 1565339141, 1565497044, 1565655699, 1565818978, 1565981118, 1566141639, 1566302976,
|
||||
1566465005, 1566625379, 1566786170, 1566946313, 1567108918, 1567270890, 1567430354, 1567588684, 1567749712,
|
||||
1567908836, 1568068607, 1568229349, 1568390117, 1568549698, 1568709328, 1568867513, 1569025318, 1569185425,
|
||||
1569344544, 1569503324, 1569664889, 1569823995, 1569986419, 1570146118, 1570306012, 1570467311, 1570628581,
|
||||
1570787594, 1570948886, 1571110097, 1571270823, 1571430280, 1571588101, 1571750508, 1571911503, 1572071265,
|
||||
1572232449, 1572392837, 1572553922, 1572713730, 1572875606, 1573036488, 1573196968, 1573362152, 1573521448,
|
||||
1573683075, 1573843849, 1574003422, 1574162840, 1574323813, 1574484633, 1574643754, 1574807310, 1574966691,
|
||||
1575125631, 1575285152, 1575444009, 1575605511, 1575765796, 1575924265, 1576087060, 1576246324, 1576404707,
|
||||
1576562942, 1576724468, 1576885925, 1577045396, 1577204522, 1577365567, 1577524358, 1577684296, 1577847223,
|
||||
1578008388, 1578166426, 1578327792, 1578489088, 1578650676, 1578809573, 1578967948, 1579125782, 1579285861,
|
||||
1579449027, 1579608917, 1579768221, 1579927859, 1580086975, 1580247978, 1580408360, 1580569255, 1580728734,
|
||||
1580888441, 1581047850, 1581208990, 1581369823, 1581528564, 1581689593, 1581848711, 1582008320, 1582169619,
|
||||
1582329254, 1582490606, 1582653889, 1582813888, 1582976432, 1583137185, 1583299285, 1583463034, 1583625464,
|
||||
1583786150, 1583949616, 1584109878, 1584268907, 1584428799, 1584589342, 1584749487, 1584911825, 1585073681,
|
||||
1585234424, 1585394114, 1585556112, 1585716022, 1585876422, 1586036811, 1586196631, 1586356275, 1586519756,
|
||||
1586678956, 1586838190, 1587004938, 1587160029
|
||||
]
|
||||
+46
-55
@@ -5,7 +5,6 @@ import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
@@ -13,7 +12,7 @@ from binascii import hexlify, unhexlify
|
||||
|
||||
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
||||
from lbry.wallet.util import ArithUint256, date_to_julian_day
|
||||
from .checkpoints import HASHES, CHUNK_TIMES
|
||||
from .checkpoints import HASHES
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -42,23 +41,22 @@ class Headers:
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.io = None
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.io = BytesIO()
|
||||
if self.path != ':memory:':
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
def _readit():
|
||||
if os.path.exists(self.path):
|
||||
with open(self.path, 'r+b') as header_file:
|
||||
self.io.seek(0)
|
||||
self.io.write(header_file.read())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
@@ -72,10 +70,14 @@ class Headers:
|
||||
await self.get_all_missing_headers()
|
||||
|
||||
async def close(self):
|
||||
if self.executor:
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
self.io.close()
|
||||
if self.io is not None:
|
||||
def _close():
|
||||
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||
with open(self.path, flags) as header_file:
|
||||
header_file.write(self.io.getbuffer())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||
self.io.close()
|
||||
self.io = None
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
@@ -135,34 +137,30 @@ class Headers:
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height):
|
||||
def estimated_timestamp(self, height, try_real_headers=True):
|
||||
if height <= 0:
|
||||
return
|
||||
if height >= max(self.checkpoints.keys()):
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
chunk_index = min(height // 1000, len(CHUNK_TIMES) - 1)
|
||||
offset = height - chunk_index * 1000
|
||||
chunk_start_time = CHUNK_TIMES[chunk_index]
|
||||
chunk_end_time = CHUNK_TIMES[chunk_index + 1]
|
||||
return int(offset * ((chunk_end_time - chunk_start_time) / 1000) + chunk_start_time)
|
||||
if try_real_headers and self.has_header(height):
|
||||
offset = height * self.header_size
|
||||
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
|
||||
return self._read(height)
|
||||
|
||||
def _read(self, height, count=1):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size * count)
|
||||
offset = height * self.header_size
|
||||
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
self.io.seek(start * self.header_size, os.SEEK_SET)
|
||||
return self.hash_header(self.io.read(count * self.header_size)).decode()
|
||||
return self.hash_header(self._read(start, count)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
@@ -171,7 +169,7 @@ class Headers:
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if await self.has_header(height):
|
||||
if self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
@@ -185,7 +183,7 @@ class Headers:
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
|
||||
self._write(start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
@@ -195,27 +193,23 @@ class Headers:
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
async def has_header(self, height):
|
||||
def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
def _has_header(height):
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
def _io_checkall():
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
@@ -247,7 +241,7 @@ class Headers:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
|
||||
added += self._write(height, chunk)
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
@@ -312,9 +306,7 @@ class Headers:
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self._read, height, batch_size
|
||||
)
|
||||
headers = self._read(height, batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
@@ -330,12 +322,11 @@ class Headers:
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
def __truncate(at_height):
|
||||
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -929,7 +929,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
item = {
|
||||
'txid': tx.id,
|
||||
'timestamp': ts,
|
||||
'date': datetime.utcfromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
|
||||
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
|
||||
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
|
||||
'claim_info': [],
|
||||
'update_info': [],
|
||||
|
||||
@@ -547,10 +547,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
if for_count or attr == 'tag':
|
||||
if attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
(claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
@@ -606,10 +606,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
if for_count:
|
||||
if attr == 'tag':
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
(claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import textwrap
|
||||
import os
|
||||
|
||||
from lbry.extras.cli import ensure_directory_exists
|
||||
@@ -25,18 +24,11 @@ async def main():
|
||||
return
|
||||
print(f"Headers file at {headers.height}, checkpointing up to {target}."
|
||||
f"Current checkpoint at {current_checkpoint_tip}.")
|
||||
timestamps = []
|
||||
with open(outpath, 'w') as outfile:
|
||||
print('HASHES = {', file=outfile)
|
||||
for height in range(0, target, 1000):
|
||||
timestamps.append((await headers.get(height))['timestamp'])
|
||||
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
|
||||
print('}', file=outfile)
|
||||
print('CHUNK_TIMES = [', file=outfile)
|
||||
for line in textwrap.wrap(f'{", ".join(map(str, timestamps))}', 120, initial_indent=' '*4, subsequent_indent=' '*4):
|
||||
print(line, file=outfile)
|
||||
print(']', file=outfile)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -661,12 +661,15 @@ class ClaimCommands(ClaimTestCase):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
|
||||
await self.stream_update(stream_id, title='foo')
|
||||
|
||||
# type filtering
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type='stream')
|
||||
# catch a bug where cli sends is_spent=False by default
|
||||
r = await self.claim_list(claim_type='stream', is_spent=False)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
|
||||
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
|
||||
for save_blobs in (False, True):
|
||||
await self.setup_blob_manager(save_blobs=save_blobs)
|
||||
# add a blob file
|
||||
@@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
blob.save_verified_blob(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.blob_manager.blob_completed(blob)
|
||||
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
|
||||
|
||||
@@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
write_blob = blob._write_blob
|
||||
write_called_count = 0
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
async def _wrap_write_blob(blob_bytes):
|
||||
nonlocal write_called_count
|
||||
write_called_count += 1
|
||||
write_blob(blob_bytes)
|
||||
await write_blob(blob_bytes)
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
return asyncio.create_task(_wrap_write_blob(blob_bytes))
|
||||
|
||||
blob._write_blob = wrap_write_blob
|
||||
|
||||
writer1 = blob.get_blob_writer(peer_port=1)
|
||||
@@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
|
||||
self.assertEqual(1, write_called_count)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.assertDictEqual({}, blob.writers)
|
||||
|
||||
|
||||
@@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
await self.ledger.headers.open()
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase):
|
||||
async def test_deserialize(self):
|
||||
self.maxDiff = None
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS)
|
||||
await h.open()
|
||||
await h.connect(0, HEADERS)
|
||||
self.assertEqual(await h.get(0), {
|
||||
'bits': 520159231,
|
||||
'block_height': 0,
|
||||
@@ -52,8 +52,11 @@ class TestHeaders(AsyncioTestCase):
|
||||
self.assertEqual(headers.height, 19)
|
||||
|
||||
async def test_connect_from_middle(self):
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS[:block_bytes(10)])
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
h = Headers(headers_temporary_file)
|
||||
await h.open()
|
||||
self.assertEqual(h.height, 9)
|
||||
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
|
||||
@@ -115,6 +118,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_bounds(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS)
|
||||
self.assertEqual(19, headers.height)
|
||||
with self.assertRaises(IndexError):
|
||||
@@ -126,6 +130,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_repair(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS[:block_bytes(11)])
|
||||
self.assertEqual(10, headers.height)
|
||||
await headers.repair()
|
||||
@@ -147,33 +152,39 @@ class TestHeaders(AsyncioTestCase):
|
||||
await headers.repair(start_height=10)
|
||||
self.assertEqual(19, headers.height)
|
||||
|
||||
def test_do_not_estimate_unconfirmed(self):
|
||||
async def test_do_not_estimate_unconfirmed(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
self.assertIsNone(headers.estimated_timestamp(-1))
|
||||
self.assertIsNone(headers.estimated_timestamp(0))
|
||||
self.assertIsNotNone(headers.estimated_timestamp(1))
|
||||
|
||||
def test_estimate_from_checkpoints(self):
|
||||
headers = _Headers(':memory:') # here we use the real version, with checkpoints
|
||||
# check estimations matches real timestamps (LBRY, main net) by at most 20 minutes
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(101010), 1482899706, delta=60 * 20)
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(505050), 1547865484, delta=60 * 20)
|
||||
self.assertAlmostEqual(headers.estimated_timestamp(707070), 1580258295, delta=60 * 20)
|
||||
# tip uses old algorithm as the range doesnt exist
|
||||
self.assertIsNotNone(headers.estimated_timestamp(max(headers.checkpoints.keys())))
|
||||
async def test_dont_estimate_whats_there(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
estimated = headers.estimated_timestamp(10)
|
||||
await headers.connect(0, HEADERS)
|
||||
real_time = (await headers.get(10))['timestamp']
|
||||
after_downloading_header_estimated = headers.estimated_timestamp(10)
|
||||
self.assertNotEqual(estimated, after_downloading_header_estimated)
|
||||
self.assertEqual(after_downloading_header_estimated, real_time)
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
headers = Headers(':memory:')
|
||||
headers.io.seek(0)
|
||||
headers.io.write(HEADERS)
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS)
|
||||
headers = Headers(headers_temporary_file)
|
||||
with self.assertLogs(level='WARN') as cm:
|
||||
await headers.open()
|
||||
await headers.close()
|
||||
self.assertEqual(cm.output, [])
|
||||
headers.io.seek(0)
|
||||
headers.io.truncate()
|
||||
headers.io.write(HEADERS[:block_bytes(10)])
|
||||
headers.io.write(b'ops')
|
||||
headers.io.write(HEADERS[block_bytes(10):])
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.seek(0)
|
||||
headers_file.truncate()
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
headers_file.write(b'ops')
|
||||
headers_file.write(HEADERS[block_bytes(10):])
|
||||
await headers.open()
|
||||
self.assertEqual(
|
||||
cm.output, [
|
||||
@@ -201,6 +212,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
reader_task = asyncio.create_task(reader())
|
||||
await writer()
|
||||
await reader_task
|
||||
await headers.close()
|
||||
|
||||
|
||||
HEADERS = unhexlify(
|
||||
|
||||
@@ -48,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
self.ledger.headers.checkpoints = {}
|
||||
await self.ledger.headers.open()
|
||||
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
Reference in new issue
Block a user