Compare commits

..
Author SHA1 Message Date
Jack Robison 11617cefa7 debug 2020-04-27 12:30:36 -04:00
Lex Berezhny 00c0f48b02 v0.71.0 2020-04-27 12:21:04 -04:00
Lex Berezhny 94c45cf2a0 Merge pull request #2922 from lbryio/estimate_on_demand
estimate block timestamps on client only when necessary (header hasn't been downloaded yet)
2020-04-27 12:17:54 -04:00
Victor Shyba 58f77b2a1c load/dump header file using executor 2020-04-27 10:34:49 -04:00
Victor Shyba 7170e69b22 test fixes + leave tx plot always on estimations 2020-04-27 10:34:49 -04:00
Victor Shyba 239ee2437c estimate only whats not downloaded 2020-04-27 10:34:49 -04:00
Victor Shyba ced368db31 hold headers file in memory during runtime 2020-04-27 10:34:49 -04:00
Lex Berezhny 6a991e5c15 Merge pull request #2914 from StripedMonkey/SM-Config-patch
use home directory if xdg directory not found (Linux)
2020-04-27 10:33:57 -04:00
Lex Berezhny 523d22262b pin pylint to 2.4.4 2020-04-27 10:18:39 -04:00
Noah 08197a327e fix Missing xdg download location
Fixes  an error in detection xdg config locations when XDG_DOWNLOAD_DIR is not present in `user-dirs.dirs`
2020-04-27 10:11:16 -04:00
Jack Robison b7cb2a7aa5 Merge pull request #2926 from lbryio/blocking-file-io
make file i/o in stream and blob creation and reads non-blocking
2020-04-24 11:07:24 -04:00
Jack Robison decc5c74ef don't block when reading a file when creating a stream 2020-04-23 00:35:08 -04:00
Jack Robison fbe0f886b6 non blocking blob creation 2020-04-23 00:35:08 -04:00
Jack Robison 7e23d6e2ef Merge pull request #2925 from lbryio/fix-reposts-tag
fix: reposts being returned for single tags
2020-04-23 00:29:04 -04:00
Tom 49458d1085 fix: reposts being returned for single tags 2020-04-22 23:16:30 -04:00
Lex Berezhny 289c12bd9a Merge pull request #2924 from lbryio/claim_list_is_spent_fix
fix for `claim_list` incorrectly handling `--is_spent` flag
2020-04-22 11:34:16 -04:00
Lex Berezhny 9a6326b027 fix for claim_list incorrectly handling --is_spent flag 2020-04-22 10:36:09 -04:00
19 changed files with 182 additions and 225 deletions

No files matched your search

+1 -1
View File
@@ -8,7 +8,7 @@ install:
pip install -e .
tools:
pip install mypy==0.701
pip install mypy==0.701 pylint==2.4.4
pip install coverage astroid pylint
lint:
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.70.0"
__version__ = "0.71.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+23 -9
View File
@@ -110,7 +110,7 @@ class AbstractBlob:
if reader in self.readers:
self.readers.remove(reader)
def _write_blob(self, blob_bytes: bytes):
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
raise NotImplementedError()
def set_length(self, length) -> None:
@@ -198,11 +198,17 @@ class AbstractBlob:
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
def update_events(_):
self.verified.set()
self.writing.clear()
if self.is_writeable():
self.writing.set()
task = self._write_blob(verified_bytes)
task.add_done_callback(update_events)
if self.blob_completed_callback:
self.blob_completed_callback(self)
task.add_done_callback(lambda _: self.blob_completed_callback(self))
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
async def write():
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
return self.loop.create_task(write())
def delete(self):
if self._verified_bytes:
@@ -319,8 +327,14 @@ class BlobFile(AbstractBlob):
handle.close()
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def _write_blob():
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
async def write_blob():
await self.loop.run_in_executor(None, _write_blob)
return self.loop.create_task(write_blob())
def delete(self):
if os.path.isfile(self.file_path):
+2
View File
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
# wait for the io to finish
await self.blob.verified.wait()
log.info("%s at %fMB/s", msg,
round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0, 2))
+4 -3
View File
@@ -691,9 +691,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
if down_dir:
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
download_dir = re.sub('\"', '', down_dir)
except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
+24 -3
View File
@@ -2237,7 +2237,7 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
if 'is_spent' not in kwargs:
if not kwargs.get('is_spent', False):
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(**kwargs)
@@ -2971,6 +2971,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Transaction}
"""
start = time.perf_counter()
log.info("got publish request for '%s'", name)
self.valid_stream_name_or_error(name)
wallet = self.wallet_manager.get_wallet_or_default(kwargs.get('wallet_id'))
if kwargs.get('account_id'):
@@ -2980,6 +2983,7 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await self.ledger.get_claims(
wallet=wallet, accounts=accounts, claim_name=name
)
log.info("got claim list for '%s' in %fs", name, time.perf_counter()-start)
if len(claims) == 0:
if 'bid' not in kwargs:
raise Exception("'bid' is a required argument for new publishes.")
@@ -3172,14 +3176,19 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Transaction}
"""
start = time.perf_counter()
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
self.valid_stream_name_or_error(name)
account = wallet.get_account_or_default(account_id)
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
log.info("got channel for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
claim_address = await self.get_receiving_address(claim_address, account)
log.info("got claim address for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
kwargs['fee_address'] = self.get_fee_address(kwargs, claim_address)
claims = await account.get_claims(claim_name=name)
@@ -3189,35 +3198,47 @@ class Daemon(metaclass=JSONRPCServerType):
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
)
log.info("checked claims for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
file_path, spec = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
log.info("ran ffmpeg for '%s' in %fs", name, time.perf_counter()-start)
kwargs.update(spec)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
start = time.perf_counter()
tx = await Transaction.claim_create(
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
)
new_txo = tx.outputs[0]
log.info("created tx for '%s' in %fs", name, time.perf_counter()-start)
file_stream = None
if not preview:
start = time.perf_counter()
file_stream = await self.stream_manager.create_stream(file_path)
claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
log.info("created stream for '%s' in %fs", name, time.perf_counter() - start)
if channel:
new_txo.sign(channel)
await tx.sign(funding_accounts)
if not preview:
log.info("broadcast and release")
start = time.perf_counter()
await self.broadcast_or_release(tx, blocking)
log.info("broadcast '%s' in %fs", name, time.perf_counter() - start)
start = time.perf_counter()
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
log.info("saved claims '%s' in %fs", name, time.perf_counter() - start)
start = time.perf_counter()
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
log.info("saved content claim '%s' in %fs", name, time.perf_counter() - start)
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish'))
else:
await account.ledger.release_tx(tx)
+17 -10
View File
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
yield os.urandom(AES.block_size // 8)
def file_reader(file_path: str):
def read_bytes(file_path: str, offset: int, to_read: int):
with open(file_path, 'rb') as f:
f.seek(offset)
return f.read(to_read)
async def file_reader(file_path: str):
length = int(os.stat(file_path).st_size)
offset = 0
with open(file_path, 'rb') as stream_file:
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = stream_file.read(bytes_to_read)
yield blob_bytes
offset += bytes_to_read
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = await asyncio.get_event_loop().run_in_executor(
None, read_bytes, file_path, offset, bytes_to_read
)
yield blob_bytes
offset += bytes_to_read
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
@@ -245,7 +252,7 @@ class StreamDescriptor:
iv_generator = iv_generator or random_iv_generator()
key = key or os.urandom(AES.block_size // 8)
blob_num = -1
for blob_bytes in file_reader(file_path):
async for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
+10 -9
View File
@@ -343,9 +343,10 @@ class ManagedStream:
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
def _write_decrypted_blob(output_path: str, data: bytes):
with open(output_path, 'ab') as handle:
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
@@ -355,12 +356,12 @@ class ManagedStream:
self.finished_writing.clear()
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
open(output_path, 'wb').close()
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
-96
View File
@@ -740,100 +740,4 @@ HASHES = {
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
}
CHUNK_TIMES = [
1446058291, 1466796062, 1466957160, 1467112807, 1467272478, 1467432743, 1467593826, 1467753971, 1467911127,
1468070719, 1468224507, 1468384171, 1468543503, 1468703272, 1468863520, 1469023822, 1469183471, 1469342714,
1469501778, 1469667089, 1469827045, 1469989357, 1470149385, 1470308706, 1470467480, 1470628399, 1470789513,
1470949409, 1471110369, 1471271233, 1471432048, 1471589330, 1471749739, 1471910365, 1472071525, 1472235982,
1472398890, 1472558207, 1472721074, 1472885421, 1473046344, 1473205609, 1473368205, 1473533441, 1473692268,
1473853245, 1474016588, 1474178224, 1474340711, 1474502952, 1474662243, 1474821799, 1474981569, 1475145919,
1475307424, 1475467443, 1475630023, 1475791989, 1475952599, 1476111994, 1476271960, 1476433228, 1476595751,
1476756365, 1476918643, 1477080426, 1477244266, 1477404425, 1477564644, 1477729818, 1477890614, 1478052281,
1478213943, 1478372873, 1478531715, 1478694298, 1478856902, 1479021042, 1479181793, 1479344484, 1479506486,
1479663432, 1479825937, 1479985117, 1480148599, 1480310388, 1480474658, 1480639077, 1480800300, 1480960405,
1481122737, 1481282632, 1481445451, 1481605639, 1481767227, 1481926963, 1482091620, 1482252291, 1482411821,
1482573703, 1482736912, 1482897678, 1483058633, 1483218595, 1483378674, 1483541215, 1483702499, 1483863896,
1484022728, 1484180459, 1484342882, 1484507416, 1484670823, 1484829895, 1484993458, 1485156047, 1485317583,
1485481218, 1485643245, 1485805928, 1485967452, 1486133206, 1486293958, 1486456274, 1486617735, 1486778872,
1486940299, 1487107013, 1487273582, 1487438229, 1487599911, 1487759331, 1487921967, 1488080929, 1488241128,
1488404512, 1488565344, 1488725701, 1488887005, 1489047968, 1489207675, 1489371326, 1489535725, 1489703338,
1489866064, 1490029222, 1490191595, 1490351274, 1490512577, 1490671939, 1490829835, 1490994376, 1491154639,
1491317939, 1491481515, 1491643279, 1491804437, 1491964639, 1492124263, 1492284273, 1492445688, 1492605884,
1492765009, 1492927220, 1493088404, 1493251395, 1493412278, 1493573461, 1493732746, 1493893060, 1494053987,
1494216326, 1494376404, 1494540073, 1494702026, 1494862527, 1495024065, 1495185322, 1495344568, 1495506617,
1495666173, 1495826079, 1495987151, 1496148961, 1496309855, 1496473442, 1496634935, 1496796598, 1496961204,
1497121769, 1497283066, 1497443548, 1497603288, 1497767555, 1497933761, 1498094921, 1498256760, 1498420931,
1498584545, 1498749559, 1498913792, 1499077566, 1499239970, 1499404091, 1499566804, 1499731157, 1499895613,
1500059235, 1500222621, 1500388811, 1500554678, 1500721542, 1500884471, 1501044844, 1501209905, 1501371165,
1501530890, 1501694185, 1501857382, 1502018129, 1502184292, 1502350341, 1502511801, 1502676243, 1502838811,
1503002137, 1503163638, 1503325023, 1503486684, 1503650746, 1503812386, 1503973244, 1504133666, 1504297628,
1504459991, 1504623358, 1504784832, 1504946961, 1505110359, 1505273975, 1505437560, 1505598954, 1505757098,
1505917242, 1506077971, 1506237591, 1506398935, 1506559037, 1506720796, 1506885747, 1507050316, 1507212971,
1507375409, 1507537815, 1507702332, 1507864531, 1508025612, 1508185476, 1508345744, 1508508500, 1508670141,
1508832224, 1508992717, 1509152807, 1509310842, 1509473481, 1509633634, 1509796495, 1509954849, 1510116079,
1510277697, 1510439798, 1510600524, 1510761373, 1510921787, 1511081683, 1511243190, 1511403276, 1511563575,
1511723540, 1511884418, 1512045789, 1512207376, 1512368713, 1512530546, 1512691400, 1512852021, 1513012694,
1513173368, 1513334988, 1513495509, 1513657679, 1513817134, 1513978820, 1514142957, 1514304599, 1514465234,
1514624591, 1514785874, 1514945051, 1515108188, 1515268796, 1515430685, 1515590234, 1515751217, 1515910476,
1516068662, 1516228676, 1516387357, 1516547616, 1516707229, 1516867094, 1517026140, 1517186251, 1517345155,
1517505393, 1517664993, 1517824788, 1517983415, 1518144196, 1518303260, 1518464979, 1518625788, 1518785521,
1518945093, 1519103570, 1519263344, 1519423481, 1519584426, 1519746582, 1519907472, 1520066218, 1520226493,
1520385895, 1520543541, 1520702453, 1520861359, 1521018892, 1521177734, 1521341622, 1521502375, 1521662214,
1521822716, 1521982142, 1522142729, 1522301856, 1522459931, 1522620631, 1522780689, 1522940183, 1523099791,
1523258399, 1523420479, 1523579443, 1523739555, 1523899500, 1524059916, 1524219172, 1524378005, 1524537757,
1524701031, 1524861748, 1525021162, 1525180528, 1525341200, 1525501213, 1525662981, 1525823008, 1525982484,
1526142364, 1526300429, 1526459717, 1526620687, 1526781127, 1526940119, 1527099136, 1527259200, 1527418437,
1527579937, 1527739330, 1527898324, 1528059125, 1528218736, 1528379347, 1528540251, 1528700525, 1528859406,
1529018377, 1529178147, 1529339191, 1529498669, 1529656834, 1529815415, 1529977050, 1530135713, 1530295787,
1530456752, 1530616682, 1530776198, 1530937477, 1531097196, 1531259398, 1531419197, 1531579213, 1531739816,
1531899928, 1532060554, 1532220789, 1532378546, 1532537193, 1532695877, 1532852995, 1533013229, 1533170401,
1533329105, 1533488406, 1533646944, 1533807289, 1533965434, 1534125093, 1534284340, 1534442620, 1534600547,
1534759292, 1534920427, 1535082093, 1535241742, 1535404226, 1535566494, 1535726910, 1535884899, 1536042729,
1536201423, 1536361908, 1536520607, 1536679327, 1536840381, 1536999216, 1537157660, 1537318284, 1537477213,
1537635506, 1537793639, 1537953780, 1538117759, 1538278913, 1538439538, 1538597977, 1538756382, 1538915421,
1539077166, 1539240317, 1539400708, 1539562594, 1539722205, 1539881920, 1540042540, 1540202519, 1540363464,
1540521096, 1540681043, 1540838897, 1541000630, 1541159754, 1541320837, 1541479768, 1541640101, 1541798960,
1541957879, 1542114653, 1542274838, 1542433802, 1542592336, 1542752146, 1542913026, 1543072942, 1543231105,
1543390483, 1543548911, 1543705884, 1543864964, 1544025570, 1544185854, 1544343817, 1544503407, 1544661411,
1544819366, 1544980849, 1545142380, 1545303819, 1545464301, 1545624146, 1545783061, 1545942620, 1546102174,
1546262550, 1546421421, 1546580475, 1546741261, 1546901871, 1547061606, 1547221600, 1547379723, 1547538785,
1547697419, 1547857490, 1548017070, 1548176553, 1548335542, 1548495290, 1548656463, 1548815906, 1548976481,
1549134901, 1549293055, 1549453525, 1549615159, 1549775541, 1549934636, 1550095275, 1550255002, 1550416229,
1550577697, 1550739302, 1550897867, 1551057080, 1551218892, 1551379080, 1551538880, 1551700931, 1551860435,
1552018109, 1552178077, 1552337707, 1552497354, 1552659923, 1552821230, 1552980036, 1553138910, 1553298180,
1553457714, 1553616756, 1553777086, 1553937586, 1554098177, 1554257525, 1554415929, 1554574675, 1554734734,
1554895337, 1555054185, 1555214104, 1555375518, 1555532785, 1555694268, 1555851788, 1556013303, 1556174215,
1556332761, 1556494320, 1556655155, 1556816169, 1556975478, 1557132393, 1557293583, 1557451564, 1557612602,
1557772604, 1557931689, 1558091515, 1558250159, 1558409668, 1558568765, 1558726947, 1558886991, 1559046885,
1559205383, 1559364094, 1559524528, 1559686227, 1559844138, 1560003768, 1560163656, 1560324916, 1560484561,
1560645681, 1560808735, 1560971999, 1561135851, 1561297483, 1561455206, 1561614235, 1561777714, 1561945864,
1562118106, 1562278703, 1562437641, 1562603277, 1562767483, 1562927551, 1563087487, 1563248549, 1563410262,
1563573257, 1563733848, 1563894208, 1564054018, 1564212252, 1564374334, 1564532357, 1564693415, 1564854450,
1565015965, 1565175795, 1565339141, 1565497044, 1565655699, 1565818978, 1565981118, 1566141639, 1566302976,
1566465005, 1566625379, 1566786170, 1566946313, 1567108918, 1567270890, 1567430354, 1567588684, 1567749712,
1567908836, 1568068607, 1568229349, 1568390117, 1568549698, 1568709328, 1568867513, 1569025318, 1569185425,
1569344544, 1569503324, 1569664889, 1569823995, 1569986419, 1570146118, 1570306012, 1570467311, 1570628581,
1570787594, 1570948886, 1571110097, 1571270823, 1571430280, 1571588101, 1571750508, 1571911503, 1572071265,
1572232449, 1572392837, 1572553922, 1572713730, 1572875606, 1573036488, 1573196968, 1573362152, 1573521448,
1573683075, 1573843849, 1574003422, 1574162840, 1574323813, 1574484633, 1574643754, 1574807310, 1574966691,
1575125631, 1575285152, 1575444009, 1575605511, 1575765796, 1575924265, 1576087060, 1576246324, 1576404707,
1576562942, 1576724468, 1576885925, 1577045396, 1577204522, 1577365567, 1577524358, 1577684296, 1577847223,
1578008388, 1578166426, 1578327792, 1578489088, 1578650676, 1578809573, 1578967948, 1579125782, 1579285861,
1579449027, 1579608917, 1579768221, 1579927859, 1580086975, 1580247978, 1580408360, 1580569255, 1580728734,
1580888441, 1581047850, 1581208990, 1581369823, 1581528564, 1581689593, 1581848711, 1582008320, 1582169619,
1582329254, 1582490606, 1582653889, 1582813888, 1582976432, 1583137185, 1583299285, 1583463034, 1583625464,
1583786150, 1583949616, 1584109878, 1584268907, 1584428799, 1584589342, 1584749487, 1584911825, 1585073681,
1585234424, 1585394114, 1585556112, 1585716022, 1585876422, 1586036811, 1586196631, 1586356275, 1586519756,
1586678956, 1586838190, 1587004938, 1587160029
]
+46 -55
View File
@@ -5,7 +5,6 @@ import asyncio
import logging
import zlib
from datetime import date
from concurrent.futures.thread import ThreadPoolExecutor
from io import BytesIO
from typing import Optional, Iterator, Tuple, Callable
@@ -13,7 +12,7 @@ from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES, CHUNK_TIMES
from .checkpoints import HASHES
log = logging.getLogger(__name__)
@@ -42,23 +41,22 @@ class Headers:
validate_difficulty: bool = True
def __init__(self, path) -> None:
if path == ':memory:':
self.io = BytesIO()
self.io = None
self.path = path
self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.executor = ThreadPoolExecutor(1)
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self):
if not self.executor:
self.executor = ThreadPoolExecutor(1)
self.io = BytesIO()
if self.path != ':memory:':
if not os.path.exists(self.path):
self.io = open(self.path, 'w+b')
else:
self.io = open(self.path, 'r+b')
def _readit():
if os.path.exists(self.path):
with open(self.path, 'r+b') as header_file:
self.io.seek(0)
self.io.write(header_file.read())
await asyncio.get_event_loop().run_in_executor(None, _readit)
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
@@ -72,10 +70,14 @@ class Headers:
await self.get_all_missing_headers()
async def close(self):
if self.executor:
self.executor.shutdown()
self.executor = None
self.io.close()
if self.io is not None:
def _close():
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
with open(self.path, flags) as header_file:
header_file.write(self.io.getbuffer())
await asyncio.get_event_loop().run_in_executor(None, _close)
self.io.close()
self.io = None
@staticmethod
def serialize(header):
@@ -135,34 +137,30 @@ class Headers:
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height):
def estimated_timestamp(self, height, try_real_headers=True):
if height <= 0:
return
if height >= max(self.checkpoints.keys()):
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
chunk_index = min(height // 1000, len(CHUNK_TIMES) - 1)
offset = height - chunk_index * 1000
chunk_start_time = CHUNK_TIMES[chunk_index]
chunk_end_time = CHUNK_TIMES[chunk_index + 1]
return int(offset * ((chunk_end_time - chunk_start_time) / 1000) + chunk_start_time)
if try_real_headers and self.has_header(height):
offset = height * self.header_size
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
return self._read(height)
def _read(self, height, count=1):
self.io.seek(height * self.header_size, os.SEEK_SET)
return self.io.read(self.header_size * count)
offset = height * self.header_size
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
def chunk_hash(self, start, count):
self.io.seek(start * self.header_size, os.SEEK_SET)
return self.hash_header(self.io.read(count * self.header_size)).decode()
return self.hash_header(self._read(start, count)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
@@ -171,7 +169,7 @@ class Headers:
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if await self.has_header(height):
if self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
@@ -185,7 +183,7 @@ class Headers:
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
self._write(start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
@@ -195,27 +193,23 @@ class Headers:
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
async def has_header(self, height):
def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
def _has_header(height):
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
def _io_checkall():
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
@property
def height(self) -> int:
@@ -247,7 +241,7 @@ class Headers:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
if chunk:
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
added += self._write(height, chunk)
if bail:
break
return added
@@ -312,9 +306,7 @@ class Headers:
previous_header_hash = fail = None
batch_size = 36
for height in range(start_height, self.height, batch_size):
headers = await asyncio.get_running_loop().run_in_executor(
self.executor, self._read, height, batch_size
)
headers = self._read(height, batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(height, headers):
@@ -330,12 +322,11 @@ class Headers:
assert start_height > 0 and height == start_height
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
def __truncate(at_height):
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return
previous_header_hash = header_hash
@classmethod
+1 -1
View File
@@ -929,7 +929,7 @@ class Ledger(metaclass=LedgerRegistry):
item = {
'txid': tx.id,
'timestamp': ts,
'date': datetime.utcfromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
'claim_info': [],
'update_info': [],
+5 -5
View File
@@ -547,10 +547,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
if for_count or attr == 'tag':
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
@@ -606,10 +606,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
if for_count:
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
constraints[f'#_not_{attr}'] = f"""
-8
View File
@@ -1,5 +1,4 @@
import asyncio
import textwrap
import os
from lbry.extras.cli import ensure_directory_exists
@@ -25,18 +24,11 @@ async def main():
return
print(f"Headers file at {headers.height}, checkpointing up to {target}."
f"Current checkpoint at {current_checkpoint_tip}.")
timestamps = []
with open(outpath, 'w') as outfile:
print('HASHES = {', file=outfile)
for height in range(0, target, 1000):
timestamps.append((await headers.get(height))['timestamp'])
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
print('}', file=outfile)
print('CHUNK_TIMES = [', file=outfile)
for line in textwrap.wrap(f'{", ".join(map(str, timestamps))}', 120, initial_indent=' '*4, subsequent_indent=' '*4):
print(line, file=outfile)
print(']', file=outfile)
if __name__ == "__main__":
@@ -661,12 +661,15 @@ class ClaimCommands(ClaimTestCase):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, title='foo')
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
r = await self.claim_list(claim_type='stream')
# catch a bug where cli sends is_spent=False by default
r = await self.claim_list(claim_type='stream', is_spent=False)
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
+2 -1
View File
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
@@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase):
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
@@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase):
write_blob = blob._write_blob
write_called_count = 0
def wrap_write_blob(blob_bytes):
async def _wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
write_blob(blob_bytes)
await write_blob(blob_bytes)
def wrap_write_blob(blob_bytes):
return asyncio.create_task(_wrap_write_blob(blob_bytes))
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
@@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase):
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
+1
View File
@@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
await self.ledger.headers.open()
self.wallet = Wallet()
await self.ledger.db.open()
+32 -20
View File
@@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase):
async def test_deserialize(self):
self.maxDiff = None
h = Headers(':memory:')
h.io.write(HEADERS)
await h.open()
await h.connect(0, HEADERS)
self.assertEqual(await h.get(0), {
'bits': 520159231,
'block_height': 0,
@@ -52,8 +52,11 @@ class TestHeaders(AsyncioTestCase):
self.assertEqual(headers.height, 19)
async def test_connect_from_middle(self):
h = Headers(':memory:')
h.io.write(HEADERS[:block_bytes(10)])
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS[:block_bytes(10)])
h = Headers(headers_temporary_file)
await h.open()
self.assertEqual(h.height, 9)
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
@@ -115,6 +118,7 @@ class TestHeaders(AsyncioTestCase):
async def test_bounds(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS)
self.assertEqual(19, headers.height)
with self.assertRaises(IndexError):
@@ -126,6 +130,7 @@ class TestHeaders(AsyncioTestCase):
async def test_repair(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS[:block_bytes(11)])
self.assertEqual(10, headers.height)
await headers.repair()
@@ -147,33 +152,39 @@ class TestHeaders(AsyncioTestCase):
await headers.repair(start_height=10)
self.assertEqual(19, headers.height)
def test_do_not_estimate_unconfirmed(self):
async def test_do_not_estimate_unconfirmed(self):
headers = Headers(':memory:')
await headers.open()
self.assertIsNone(headers.estimated_timestamp(-1))
self.assertIsNone(headers.estimated_timestamp(0))
self.assertIsNotNone(headers.estimated_timestamp(1))
def test_estimate_from_checkpoints(self):
headers = _Headers(':memory:') # here we use the real version, with checkpoints
# check estimations matches real timestamps (LBRY, main net) by at most 20 minutes
self.assertAlmostEqual(headers.estimated_timestamp(101010), 1482899706, delta=60 * 20)
self.assertAlmostEqual(headers.estimated_timestamp(505050), 1547865484, delta=60 * 20)
self.assertAlmostEqual(headers.estimated_timestamp(707070), 1580258295, delta=60 * 20)
# tip uses old algorithm as the range doesnt exist
self.assertIsNotNone(headers.estimated_timestamp(max(headers.checkpoints.keys())))
async def test_dont_estimate_whats_there(self):
headers = Headers(':memory:')
await headers.open()
estimated = headers.estimated_timestamp(10)
await headers.connect(0, HEADERS)
real_time = (await headers.get(10))['timestamp']
after_downloading_header_estimated = headers.estimated_timestamp(10)
self.assertNotEqual(estimated, after_downloading_header_estimated)
self.assertEqual(after_downloading_header_estimated, real_time)
async def test_misalignment_triggers_repair_on_open(self):
headers = Headers(':memory:')
headers.io.seek(0)
headers.io.write(HEADERS)
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS)
headers = Headers(headers_temporary_file)
with self.assertLogs(level='WARN') as cm:
await headers.open()
await headers.close()
self.assertEqual(cm.output, [])
headers.io.seek(0)
headers.io.truncate()
headers.io.write(HEADERS[:block_bytes(10)])
headers.io.write(b'ops')
headers.io.write(HEADERS[block_bytes(10):])
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.seek(0)
headers_file.truncate()
headers_file.write(HEADERS[:block_bytes(10)])
headers_file.write(b'ops')
headers_file.write(HEADERS[block_bytes(10):])
await headers.open()
self.assertEqual(
cm.output, [
@@ -201,6 +212,7 @@ class TestHeaders(AsyncioTestCase):
reader_task = asyncio.create_task(reader())
await writer()
await reader_task
await headers.close()
HEADERS = unhexlify(
+2
View File
@@ -48,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
self.ledger.headers.checkpoints = {}
await self.ledger.headers.open()
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
await self.ledger.db.open()