Compare commits

..
Author SHA1 Message Date
Victor Shyba 6c072d2591 lbrycrd bindings (depends boost, claimtrie, etc) 2020-04-11 02:18:53 -03:00
29 changed files with 394 additions and 317 deletions

No files matched your search

+5 -7
View File
@@ -6,7 +6,7 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -17,7 +17,7 @@ jobs:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -37,14 +37,12 @@ jobs:
- blockchain
- other
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
run: sudo apt install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
@@ -59,7 +57,7 @@ jobs:
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
+1 -1
View File
@@ -8,7 +8,7 @@ install:
pip install -e .
tools:
pip install mypy==0.701 pylint==2.4.4
pip install mypy==0.701
pip install coverage astroid pylint
lint:
+133
View File
@@ -0,0 +1,133 @@
# from https://github.com/pybind/python_example/blob/master/setup.py
import sys
import os
import setuptools
import glob
from setuptools import Extension
from setuptools.command.build_ext import build_ext
class get_pybind_include(object):
"""Helper class to determine the pybind11 include path
The purpose of this class is to postpone importing pybind11
until it is actually installed, so that the ``get_include()``
method can be invoked. """
def __init__(self, user=False):
self.user = user
def __str__(self):
import pybind11
return pybind11.get_include(self.user)
#claimtrie_sources = glob.glob('../lbrycrd/src/**/*.cpp')
#claimtrie_sources += glob.glob('../lbrycrd/src/**/*.c')
claimtrie_sources = []
ext_modules = [
Extension(
'lbrycrd',
[
os.path.join("..", "lbrycrd", "src", "claimtrie", "blob.cpp"),
os.path.join("..", "lbrycrd", "src", "claimtrie", "uints.cpp"),
os.path.join("..", "lbrycrd", "src", "claimtrie", "txoutpoint.cpp"),
os.path.join("..", "lbrycrd", "src", "random.cpp"),
os.path.join("..", "lbrycrd", "src", "support", "lockedpool.cpp"),
os.path.join("..", "lbrycrd", "src", "support", "cleanse.cpp"),
os.path.join("..", "lbrycrd", "src", "util", "time.cpp"),
os.path.join("..", "lbrycrd", "src", "util", "threadnames.cpp"),
os.path.join("..", "lbrycrd", "src", "hash.cpp"),
os.path.join("..", "lbrycrd", "src", "fs.cpp"),
os.path.join("..", "lbrycrd", "src", "logging.cpp"),
os.path.join("..", "lbrycrd", "src", "uint256.cpp"),
os.path.join("..", "lbrycrd", "src", "arith_uint256.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "sha512.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "hmac_sha512.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "ripemd160.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "chacha20.cpp"),
os.path.join("..", "lbrycrd", "src", "primitives", "transaction.cpp"),
os.path.join("..", "lbrycrd", "src", "primitives", "block.cpp"),
os.path.join("..", "lbrycrd", "src", "blockfilter.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "sha256.cpp"),
os.path.join("..", "lbrycrd", "src", "crypto", "siphash.cpp"),
os.path.join("..", "lbrycrd", "src", "script", "script.cpp"),
os.path.join("..", "lbrycrd", "src", "util", "strencodings.cpp"),
os.path.join("..", "lbrycrd", "src", "util", "bytevectorhash.cpp"),
os.path.join("lbrycrd_cpp_bindings", "block_filter.cpp"),
os.path.join("lbrycrd_cpp_bindings", "lbrycrd.cpp"),
] + claimtrie_sources,
include_dirs=[
# Path to pybind11 headers
get_pybind_include(),
get_pybind_include(user=True),
os.path.join("..", "lbrycrd", "src"),
os.path.join("..", "lbrycrd", "src", "claimtrie"),
],
libraries=['boost_thread', 'crypto', 'boost_chrono', 'boost_filesystem'],
language='c++'
),
]
# As of Python 3.6, CCompiler has a `has_flag` method.
# cf http://bugs.python.org/issue26689
def has_flag(compiler, flagname):
"""Return a boolean indicating whether a flag name is supported on
the specified compiler.
"""
import tempfile
with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f:
f.write('int main (int argc, char **argv) { return 0; }')
try:
compiler.compile([f.name], extra_postargs=[flagname])
except setuptools.distutils.errors.CompileError:
return False
return True
def cpp_flag(compiler):
"""Return the -std=c++[11/14/17] compiler flag.
The newer version is prefered over c++11 (when it is available).
"""
flags = ['-std=c++17', '-std=c++14', '-std=c++11']
for flag in flags:
if has_flag(compiler, flag): return flag
raise RuntimeError('Unsupported compiler -- at least C++11 support '
'is needed!')
class BuildExt(build_ext):
"""A custom build extension for adding compiler-specific options."""
c_opts = {
'msvc': ['/EHsc'],
'unix': [],
}
l_opts = {
'msvc': [],
'unix': [],
}
if sys.platform == 'darwin':
darwin_opts = ['-stdlib=libc++', '-mmacosx-version-min=10.14']
c_opts['unix'] += darwin_opts
l_opts['unix'] += darwin_opts
def build_extensions(self):
ct = self.compiler.compiler_type
opts = self.c_opts.get(ct, [])
opts.append('-DHAVE_WORKING_BOOST_SLEEP_FOR=1')
link_opts = self.l_opts.get(ct, [])
if ct == 'unix':
opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
opts.append(cpp_flag(self.compiler))
if has_flag(self.compiler, '-fvisibility=hidden'):
opts.append('-fvisibility=hidden')
elif ct == 'msvc':
opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version())
for ext in self.extensions:
ext.extra_compile_args = opts
ext.extra_link_args = link_opts
build_ext.build_extensions(self)
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.71.0"
__version__ = "0.68.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+9 -23
View File
@@ -110,7 +110,7 @@ class AbstractBlob:
if reader in self.readers:
self.readers.remove(reader)
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
def set_length(self, length) -> None:
@@ -198,17 +198,11 @@ class AbstractBlob:
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
def update_events(_):
self.verified.set()
self.writing.clear()
if self.is_writeable():
self.writing.set()
task = self._write_blob(verified_bytes)
task.add_done_callback(update_events)
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
task.add_done_callback(lambda _: self.blob_completed_callback(self))
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
@@ -267,11 +261,9 @@ class BlobBuffer(AbstractBlob):
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
async def write():
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
return self.loop.create_task(write())
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
@@ -327,14 +319,8 @@ class BlobFile(AbstractBlob):
handle.close()
def _write_blob(self, blob_bytes: bytes):
def _write_blob():
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
async def write_blob():
await self.loop.run_in_executor(None, _write_blob)
return self.loop.create_task(write_blob())
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def delete(self):
if os.path.isfile(self.file_path):
-2
View File
@@ -152,8 +152,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
# wait for the io to finish
await self.blob.verified.wait()
log.info("%s at %fMB/s", msg,
round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0, 2))
+3 -4
View File
@@ -691,10 +691,9 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
if down_dir:
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
download_dir = re.sub('\"', '', down_dir)
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
+14 -37
View File
@@ -1917,8 +1917,9 @@ class Daemon(metaclass=JSONRPCServerType):
"""
@requires(STREAM_MANAGER_COMPONENT)
async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None,
page_size=None, **kwargs):
async def jsonrpc_file_list(
self, sort=None, reverse=False, comparison=None,
wallet_id=None, page=None, page_size=None, **kwargs):
"""
List files limited by optional filters
@@ -1939,17 +1940,17 @@ class Daemon(metaclass=JSONRPCServerType):
--stream_hash=<stream_hash> : (str) get file with matching stream hash
--rowid=<rowid> : (int) get file with matching row id
--added_on=<added_on> : (int) get file with matching time of insertion
--claim_id=<claim_id> : (str) get file with matching claim id(s)
--outpoint=<outpoint> : (str) get file with matching claim outpoint(s)
--claim_id=<claim_id> : (str) get file with matching claim id
--outpoint=<outpoint> : (str) get file with matching claim outpoint
--txid=<txid> : (str) get file with matching claim txid
--nout=<nout> : (int) get file with matching claim nout
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id(s)
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
--channel_name=<channel_name> : (str) get file with matching channel name
--claim_name=<claim_name> : (str) get file with matching claim name
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in)
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le)
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--wallet_id=<wallet_id> : (str) add purchase receipts from this wallet
@@ -1959,7 +1960,6 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
sort = sort or 'rowid'
comparison = comparison or 'eq'
paginated = paginate_list(
self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size
)
@@ -2237,7 +2237,7 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
if not kwargs.get('is_spent', False):
if 'is_spent' not in kwargs:
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(**kwargs)
@@ -2971,9 +2971,6 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Transaction}
"""
start = time.perf_counter()
log.info("got publish request for '%s'", name)
self.valid_stream_name_or_error(name)
wallet = self.wallet_manager.get_wallet_or_default(kwargs.get('wallet_id'))
if kwargs.get('account_id'):
@@ -2983,7 +2980,6 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await self.ledger.get_claims(
wallet=wallet, accounts=accounts, claim_name=name
)
log.info("got claim list for '%s' in %fs", name, time.perf_counter()-start)
if len(claims) == 0:
if 'bid' not in kwargs:
raise Exception("'bid' is a required argument for new publishes.")
@@ -3176,19 +3172,14 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Transaction}
"""
start = time.perf_counter()
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
self.valid_stream_name_or_error(name)
account = wallet.get_account_or_default(account_id)
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
log.info("got channel for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
claim_address = await self.get_receiving_address(claim_address, account)
log.info("got claim address for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
kwargs['fee_address'] = self.get_fee_address(kwargs, claim_address)
claims = await account.get_claims(claim_name=name)
@@ -3198,47 +3189,35 @@ class Daemon(metaclass=JSONRPCServerType):
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
)
log.info("checked claims for '%s' in %fs", name, time.perf_counter()-start)
start = time.perf_counter()
file_path, spec = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
log.info("ran ffmpeg for '%s' in %fs", name, time.perf_counter()-start)
kwargs.update(spec)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
start = time.perf_counter()
tx = await Transaction.claim_create(
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
)
new_txo = tx.outputs[0]
log.info("created tx for '%s' in %fs", name, time.perf_counter()-start)
file_stream = None
if not preview:
start = time.perf_counter()
file_stream = await self.stream_manager.create_stream(file_path)
claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
log.info("created stream for '%s' in %fs", name, time.perf_counter() - start)
if channel:
new_txo.sign(channel)
await tx.sign(funding_accounts)
if not preview:
log.info("broadcast and release")
start = time.perf_counter()
await self.broadcast_or_release(tx, blocking)
log.info("broadcast '%s' in %fs", name, time.perf_counter() - start)
start = time.perf_counter()
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
log.info("saved claims '%s' in %fs", name, time.perf_counter() - start)
start = time.perf_counter()
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
log.info("saved content claim '%s' in %fs", name, time.perf_counter() - start)
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish'))
else:
await account.ledger.release_tx(tx)
@@ -5175,11 +5154,10 @@ class Daemon(metaclass=JSONRPCServerType):
--comment_ids=<comment_ids> : (str, list) one or more comment_id to hide.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
Returns:
(dict) keyed by comment_id, containing success info
'<comment_id>': {
"hidden": (bool) flag indicating if comment_id was hidden
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
@@ -5190,7 +5168,6 @@ class Daemon(metaclass=JSONRPCServerType):
comments = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_comments_by_id', comment_ids=comment_ids
)
comments = comments['items']
claim_ids = {comment['claim_id'] for comment in comments}
claims = {cid: await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id=cid) for cid in claim_ids}
pieces = []
+2 -6
View File
@@ -108,9 +108,7 @@ def encode_file_doc():
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
'claim_name': '(str) None if claim is not found else the claim name'
}
@@ -309,9 +307,7 @@ class JSONResponseEncoder(JSONEncoder):
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
'is_fully_reflected': managed_stream.is_fully_reflected
}
def encode_claim(self, claim):
+10 -17
View File
@@ -44,25 +44,18 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
yield os.urandom(AES.block_size // 8)
def read_bytes(file_path: str, offset: int, to_read: int):
with open(file_path, 'rb') as f:
f.seek(offset)
return f.read(to_read)
async def file_reader(file_path: str):
def file_reader(file_path: str):
length = int(os.stat(file_path).st_size)
offset = 0
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = await asyncio.get_event_loop().run_in_executor(
None, read_bytes, file_path, offset, bytes_to_read
)
yield blob_bytes
offset += bytes_to_read
with open(file_path, 'rb') as stream_file:
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = stream_file.read(bytes_to_read)
yield blob_bytes
offset += bytes_to_read
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
@@ -252,7 +245,7 @@ class StreamDescriptor:
iv_generator = iv_generator or random_iv_generator()
key = key or os.urandom(AES.block_size // 8)
blob_num = -1
async for blob_bytes in file_reader(file_path):
for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
+11 -19
View File
@@ -65,7 +65,6 @@ class ManagedStream:
'downloader',
'analytics_manager',
'fully_reflected',
'reflector_progress',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
@@ -74,8 +73,7 @@ class ManagedStream:
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt',
'uploading_to_reflector'
'finished_write_attempt'
]
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
@@ -103,8 +101,6 @@ class ManagedStream:
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
@@ -343,10 +339,9 @@ class ManagedStream:
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(output_path: str, data: bytes):
with open(output_path, 'ab') as handle:
handle.write(data)
handle.flush()
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
@@ -356,12 +351,12 @@ class ManagedStream:
self.finished_writing.clear()
self.started_writing.clear()
try:
open(output_path, 'wb').close()
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
@@ -435,7 +430,6 @@ class ManagedStream:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
@@ -451,10 +445,9 @@ class ManagedStream:
]
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
self.claim_name, self.claim_id)
for i, blob_hash in enumerate(we_have):
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
@@ -462,7 +455,6 @@ class ManagedStream:
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+1 -30
View File
@@ -23,9 +23,6 @@ if typing.TYPE_CHECKING:
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.wallet.transaction import Transaction
from lbry.wallet.manager import WalletManager
from lbry.wallet.wallet import Wallet
log = logging.getLogger(__name__)
@@ -49,12 +46,6 @@ FILTER_FIELDS = [
'blobs_in_stream'
]
SET_FILTER_FIELDS = {
"claim_ids": "claim_id",
"channel_claim_ids": "channel_claim_id",
"outpoints": "outpoint"
}
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
@@ -62,7 +53,6 @@ COMPARISON_OPERATORS = {
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
'in': lambda a, b: a in b
}
@@ -286,34 +276,15 @@ class StreamManager:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in FILTER_FIELDS:
raise ValueError(f"'{search}' is not a valid search operation")
compare_sets = {}
if isinstance(search_by.get('claim_id'), list):
compare_sets['claim_ids'] = search_by.pop('claim_id')
if isinstance(search_by.get('outpoint'), list):
compare_sets['outpoints'] = search_by.pop('outpoint')
if isinstance(search_by.get('channel_claim_id'), list):
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams.values():
matched = False
for set_search, val in compare_sets.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val):
streams.append(stream)
matched = True
break
if matched:
continue
for search, val in search_by.items():
this_stream = getattr(stream, search)
if COMPARISON_OPERATORS[comparison](this_stream, val):
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
streams.append(stream)
break
else:
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+1 -2
View File
@@ -733,9 +733,8 @@ class Database(SQLiteMixin):
include_is_my_input = True
constraints['exclude_internal_payments__or'] = {
'txo.txo_type__not': TXO_TYPES['other'],
'txo.address__not_in': my_addresses,
'txi.address__is_null': True,
'txi.address__not_in': my_addresses,
'txi.address__not_in': my_addresses
}
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
if is_spent:
+47 -44
View File
@@ -5,6 +5,7 @@ import asyncio
import logging
import zlib
from datetime import date
from concurrent.futures.thread import ThreadPoolExecutor
from io import BytesIO
from typing import Optional, Iterator, Tuple, Callable
@@ -41,22 +42,23 @@ class Headers:
validate_difficulty: bool = True
def __init__(self, path) -> None:
self.io = None
if path == ':memory:':
self.io = BytesIO()
self.path = path
self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.executor = ThreadPoolExecutor(1)
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self):
self.io = BytesIO()
if not self.executor:
self.executor = ThreadPoolExecutor(1)
if self.path != ':memory:':
def _readit():
if os.path.exists(self.path):
with open(self.path, 'r+b') as header_file:
self.io.seek(0)
self.io.write(header_file.read())
await asyncio.get_event_loop().run_in_executor(None, _readit)
if not os.path.exists(self.path):
self.io = open(self.path, 'w+b')
else:
self.io = open(self.path, 'r+b')
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
@@ -70,14 +72,10 @@ class Headers:
await self.get_all_missing_headers()
async def close(self):
if self.io is not None:
def _close():
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
with open(self.path, flags) as header_file:
header_file.write(self.io.getbuffer())
await asyncio.get_event_loop().run_in_executor(None, _close)
self.io.close()
self.io = None
if self.executor:
self.executor.shutdown()
self.executor = None
self.io.close()
@staticmethod
def serialize(header):
@@ -137,30 +135,28 @@ class Headers:
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height, try_real_headers=True):
def estimated_timestamp(self, height):
if height <= 0:
return
if try_real_headers and self.has_header(height):
offset = height * self.header_size
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self._read(height)
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
def _read(self, height, count=1):
offset = height * self.header_size
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
self.io.seek(height * self.header_size, os.SEEK_SET)
return self.io.read(self.header_size * count)
def chunk_hash(self, start, count):
return self.hash_header(self._read(start, count)).decode()
self.io.seek(start * self.header_size, os.SEEK_SET)
return self.hash_header(self.io.read(count * self.header_size)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
@@ -169,7 +165,7 @@ class Headers:
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if self.has_header(height):
if await self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
@@ -183,7 +179,7 @@ class Headers:
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
self._write(start, chunk)
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
@@ -193,23 +189,27 @@ class Headers:
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
def has_header(self, height):
async def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
def _has_header(height):
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
def _io_checkall():
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
@property
def height(self) -> int:
@@ -241,7 +241,7 @@ class Headers:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
if chunk:
added += self._write(height, chunk)
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
if bail:
break
return added
@@ -306,7 +306,9 @@ class Headers:
previous_header_hash = fail = None
batch_size = 36
for height in range(start_height, self.height, batch_size):
headers = self._read(height, batch_size)
headers = await asyncio.get_running_loop().run_in_executor(
self.executor, self._read, height, batch_size
)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(height, headers):
@@ -322,11 +324,12 @@ class Headers:
assert start_height > 0 and height == start_height
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return
def __truncate(at_height):
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
previous_header_hash = header_hash
@classmethod
+10 -26
View File
@@ -545,19 +545,11 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
@@ -604,19 +596,11 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
+67
View File
@@ -0,0 +1,67 @@
#include "block_filter.h"
PYBlockFilter::PYBlockFilter(std::vector< std::vector< unsigned char > >& hashes)
{
GCSFilter::ElementSet elements;
for (int i = 0; i < hashes.size(); ++i)
{
GCSFilter::Element element(hashes[i].size());
for(int j=0;j<hashes[i].size();j++)
{
element[j] = hashes[i][j];
}
elements.insert(std::move(element));
}
filter=new GCSFilter({0, 0, 20, 1 << 20},elements);
}
PYBlockFilter::PYBlockFilter(std::vector< unsigned char > & encoded_filter)
{
filter=new GCSFilter({0, 0, 20, 1 << 20}, encoded_filter);
}
PYBlockFilter::PYBlockFilter(std::string & block_hash, std::vector< unsigned char > & encoded_filter)
{
uint256 m_block_hash = uint256S(block_hash);
b_filter = new BlockFilter(BlockFilterType::BASIC, m_block_hash, encoded_filter);
const GCSFilter _filter = b_filter->GetFilter();
filter=new GCSFilter(_filter.GetParams(), _filter.GetEncoded());
}
const std::vector<unsigned char>& PYBlockFilter::GetEncoded()
{
return filter->GetEncoded();
}
PYBlockFilter::~PYBlockFilter()
{
delete filter;
}
bool PYBlockFilter::Match(std::vector< unsigned char >& hash)
{
GCSFilter::Element element(hash.size());
for(int j=0;j<hash.size();j++)
{
element[j] = hash[j];
}
return filter->Match(element);
}
bool PYBlockFilter::MatchAny(std::vector< std::vector< unsigned char > >& hashes)
{
GCSFilter::ElementSet elements;
for (int i = 0; i < hashes.size(); ++i)
{
GCSFilter::Element element(hashes[i].size());
for(int j=0;j<hashes[i].size();j++)
{
element[j] = hashes[i][j];
}
elements.insert(std::move(element));
}
return filter->MatchAny(elements);
}
+24
View File
@@ -0,0 +1,24 @@
#ifndef PY_BLOCK_FILTER_H
#define PY_BLOCK_FILTER_H
#include <blockfilter.h>
class PYBlockFilter
{
public:
GCSFilter *filter;
BlockFilter *b_filter;
public:
PYBlockFilter(std::vector< std::vector< unsigned char > >& hashes);
PYBlockFilter(std::vector< unsigned char > & encoded_filter);
PYBlockFilter(std::string & block_hash, std::vector< unsigned char > & encoded_filter);
const std::vector<unsigned char>& GetEncoded();
~PYBlockFilter();
bool Match(std::vector< unsigned char >& hash);
bool MatchAny(std::vector< std::vector< unsigned char > >& hashes);
};
#endif
+18
View File
@@ -0,0 +1,18 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/operators.h>
#include "block_filter.h"
namespace py = pybind11;
PYBIND11_MODULE(lbrycrd, mod) {
py::class_<PYBlockFilter, std::shared_ptr<PYBlockFilter>> clsPYBlockFilter(mod, "PYBlockFilter");
clsPYBlockFilter.def(py::init<std::vector< std::vector< unsigned char > >&>());
clsPYBlockFilter.def(py::init< std::vector< unsigned char > &>());
clsPYBlockFilter.def(py::init< std::string &, std::vector< unsigned char > &>());
clsPYBlockFilter.def("GetEncoded",(const std::vector< unsigned char >& (PYBlockFilter::*)()) &PYBlockFilter::GetEncoded);
clsPYBlockFilter.def("Match", (bool (PYBlockFilter::*)(std::vector< unsigned char >&)) &PYBlockFilter::Match);
clsPYBlockFilter.def("MatchAny", (bool (PYBlockFilter::*)(std::vector< std::vector< unsigned char > >&)) &PYBlockFilter::MatchAny);
}
+4
View File
@@ -1,5 +1,6 @@
import os
import sys
from extensions_setup import ext_modules, BuildExt
from lbry import __name__, __version__
from setuptools import setup, find_packages
@@ -32,6 +33,9 @@ setup(
'orchstr8=lbry.wallet.orchstr8.cli:main',
],
},
setup_requires=["pybind11"],
ext_modules=ext_modules,
cmdclass={'build_ext': BuildExt},
install_requires=[
'aiohttp==3.5.4',
'aioupnp==0.0.17',
@@ -503,35 +503,22 @@ class TransactionOutputCommands(ClaimTestCase):
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create('@kept-channel')
await self.channel_create('@sent-channel', claim_address=address2)
await self.wallet_send('2.9', address2)
# all txos on second wallet
received_payment, received_channel = await self.txo_list(
wallet_id=wallet2.id, is_my_input_or_output=True)
received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True)
self.assertEqual('1.0', received_channel['amount'])
self.assertFalse(received_channel['is_my_input'])
self.assertTrue(received_channel['is_my_output'])
self.assertFalse(received_channel['is_internal_transfer'])
self.assertEqual('2.9', received_payment['amount'])
self.assertFalse(received_payment['is_my_input'])
self.assertTrue(received_payment['is_my_output'])
self.assertFalse(received_payment['is_internal_transfer'])
# all txos on default wallet
r = await self.txo_list(is_my_input_or_output=True)
self.assertEqual(
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
['1.0', '7.947786', '1.0', '8.973893', '10.0'],
[t['amount'] for t in r]
)
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_payment['is_my_input'])
self.assertFalse(sent_payment['is_my_output'])
self.assertFalse(sent_payment['is_internal_transfer'])
self.assertTrue(change3['is_my_input'])
self.assertTrue(change3['is_my_output'])
self.assertTrue(change3['is_internal_transfer'])
sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_channel['is_my_input'])
self.assertFalse(sent_channel['is_my_output'])
@@ -553,31 +540,31 @@ class TransactionOutputCommands(ClaimTestCase):
# my stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
self.assertEqual([sent_channel, kept_channel, initial_funds], r)
# my unspent stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
self.assertEqual([sent_channel, kept_channel], r)
# only "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
self.assertEqual([change3, change2, change1], r)
self.assertEqual([change2, change1], r)
# only unspent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
self.assertEqual([change3], r)
self.assertEqual([change2], r)
# only spent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
self.assertEqual([change2, change1], r)
self.assertEqual([change1], r)
# all my unspent stuff
r = await self.txo_list(is_my_output=True, is_not_spent=True)
self.assertEqual([change3, kept_channel], r)
self.assertEqual([change2, kept_channel], r)
# stuff i sent
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_payment, sent_channel], r)
self.assertEqual([sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
@@ -661,15 +648,12 @@ class ClaimCommands(ClaimTestCase):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, title='foo')
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
# catch a bug where cli sends is_spent=False by default
r = await self.claim_list(claim_type='stream', is_spent=False)
r = await self.claim_list(claim_type='stream')
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
@@ -21,11 +21,6 @@ class FileCommands(CommandTestCase):
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
@@ -106,16 +106,11 @@ class MockedCommentServer:
return False
def hide_comments(self, pieces: list):
hidden = []
comments_hidden = []
for p in pieces:
if self.hide_comment(**p):
hidden.append(p['comment_id'])
comment_ids = {c['comment_id'] for c in pieces}
return {
'hidden': hidden,
'visible': list(comment_ids - set(hidden))
}
comments_hidden.append(p['comment_id'])
return {'hidden': comments_hidden}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
@@ -143,19 +138,12 @@ class MockedCommentServer:
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[self.get_comment_id(comment_id)]
return {
'channel_id': comment['channel_id'],
'channel_name': comment['channel_name'],
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
}
def get_comments_by_id(self, comment_ids: list):
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
return {
'page': 1,
'page_size': len(comment_ids),
'total_pages': 1,
'items': comments,
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
methods = {
'get_claim_comments': get_claim_comments,
+1 -2
View File
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
@@ -24,7 +24,6 @@ class TestBlobManager(AsyncioTestCase):
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
@@ -130,14 +130,10 @@ class TestBlobExchange(BlobExchangeTestBase):
write_blob = blob._write_blob
write_called_count = 0
async def _wrap_write_blob(blob_bytes):
def wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
await write_blob(blob_bytes)
def wrap_write_blob(blob_bytes):
return asyncio.create_task(_wrap_write_blob(blob_bytes))
write_blob(blob_bytes)
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
@@ -170,7 +166,6 @@ class TestBlobExchange(BlobExchangeTestBase):
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
-2
View File
@@ -46,9 +46,7 @@ class TestStreamAssembler(AsyncioTestCase):
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(100, self.stream.reflector_progress)
self.assertSetEqual(
set(sent),
set(map(lambda b: b.blob_hash,
-1
View File
@@ -211,7 +211,6 @@ class TestQueries(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
await self.ledger.headers.open()
self.wallet = Wallet()
await self.ledger.db.open()
+12 -33
View File
@@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase):
async def test_deserialize(self):
self.maxDiff = None
h = Headers(':memory:')
h.io.write(HEADERS)
await h.open()
await h.connect(0, HEADERS)
self.assertEqual(await h.get(0), {
'bits': 520159231,
'block_height': 0,
@@ -52,11 +52,8 @@ class TestHeaders(AsyncioTestCase):
self.assertEqual(headers.height, 19)
async def test_connect_from_middle(self):
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS[:block_bytes(10)])
h = Headers(headers_temporary_file)
h = Headers(':memory:')
h.io.write(HEADERS[:block_bytes(10)])
await h.open()
self.assertEqual(h.height, 9)
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
@@ -118,7 +115,6 @@ class TestHeaders(AsyncioTestCase):
async def test_bounds(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS)
self.assertEqual(19, headers.height)
with self.assertRaises(IndexError):
@@ -130,7 +126,6 @@ class TestHeaders(AsyncioTestCase):
async def test_repair(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS[:block_bytes(11)])
self.assertEqual(10, headers.height)
await headers.repair()
@@ -152,39 +147,24 @@ class TestHeaders(AsyncioTestCase):
await headers.repair(start_height=10)
self.assertEqual(19, headers.height)
async def test_do_not_estimate_unconfirmed(self):
def test_do_not_estimate_unconfirmed(self):
headers = Headers(':memory:')
await headers.open()
self.assertIsNone(headers.estimated_timestamp(-1))
self.assertIsNone(headers.estimated_timestamp(0))
self.assertIsNotNone(headers.estimated_timestamp(1))
async def test_dont_estimate_whats_there(self):
headers = Headers(':memory:')
await headers.open()
estimated = headers.estimated_timestamp(10)
await headers.connect(0, HEADERS)
real_time = (await headers.get(10))['timestamp']
after_downloading_header_estimated = headers.estimated_timestamp(10)
self.assertNotEqual(estimated, after_downloading_header_estimated)
self.assertEqual(after_downloading_header_estimated, real_time)
async def test_misalignment_triggers_repair_on_open(self):
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS)
headers = Headers(headers_temporary_file)
headers = Headers(':memory:')
headers.io.seek(0)
headers.io.write(HEADERS)
with self.assertLogs(level='WARN') as cm:
await headers.open()
await headers.close()
self.assertEqual(cm.output, [])
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.seek(0)
headers_file.truncate()
headers_file.write(HEADERS[:block_bytes(10)])
headers_file.write(b'ops')
headers_file.write(HEADERS[block_bytes(10):])
headers.io.seek(0)
headers.io.truncate()
headers.io.write(HEADERS[:block_bytes(10)])
headers.io.write(b'ops')
headers.io.write(HEADERS[block_bytes(10):])
await headers.open()
self.assertEqual(
cm.output, [
@@ -212,7 +192,6 @@ class TestHeaders(AsyncioTestCase):
reader_task = asyncio.create_task(reader())
await writer()
await reader_task
await headers.close()
HEADERS = unhexlify(
-2
View File
@@ -48,8 +48,6 @@ class LedgerTestCase(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
self.ledger.headers.checkpoints = {}
await self.ledger.headers.open()
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
await self.ledger.db.open()