Compare commits
65
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa | ||
|
|
149d343201 | ||
|
|
496cc79ba8 | ||
|
|
3591768745 | ||
|
|
d615f6761a | ||
|
|
5b29894048 | ||
|
|
d17f0cfa40 | ||
|
|
b2f70c7120 | ||
|
|
238707bd93 | ||
|
|
87cdf1e3a0 | ||
|
|
57fd47022e | ||
|
|
5eafd3bf6b | ||
|
|
640b5b0ea9 | ||
|
|
e4fb2f4680 | ||
|
|
a4909f54e4 | ||
|
|
f7065c6f0c | ||
|
|
64f7f837e7 | ||
|
|
130d36acd9 | ||
|
|
e94c28cfa2 | ||
|
|
a697dec30c | ||
|
|
bac09e9b9f | ||
|
|
b179f8f86a | ||
|
|
962d04ae17 | ||
|
|
f28e3bfe37 | ||
|
|
6474c86d32 | ||
|
|
769ea8cdfe |
No files matched your search
@@ -6,7 +6,7 @@ jobs:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -37,12 +37,14 @@ jobs:
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: sudo apt install -y --no-install-recommends ffmpeg
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
@@ -57,7 +59,7 @@ jobs:
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
|
||||
@@ -8,7 +8,7 @@ install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.67.2"
|
||||
__version__ = "0.71.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+23
-9
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -198,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -319,8 +327,14 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
+4
-5
@@ -602,8 +602,6 @@ class Config(CLIConfig):
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
@@ -693,9 +691,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
@@ -1917,9 +1917,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_list(
|
||||
self, sort=None, reverse=False, comparison=None,
|
||||
wallet_id=None, page=None, page_size=None, **kwargs):
|
||||
async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None,
|
||||
page_size=None, **kwargs):
|
||||
"""
|
||||
List files limited by optional filters
|
||||
|
||||
@@ -1940,17 +1939,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--stream_hash=<stream_hash> : (str) get file with matching stream hash
|
||||
--rowid=<rowid> : (int) get file with matching row id
|
||||
--added_on=<added_on> : (int) get file with matching time of insertion
|
||||
--claim_id=<claim_id> : (str) get file with matching claim id
|
||||
--outpoint=<outpoint> : (str) get file with matching claim outpoint
|
||||
--claim_id=<claim_id> : (str) get file with matching claim id(s)
|
||||
--outpoint=<outpoint> : (str) get file with matching claim outpoint(s)
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id(s)
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le)
|
||||
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in)
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
--wallet_id=<wallet_id> : (str) add purchase receipts from this wallet
|
||||
@@ -1960,6 +1959,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
sort = sort or 'rowid'
|
||||
comparison = comparison or 'eq'
|
||||
|
||||
paginated = paginate_list(
|
||||
self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size
|
||||
)
|
||||
@@ -2213,7 +2213,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
List my stream and channel claims.
|
||||
|
||||
Usage:
|
||||
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...]
|
||||
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...] [--is_spent]
|
||||
[--channel_id=<channel_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
[--resolve] [--order_by=<order_by>] [--no_totals] [--include_received_tips]
|
||||
@@ -2223,6 +2223,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) streams in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--is_spent : (bool) shows previous claim updates and abandons
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -2236,7 +2237,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
|
||||
kwargs['unspent'] = True
|
||||
if not kwargs.get('is_spent', False):
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(**kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -2750,12 +2752,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
channel_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...] [--is_spent]
|
||||
[--page=<page>] [--page_size=<page_size>] [--resolve] [--no_totals]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) channel name
|
||||
--claim_id=<claim_id> : (str or list) channel id
|
||||
--is_spent : (bool) shows previous channel updates and abandons
|
||||
--account_id=<account_id> : (str) id of the account to use
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -2767,7 +2770,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = 'channel'
|
||||
kwargs['unspent'] = True
|
||||
if 'is_spent' not in kwargs:
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -3504,12 +3508,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
stream_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...] [--is_spent]
|
||||
[--page=<page>] [--page_size=<page_size>] [--resolve] [--no_totals]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) stream name
|
||||
--claim_id=<claim_id> : (str or list) stream id
|
||||
--is_spent : (bool) shows previous stream updates and abandons
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -3521,7 +3526,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = 'stream'
|
||||
kwargs['unspent'] = True
|
||||
if 'is_spent' not in kwargs:
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
|
||||
@@ -3968,19 +3974,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_support_list(self, *args, tips=None, **kwargs):
|
||||
def jsonrpc_support_list(self, *args, received=False, sent=False, staked=False, **kwargs):
|
||||
"""
|
||||
List supports and tips in my control.
|
||||
List staked supports and sent/received tips.
|
||||
|
||||
Usage:
|
||||
support_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...] [--tips]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...]
|
||||
[--received | --sent | --staked] [--is_spent]
|
||||
[--page=<page>] [--page_size=<page_size>] [--no_totals]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) claim name
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--tips : (bool) only show tips
|
||||
--received : (bool) only show received (tips)
|
||||
--sent : (bool) only show sent (tips)
|
||||
--staked : (bool) only show my staked supports
|
||||
--is_spent : (bool) show abandoned supports
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -3991,9 +4001,20 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = 'support'
|
||||
kwargs['unspent'] = True
|
||||
if tips is True:
|
||||
if 'is_spent' not in kwargs:
|
||||
kwargs['is_not_spent'] = True
|
||||
if received:
|
||||
kwargs['is_not_my_input'] = True
|
||||
kwargs['is_my_output'] = True
|
||||
elif sent:
|
||||
kwargs['is_my_input'] = True
|
||||
kwargs['is_not_my_output'] = True
|
||||
# spent for not my outputs is undetermined
|
||||
kwargs.pop('is_spent', None)
|
||||
kwargs.pop('is_not_spent', None)
|
||||
elif staked:
|
||||
kwargs['is_my_input'] = True
|
||||
kwargs['is_my_output'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -4168,11 +4189,15 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@staticmethod
|
||||
def _constrain_txo_from_kwargs(
|
||||
constraints, type=None, txid=None, # pylint: disable=redefined-builtin
|
||||
claim_id=None, channel_id=None, name=None, unspent=False, reposted_claim_id=None,
|
||||
claim_id=None, channel_id=None, name=None, reposted_claim_id=None,
|
||||
is_spent=False, is_not_spent=False,
|
||||
is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
is_my_output=None, is_not_my_output=None,
|
||||
is_my_input=None, is_not_my_input=None):
|
||||
constraints['unspent'] = unspent
|
||||
if is_spent:
|
||||
constraints['is_spent'] = True
|
||||
elif is_not_spent:
|
||||
constraints['is_spent'] = False
|
||||
constraints['exclude_internal_transfers'] = exclude_internal_transfers
|
||||
if is_my_input_or_output is True:
|
||||
constraints['is_my_input_or_output'] = True
|
||||
@@ -4201,8 +4226,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
List my transaction outputs.
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--unspent]
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
|
||||
[--is_spent | --is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4217,7 +4243,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--unspent : (bool) hide spent outputs, show only unspent ones
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
if using this flag the other related flags
|
||||
are ignored (--is_my_output, --is_my_input, etc)
|
||||
@@ -4306,7 +4333,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
accounts = [wallet.get_account_or_error(account_id)] if account_id else wallet.accounts
|
||||
txos = await self.ledger.get_txos(
|
||||
wallet=wallet, accounts=accounts, read_only=True,
|
||||
**self._constrain_txo_from_kwargs({}, unspent=True, is_my_output=True, **kwargs)
|
||||
**self._constrain_txo_from_kwargs({}, is_not_spent=True, is_my_output=True, **kwargs)
|
||||
)
|
||||
txs = []
|
||||
while txos:
|
||||
@@ -4330,7 +4357,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...]
|
||||
[--is_spent] [--is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4342,7 +4370,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--unspent : (bool) hide spent outputs, show only unspent ones
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
if using this flag the other related flags
|
||||
are ignored (--is_my_output, --is_my_input, etc)
|
||||
@@ -4374,7 +4403,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4389,7 +4418,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--unspent : (bool) hide spent outputs, show only unspent ones
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
if using this flag the other related flags
|
||||
are ignored (--is_my_output, --is_my_input, etc)
|
||||
@@ -4446,7 +4476,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
kwargs['type'] = ['other', 'purchase']
|
||||
kwargs['unspent'] = True
|
||||
kwargs['is_not_spent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -5124,10 +5154,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--comment_ids=<comment_ids> : (str, list) one or more comment_id to hide.
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
|
||||
Returns:
|
||||
(dict) keyed by comment_id, containing success info
|
||||
'<comment_id>': {
|
||||
"hidden": (bool) flag indicating if comment_id was hidden
|
||||
Returns: lists containing the ids comments that are hidden and visible.
|
||||
|
||||
{
|
||||
"hidden": (list) IDs of hidden comments.
|
||||
"visible": (list) IDs of visible comments.
|
||||
}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
@@ -5138,6 +5169,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
comments = await comment_client.jsonrpc_post(
|
||||
self.conf.comment_server, 'get_comments_by_id', comment_ids=comment_ids
|
||||
)
|
||||
comments = comments['items']
|
||||
claim_ids = {comment['claim_id'] for comment in comments}
|
||||
claims = {cid: await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id=cid) for cid in claim_ids}
|
||||
pieces = []
|
||||
|
||||
@@ -108,7 +108,9 @@ def encode_file_doc():
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
}
|
||||
|
||||
|
||||
@@ -307,7 +309,9 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
|
||||
+17
-10
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def file_reader(file_path: str):
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -245,7 +252,7 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
for blob_bytes in file_reader(file_path):
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
|
||||
@@ -65,6 +65,7 @@ class ManagedStream:
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'reflector_progress',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
@@ -73,7 +74,8 @@ class ManagedStream:
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt'
|
||||
'finished_write_attempt',
|
||||
'uploading_to_reflector'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
@@ -101,6 +103,8 @@ class ManagedStream:
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.reflector_progress = 0
|
||||
self.uploading_to_reflector = False
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
@@ -339,9 +343,10 @@ class ManagedStream:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -351,12 +356,12 @@ class ManagedStream:
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
open(output_path, 'wb').close()
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
@@ -430,6 +435,7 @@ class ManagedStream:
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
self.uploading_to_reflector = True
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
@@ -445,9 +451,10 @@ class ManagedStream:
|
||||
]
|
||||
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
||||
self.claim_name, self.claim_id)
|
||||
for blob_hash in we_have:
|
||||
for i, blob_hash in enumerate(we_have):
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
@@ -455,6 +462,7 @@ class ManagedStream:
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
self.uploading_to_reflector = False
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
|
||||
@@ -23,6 +23,9 @@ if typing.TYPE_CHECKING:
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.wallet.manager import WalletManager
|
||||
from lbry.wallet.wallet import Wallet
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,6 +49,12 @@ FILTER_FIELDS = [
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
SET_FILTER_FIELDS = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
@@ -53,6 +62,7 @@ COMPARISON_OPERATORS = {
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
'in': lambda a, b: a in b
|
||||
}
|
||||
|
||||
|
||||
@@ -276,15 +286,34 @@ class StreamManager:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
matched = False
|
||||
for set_search, val in compare_sets.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val):
|
||||
streams.append(stream)
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
this_stream = getattr(stream, search)
|
||||
if COMPARISON_OPERATORS[comparison](this_stream, val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -734,4 +734,10 @@ HASHES = {
|
||||
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
|
||||
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
|
||||
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
|
||||
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
|
||||
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
|
||||
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
|
||||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
}
|
||||
+51
-40
@@ -73,6 +73,7 @@ class AIOSQLite:
|
||||
self.write_lock = asyncio.Lock()
|
||||
self.writers = 0
|
||||
self.read_ready = asyncio.Event()
|
||||
self.urgent_read_done = asyncio.Event()
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
@@ -88,6 +89,7 @@ class AIOSQLite:
|
||||
)
|
||||
await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer)
|
||||
db.read_ready.set()
|
||||
db.urgent_read_done.set()
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
@@ -112,12 +114,25 @@ class AIOSQLite:
|
||||
read_only=False, fetch_all: bool = False) -> List[dict]:
|
||||
read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone
|
||||
parameters = parameters if parameters is not None else []
|
||||
still_waiting = False
|
||||
urgent_read = False
|
||||
if read_only:
|
||||
while self.writers:
|
||||
await self.read_ready.wait()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
try:
|
||||
while self.writers: # more writes can come in while we are waiting for the first
|
||||
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
|
||||
# throttle the writes if they pile up
|
||||
self.urgent_read_done.clear()
|
||||
urgent_read = True
|
||||
# wait until the running writes have finished
|
||||
await self.read_ready.wait()
|
||||
still_waiting = True
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
finally:
|
||||
if urgent_read:
|
||||
# unthrottle the writers if they had to be throttled
|
||||
self.urgent_read_done.set()
|
||||
if fetch_all:
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchall())
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchone())
|
||||
@@ -135,17 +150,18 @@ class AIOSQLite:
|
||||
return self.run(lambda conn: conn.execute(sql, parameters))
|
||||
|
||||
async def run(self, fun, *args, **kwargs):
|
||||
await self.urgent_read_done.wait()
|
||||
self.writers += 1
|
||||
self.read_ready.clear()
|
||||
async with self.write_lock:
|
||||
try:
|
||||
try:
|
||||
async with self.write_lock:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
|
||||
)
|
||||
finally:
|
||||
self.writers -= 1
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
finally:
|
||||
self.writers -= 1
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
|
||||
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
|
||||
self.writer_connection.execute('begin')
|
||||
@@ -694,7 +710,7 @@ class Database(SQLiteMixin):
|
||||
self, cols, accounts=None, is_my_input=None, is_my_output=True,
|
||||
is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
include_is_spent=False, include_is_my_input=False,
|
||||
read_only=False, **constraints):
|
||||
is_spent=None, read_only=False, **constraints):
|
||||
for rename_col in ('txid', 'txoid'):
|
||||
for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'):
|
||||
if rename_constraint in constraints:
|
||||
@@ -733,27 +749,23 @@ class Database(SQLiteMixin):
|
||||
include_is_my_input = True
|
||||
constraints['exclude_internal_payments__or'] = {
|
||||
'txo.txo_type__not': TXO_TYPES['other'],
|
||||
'txo.address__not_in': my_addresses,
|
||||
'txi.address__is_null': True,
|
||||
'txi.address__not_in': my_addresses
|
||||
'txi.address__not_in': my_addresses,
|
||||
}
|
||||
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
|
||||
if include_is_spent:
|
||||
if is_spent:
|
||||
constraints['spent.txoid__is_not_null'] = True
|
||||
elif is_spent is False:
|
||||
constraints['is_reserved'] = False
|
||||
constraints['spent.txoid__is_null'] = True
|
||||
if include_is_spent or is_spent is not None:
|
||||
sql.append("LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid)")
|
||||
if include_is_my_input:
|
||||
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
|
||||
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)
|
||||
|
||||
@staticmethod
|
||||
def constrain_unspent(constraints):
|
||||
constraints['is_reserved'] = False
|
||||
constraints['include_is_spent'] = True
|
||||
constraints['spent.txoid__is_null'] = True
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, unspent=False, read_only=False, **constraints):
|
||||
|
||||
if unspent:
|
||||
self.constrain_unspent(constraints)
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraints):
|
||||
include_is_spent = constraints.get('include_is_spent', False)
|
||||
include_is_my_input = constraints.get('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
@@ -869,7 +881,8 @@ class Database(SQLiteMixin):
|
||||
|
||||
return txos
|
||||
|
||||
def _clean_txo_constraints_for_aggregation(self, unspent, constraints):
|
||||
@staticmethod
|
||||
def _clean_txo_constraints_for_aggregation(constraints):
|
||||
constraints.pop('include_is_spent', None)
|
||||
constraints.pop('include_is_my_input', None)
|
||||
constraints.pop('include_is_my_output', None)
|
||||
@@ -879,22 +892,19 @@ class Database(SQLiteMixin):
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
if unspent:
|
||||
self.constrain_unspent(constraints)
|
||||
|
||||
async def get_txo_count(self, unspent=False, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(unspent, constraints)
|
||||
async def get_txo_count(self, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(constraints)
|
||||
count = await self.select_txos('COUNT(*) AS total', **constraints)
|
||||
return count[0]['total'] or 0
|
||||
|
||||
async def get_txo_sum(self, unspent=False, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(unspent, constraints)
|
||||
async def get_txo_sum(self, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(constraints)
|
||||
result = await self.select_txos('SUM(amount) AS total', **constraints)
|
||||
return result[0]['total'] or 0
|
||||
|
||||
async def get_txo_plot(
|
||||
self, unspent=False, start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(unspent, constraints)
|
||||
async def get_txo_plot(self, start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
|
||||
self._clean_txo_constraints_for_aggregation(constraints)
|
||||
if start_day is None:
|
||||
constraints['day__gte'] = self.ledger.headers.estimated_julian_day(
|
||||
self.ledger.headers.height
|
||||
@@ -915,17 +925,18 @@ class Database(SQLiteMixin):
|
||||
)
|
||||
|
||||
def get_utxos(self, read_only=False, **constraints):
|
||||
return self.get_txos(unspent=True, read_only=read_only, **constraints)
|
||||
return self.get_txos(is_spent=False, read_only=read_only, **constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
return self.get_txo_count(unspent=True, **constraints)
|
||||
return self.get_txo_count(is_spent=False, **constraints)
|
||||
|
||||
async def get_balance(self, wallet=None, accounts=None, read_only=False, **constraints):
|
||||
assert wallet or accounts, \
|
||||
"'wallet' or 'accounts' constraints required to calculate balance"
|
||||
constraints['accounts'] = accounts or wallet.accounts
|
||||
self.constrain_unspent(constraints)
|
||||
balance = await self.select_txos('SUM(amount) as total', read_only=read_only, **constraints)
|
||||
balance = await self.select_txos(
|
||||
'SUM(amount) as total', is_spent=False, read_only=read_only, **constraints
|
||||
)
|
||||
return balance[0]['total'] or 0
|
||||
|
||||
async def select_addresses(self, cols, read_only=False, **constraints):
|
||||
@@ -1084,7 +1095,7 @@ class Database(SQLiteMixin):
|
||||
def get_supports_summary(self, read_only=False, **constraints):
|
||||
return self.get_txos(
|
||||
txo_type=TXO_TYPES['support'],
|
||||
unspent=True, is_my_output=True,
|
||||
is_spent=False, is_my_output=True,
|
||||
include_is_my_input=True,
|
||||
no_tx=True, read_only=read_only,
|
||||
**constraints
|
||||
|
||||
+46
-47
@@ -5,7 +5,6 @@ import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
@@ -42,23 +41,22 @@ class Headers:
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.io = None
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.io = BytesIO()
|
||||
if self.path != ':memory:':
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
def _readit():
|
||||
if os.path.exists(self.path):
|
||||
with open(self.path, 'r+b') as header_file:
|
||||
self.io.seek(0)
|
||||
self.io.write(header_file.read())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
@@ -72,10 +70,14 @@ class Headers:
|
||||
await self.get_all_missing_headers()
|
||||
|
||||
async def close(self):
|
||||
if self.executor:
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
self.io.close()
|
||||
if self.io is not None:
|
||||
def _close():
|
||||
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||
with open(self.path, flags) as header_file:
|
||||
header_file.write(self.io.getbuffer())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||
self.io.close()
|
||||
self.io = None
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
@@ -135,26 +137,30 @@ class Headers:
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height):
|
||||
def estimated_timestamp(self, height, try_real_headers=True):
|
||||
if height <= 0:
|
||||
return
|
||||
if try_real_headers and self.has_header(height):
|
||||
offset = height * self.header_size
|
||||
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
|
||||
return self._read(height)
|
||||
|
||||
def _read(self, height, count=1):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size * count)
|
||||
offset = height * self.header_size
|
||||
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
self.io.seek(start * self.header_size, os.SEEK_SET)
|
||||
return self.hash_header(self.io.read(count * self.header_size)).decode()
|
||||
return self.hash_header(self._read(start, count)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
@@ -163,7 +169,7 @@ class Headers:
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if await self.has_header(height):
|
||||
if self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
@@ -177,7 +183,7 @@ class Headers:
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
|
||||
self._write(start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
@@ -187,27 +193,23 @@ class Headers:
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
async def has_header(self, height):
|
||||
def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
def _has_header(height):
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
def _io_checkall():
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
@@ -239,7 +241,7 @@ class Headers:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
|
||||
added += self._write(height, chunk)
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
@@ -304,9 +306,7 @@ class Headers:
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self._read, height, batch_size
|
||||
)
|
||||
headers = self._read(height, batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
@@ -322,12 +322,11 @@ class Headers:
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
def __truncate(at_height):
|
||||
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@classmethod
|
||||
|
||||
+13
-3
@@ -24,6 +24,7 @@ from .account import Account, AddressManager, SingleKey
|
||||
from .network import Network
|
||||
from .transaction import Transaction, Output
|
||||
from .header import Headers, UnvalidatedHeaders
|
||||
from .checkpoints import HASHES
|
||||
from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
|
||||
from .bip32 import PubKey, PrivateKey
|
||||
from .coinselection import CoinSelector
|
||||
@@ -108,6 +109,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
default_fee_per_byte = 50
|
||||
default_fee_per_name_char = 200000
|
||||
|
||||
checkpoints = HASHES
|
||||
|
||||
def __init__(self, config=None):
|
||||
self.config = config or {}
|
||||
self.db: Database = self.config.get('db') or Database(
|
||||
@@ -117,6 +120,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self.headers: Headers = self.config.get('headers') or self.headers_class(
|
||||
os.path.join(self.path, "headers")
|
||||
)
|
||||
self.headers.checkpoints = self.checkpoints
|
||||
self.network: Network = self.config.get('network') or Network(self)
|
||||
self.network.on_header.listen(self.receive_header)
|
||||
self.network.on_status.listen(self.process_status_update)
|
||||
@@ -714,7 +718,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if include_is_my_output:
|
||||
mine = await self.db.get_txo_count(
|
||||
claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True,
|
||||
unspent=True, accounts=accounts
|
||||
is_spent=False, accounts=accounts
|
||||
)
|
||||
if mine:
|
||||
txo_copy.is_my_output = True
|
||||
@@ -724,7 +728,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
supports = await self.db.get_txo_sum(
|
||||
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||
is_my_input=True, is_my_output=True,
|
||||
unspent=True, accounts=accounts
|
||||
is_spent=False, accounts=accounts
|
||||
)
|
||||
txo_copy.sent_supports = supports
|
||||
if include_sent_tips:
|
||||
@@ -748,7 +752,11 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
|
||||
async def resolve(self, accounts, urls, **kwargs):
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
txos = (await self._inflate_outputs(resolve(urls), accounts, **kwargs))[0]
|
||||
urls_copy = list(urls)
|
||||
txos = []
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:500], urls_copy[500:]
|
||||
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
|
||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||
result = {}
|
||||
for url, txo in zip(urls, txos):
|
||||
@@ -1056,6 +1064,7 @@ class TestNetLedger(Ledger):
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
class RegTestLedger(Ledger):
|
||||
@@ -1070,3 +1079,4 @@ class RegTestLedger(Ledger):
|
||||
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
genesis_bits = 0x207fffff
|
||||
target_timespan = 1
|
||||
checkpoints = {}
|
||||
@@ -87,7 +87,7 @@ class ClientSession(BaseClientSession):
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
@@ -55,7 +55,8 @@ class Conductor:
|
||||
|
||||
async def start_blockchain(self):
|
||||
if not self.blockchain_started:
|
||||
await self.blockchain_node.start()
|
||||
asyncio.create_task(self.blockchain_node.start())
|
||||
await self.blockchain_node.running.wait()
|
||||
await self.blockchain_node.generate(200)
|
||||
self.blockchain_started = True
|
||||
|
||||
@@ -255,6 +256,10 @@ class BlockchainNode:
|
||||
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
|
||||
self.rpcuser = 'rpcuser'
|
||||
self.rpcpassword = 'rpcpassword'
|
||||
self.stopped = False
|
||||
self.restart_ready = asyncio.Event()
|
||||
self.restart_ready.set()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
@@ -315,13 +320,27 @@ class BlockchainNode:
|
||||
f'-port={self.peerport}'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
while not self.stopped:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
await self.restart_ready.wait()
|
||||
try:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
self.running.set()
|
||||
except asyncio.CancelledError:
|
||||
self.running.clear()
|
||||
raise
|
||||
except Exception as e:
|
||||
self.running.clear()
|
||||
log.exception('failed to start lbrycrdd', exc_info=e)
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
self.stopped = True
|
||||
try:
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
@@ -330,6 +349,16 @@ class BlockchainNode:
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
|
||||
async def clear_mempool(self):
|
||||
self.restart_ready.clear()
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
self.running.clear()
|
||||
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
|
||||
self.restart_ready.set()
|
||||
await self.running.wait()
|
||||
|
||||
def cleanup(self):
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
@@ -361,6 +390,12 @@ class BlockchainNode:
|
||||
def get_block_hash(self, block):
|
||||
return self._cli_cmnd('getblockhash', str(block))
|
||||
|
||||
def sendrawtransaction(self, tx):
|
||||
return self._cli_cmnd('sendrawtransaction', tx)
|
||||
|
||||
async def get_block(self, block_hash):
|
||||
return json.loads(await self._cli_cmnd('getblock', block_hash, '1'))
|
||||
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
|
||||
|
||||
@@ -39,8 +39,7 @@ from lbry.wallet.tasks import TaskGroup
|
||||
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
||||
from .jsonrpc import RPCError, ProtocolError
|
||||
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
||||
from .util import Concurrency
|
||||
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT
|
||||
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT, RESET_CONNECTIONS
|
||||
|
||||
|
||||
class Connector:
|
||||
@@ -389,6 +388,7 @@ class RPCSession(SessionBase):
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self._close()
|
||||
return
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import time
|
||||
import asyncio
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Optional
|
||||
import lbry
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
@@ -10,7 +10,7 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES, REORG_COUNT
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -219,7 +219,7 @@ class BlockProcessor:
|
||||
'resetting the prefetcher')
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
|
||||
async def reorg_chain(self, count=None):
|
||||
async def reorg_chain(self, count: Optional[int] = None):
|
||||
"""Handle a chain reorganisation.
|
||||
|
||||
Count is the number of blocks to simulate a reorg, or None for
|
||||
@@ -253,7 +253,9 @@ class BlockProcessor:
|
||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||
await self.run_in_thread_with_lock(flush_backup)
|
||||
last -= len(raw_blocks)
|
||||
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
REORG_COUNT.inc()
|
||||
|
||||
async def reorg_hashes(self, count):
|
||||
"""Return a pair (start, last, hashes) of blocks to back up during a
|
||||
@@ -270,7 +272,7 @@ class BlockProcessor:
|
||||
|
||||
return start, last, await self.db.fs_block_hashes(start, count)
|
||||
|
||||
async def calc_reorg_range(self, count):
|
||||
async def calc_reorg_range(self, count: Optional[int]):
|
||||
"""Calculate the reorg range"""
|
||||
|
||||
def diff_pos(hashes1, hashes2):
|
||||
|
||||
@@ -545,11 +545,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
if attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
@@ -596,11 +604,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
if attr == 'tag':
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
|
||||
@@ -433,6 +433,15 @@ class SQLDB:
|
||||
return {r.channel_hash for r in affected_channels}
|
||||
return set()
|
||||
|
||||
def delete_claims_above_height(self, height: int):
|
||||
claim_hashes = [x[0] for x in self.execute(
|
||||
"SELECT claim_hash FROM claim WHERE height>?", (height, )
|
||||
).fetchall()]
|
||||
while claim_hashes:
|
||||
batch = set(claim_hashes[:500])
|
||||
claim_hashes = claim_hashes[500:]
|
||||
self.delete_claims(batch)
|
||||
|
||||
def _clear_claim_metadata(self, claim_hashes: Set[bytes]):
|
||||
if claim_hashes:
|
||||
for table in ('tag',): # 'language', 'location', etc
|
||||
|
||||
@@ -51,6 +51,13 @@ BLOCK_COUNT = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
REORG_COUNT = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import logging
|
||||
from lbry.testcase import IntegrationTestCase
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.wallet.server.prometheus import REORG_COUNT
|
||||
|
||||
|
||||
class BlockchainReorganizationTests(IntegrationTestCase):
|
||||
class BlockchainReorganizationTests(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
@@ -13,21 +16,105 @@ class BlockchainReorganizationTests(IntegrationTestCase):
|
||||
)
|
||||
|
||||
async def test_reorg(self):
|
||||
REORG_COUNT.set(0)
|
||||
# invalidate current block, move forward 2
|
||||
self.assertEqual(self.ledger.headers.height, 200)
|
||||
await self.assertBlockHash(200)
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(200)).decode())
|
||||
self.assertEqual(self.ledger.headers.height, 206)
|
||||
await self.assertBlockHash(206)
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
await self.blockchain.generate(2)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 201)
|
||||
self.assertEqual(self.ledger.headers.height, 201)
|
||||
await self.assertBlockHash(200)
|
||||
await self.assertBlockHash(201)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 207)
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
self.assertEqual(1, REORG_COUNT._samples()[0][2])
|
||||
|
||||
# invalidate current block, move forward 3
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(200)).decode())
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
await self.blockchain.generate(3)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 202)
|
||||
self.assertEqual(self.ledger.headers.height, 202)
|
||||
await self.assertBlockHash(200)
|
||||
await self.assertBlockHash(201)
|
||||
await self.assertBlockHash(202)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 208)
|
||||
self.assertEqual(self.ledger.headers.height, 208)
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
self.assertEqual(2, REORG_COUNT._samples()[0][2])
|
||||
|
||||
async def test_reorg_change_claim_height(self):
|
||||
# sanity check
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertListEqual(txos, [])
|
||||
|
||||
still_valid = await self.daemon.jsonrpc_stream_create(
|
||||
'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!')
|
||||
)
|
||||
await self.ledger.wait(still_valid)
|
||||
await self.generate(1)
|
||||
|
||||
# create a claim and verify it's returned by claim_search
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
broadcast_tx = await self.daemon.jsonrpc_stream_create(
|
||||
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
|
||||
)
|
||||
await self.ledger.wait(broadcast_tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
|
||||
self.assertEqual(self.ledger.headers.height, 208)
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertEqual(1, len(txos))
|
||||
txo = txos[0]
|
||||
self.assertEqual(txo.tx_ref.id, broadcast_tx.id)
|
||||
self.assertEqual(txo.tx_ref.height, 208)
|
||||
|
||||
# check that our tx is in block 208 as returned by lbrycrdd
|
||||
invalidated_block_hash = (await self.ledger.headers.hash(208)).decode()
|
||||
block_207 = await self.blockchain.get_block(invalidated_block_hash)
|
||||
self.assertIn(txo.tx_ref.id, block_207['tx'])
|
||||
self.assertEqual(208, txos[0].tx_ref.height)
|
||||
|
||||
# reorg the last block dropping our claim tx
|
||||
await self.blockchain.invalidate_block(invalidated_block_hash)
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# verify the claim was dropped from block 208 as returned by lbrycrdd
|
||||
reorg_block_hash = await self.blockchain.get_block_hash(208)
|
||||
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
|
||||
block_207 = await self.blockchain.get_block(reorg_block_hash)
|
||||
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
await asyncio.wait_for(self.on_header(209), 3.0)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
await self.assertBlockHash(209)
|
||||
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
|
||||
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
|
||||
|
||||
# verify the dropped claim is no longer returned by claim search
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertListEqual(txos, [])
|
||||
|
||||
# verify the claim published a block earlier wasn't also reverted
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
|
||||
self.assertEqual(1, len(txos))
|
||||
self.assertEqual(207, txos[0].tx_ref.height)
|
||||
|
||||
# broadcast the claim in a different block
|
||||
new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode())
|
||||
self.assertEqual(broadcast_tx.id, new_txid)
|
||||
await self.blockchain.generate(1)
|
||||
|
||||
# wait for the client to catch up
|
||||
await asyncio.wait_for(self.on_header(210), 1.0)
|
||||
|
||||
# verify the claim is in the new block and that it is returned by claim_search
|
||||
block_210 = await self.blockchain.get_block((await self.ledger.headers.hash(210)).decode())
|
||||
self.assertIn(txo.tx_ref.id, block_210['tx'])
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertEqual(1, len(txos))
|
||||
self.assertEqual(txos[0].tx_ref.id, new_txid)
|
||||
self.assertEqual(210, txos[0].tx_ref.height)
|
||||
|
||||
# this should still be unchanged
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
|
||||
self.assertEqual(1, len(txos))
|
||||
self.assertEqual(207, txos[0].tx_ref.height)
|
||||
@@ -1,6 +1,7 @@
|
||||
import os.path
|
||||
import tempfile
|
||||
import logging
|
||||
import asyncio
|
||||
from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
@@ -79,6 +80,12 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
] * 23828
|
||||
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
|
||||
|
||||
# this should do nothing... if the resolve (which is retried) results in the server disconnecting,
|
||||
# it kerplodes
|
||||
await asyncio.wait_for(self.daemon.jsonrpc_resolve([
|
||||
f'0000000000000000000000000000000000000000{i}' for i in range(30000)
|
||||
]), 30)
|
||||
|
||||
# 23829 claim ids makes the request just large enough
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
@@ -423,18 +430,18 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
async def test_txo_list_and_sum_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', unspent=True)))
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
|
||||
await self.channel_update(channel_id, bid='0.5')
|
||||
self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', unspent=True)))
|
||||
self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
|
||||
self.assertEqual('1.5', lbc(await self.txo_sum(type='channel')))
|
||||
|
||||
stream_id = self.get_claim_id(await self.stream_create(bid='1.3'))
|
||||
self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', unspent=True)))
|
||||
self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
|
||||
await self.stream_update(stream_id, bid='0.7')
|
||||
self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', unspent=True)))
|
||||
self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
|
||||
self.assertEqual('2.0', lbc(await self.txo_sum(type='stream')))
|
||||
|
||||
self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], unspent=True)))
|
||||
self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], is_not_spent=True)))
|
||||
self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel'])))
|
||||
|
||||
# type filtering
|
||||
@@ -496,22 +503,35 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
|
||||
await self.channel_create('@kept-channel')
|
||||
await self.channel_create('@sent-channel', claim_address=address2)
|
||||
await self.wallet_send('2.9', address2)
|
||||
|
||||
# all txos on second wallet
|
||||
received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True)
|
||||
received_payment, received_channel = await self.txo_list(
|
||||
wallet_id=wallet2.id, is_my_input_or_output=True)
|
||||
self.assertEqual('1.0', received_channel['amount'])
|
||||
self.assertFalse(received_channel['is_my_input'])
|
||||
self.assertTrue(received_channel['is_my_output'])
|
||||
self.assertFalse(received_channel['is_internal_transfer'])
|
||||
self.assertEqual('2.9', received_payment['amount'])
|
||||
self.assertFalse(received_payment['is_my_input'])
|
||||
self.assertTrue(received_payment['is_my_output'])
|
||||
self.assertFalse(received_payment['is_internal_transfer'])
|
||||
|
||||
# all txos on default wallet
|
||||
r = await self.txo_list(is_my_input_or_output=True)
|
||||
self.assertEqual(
|
||||
['1.0', '7.947786', '1.0', '8.973893', '10.0'],
|
||||
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
|
||||
[t['amount'] for t in r]
|
||||
)
|
||||
|
||||
sent_channel, change2, kept_channel, change1, initial_funds = r
|
||||
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
|
||||
|
||||
self.assertTrue(sent_payment['is_my_input'])
|
||||
self.assertFalse(sent_payment['is_my_output'])
|
||||
self.assertFalse(sent_payment['is_internal_transfer'])
|
||||
self.assertTrue(change3['is_my_input'])
|
||||
self.assertTrue(change3['is_my_output'])
|
||||
self.assertTrue(change3['is_internal_transfer'])
|
||||
|
||||
self.assertTrue(sent_channel['is_my_input'])
|
||||
self.assertFalse(sent_channel['is_my_output'])
|
||||
@@ -533,27 +553,31 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
# my stuff and stuff i sent excluding "change"
|
||||
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
|
||||
self.assertEqual([sent_channel, kept_channel, initial_funds], r)
|
||||
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
|
||||
|
||||
# my unspent stuff and stuff i sent excluding "change"
|
||||
r = await self.txo_list(is_my_input_or_output=True, unspent=True, exclude_internal_transfers=True)
|
||||
self.assertEqual([sent_channel, kept_channel], r)
|
||||
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
|
||||
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
|
||||
|
||||
# only "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
|
||||
self.assertEqual([change2, change1], r)
|
||||
self.assertEqual([change3, change2, change1], r)
|
||||
|
||||
# only unspent "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", unspent=True)
|
||||
self.assertEqual([change2], r)
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
|
||||
self.assertEqual([change3], r)
|
||||
|
||||
# only spent "change"
|
||||
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
|
||||
self.assertEqual([change2, change1], r)
|
||||
|
||||
# all my unspent stuff
|
||||
r = await self.txo_list(is_my_output=True, unspent=True)
|
||||
self.assertEqual([change2, kept_channel], r)
|
||||
r = await self.txo_list(is_my_output=True, is_not_spent=True)
|
||||
self.assertEqual([change3, kept_channel], r)
|
||||
|
||||
# stuff i sent
|
||||
r = await self.txo_list(is_not_my_output=True)
|
||||
self.assertEqual([sent_channel], r)
|
||||
self.assertEqual([sent_payment, sent_channel], r)
|
||||
|
||||
async def test_txo_plot(self):
|
||||
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
|
||||
@@ -615,14 +639,14 @@ class TransactionOutputCommands(ClaimTestCase):
|
||||
for _ in range(10):
|
||||
await self.support_create(stream_id, '0.1')
|
||||
await self.assertBalance(self.account, '7.978478')
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='support', unspent=True)))
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
|
||||
txs = await self.txo_spend(type='support', batch_size=3, include_full_tx=True)
|
||||
self.assertEqual(4, len(txs))
|
||||
self.assertEqual(3, len(txs[0]['inputs']))
|
||||
self.assertEqual(3, len(txs[1]['inputs']))
|
||||
self.assertEqual(3, len(txs[2]['inputs']))
|
||||
self.assertEqual(1, len(txs[3]['inputs']))
|
||||
self.assertEqual('0.0', lbc(await self.txo_sum(type='support', unspent=True)))
|
||||
self.assertEqual('0.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
|
||||
await self.assertBalance(self.account, '8.977606')
|
||||
|
||||
await self.support_create(stream_id, '0.1')
|
||||
@@ -637,12 +661,15 @@ class ClaimCommands(ClaimTestCase):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
|
||||
await self.stream_update(stream_id, title='foo')
|
||||
|
||||
# type filtering
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type='stream')
|
||||
# catch a bug where cli sends is_spent=False by default
|
||||
r = await self.claim_list(claim_type='stream', is_spent=False)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(file1['claim_name'], 'foo')
|
||||
self.assertEqual(file2['claim_name'], 'foo2')
|
||||
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
|
||||
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
|
||||
|
||||
@@ -106,11 +106,16 @@ class MockedCommentServer:
|
||||
return False
|
||||
|
||||
def hide_comments(self, pieces: list):
|
||||
comments_hidden = []
|
||||
hidden = []
|
||||
for p in pieces:
|
||||
if self.hide_comment(**p):
|
||||
comments_hidden.append(p['comment_id'])
|
||||
return {'hidden': comments_hidden}
|
||||
hidden.append(p['comment_id'])
|
||||
|
||||
comment_ids = {c['comment_id'] for c in pieces}
|
||||
return {
|
||||
'hidden': hidden,
|
||||
'visible': list(comment_ids - set(hidden))
|
||||
}
|
||||
|
||||
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
@@ -138,12 +143,19 @@ class MockedCommentServer:
|
||||
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
|
||||
comment = self.comments[self.get_comment_id(comment_id)]
|
||||
return {
|
||||
'channel_id': comment.get('channel_id'),
|
||||
'channel_name': comment.get('channel_name')
|
||||
'channel_id': comment['channel_id'],
|
||||
'channel_name': comment['channel_name'],
|
||||
}
|
||||
|
||||
def get_comments_by_id(self, comment_ids: list):
|
||||
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
|
||||
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
|
||||
return {
|
||||
'page': 1,
|
||||
'page_size': len(comment_ids),
|
||||
'total_pages': 1,
|
||||
'items': comments,
|
||||
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
|
||||
}
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
|
||||
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
|
||||
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
|
||||
for save_blobs in (False, True):
|
||||
await self.setup_blob_manager(save_blobs=save_blobs)
|
||||
# add a blob file
|
||||
@@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase):
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
blob.save_verified_blob(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.blob_manager.blob_completed(blob)
|
||||
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
|
||||
|
||||
@@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
write_blob = blob._write_blob
|
||||
write_called_count = 0
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
async def _wrap_write_blob(blob_bytes):
|
||||
nonlocal write_called_count
|
||||
write_called_count += 1
|
||||
write_blob(blob_bytes)
|
||||
await write_blob(blob_bytes)
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
return asyncio.create_task(_wrap_write_blob(blob_bytes))
|
||||
|
||||
blob._write_blob = wrap_write_blob
|
||||
|
||||
writer1 = blob.get_blob_writer(peer_port=1)
|
||||
@@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
|
||||
self.assertEqual(1, write_called_count)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.assertDictEqual({}, blob.writers)
|
||||
|
||||
|
||||
@@ -46,7 +46,9 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
|
||||
self.assertEqual(100, self.stream.reflector_progress)
|
||||
self.assertSetEqual(
|
||||
set(sent),
|
||||
set(map(lambda b: b.blob_hash,
|
||||
|
||||
@@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
await self.ledger.headers.open()
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase):
|
||||
async def test_deserialize(self):
|
||||
self.maxDiff = None
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS)
|
||||
await h.open()
|
||||
await h.connect(0, HEADERS)
|
||||
self.assertEqual(await h.get(0), {
|
||||
'bits': 520159231,
|
||||
'block_height': 0,
|
||||
@@ -52,8 +52,11 @@ class TestHeaders(AsyncioTestCase):
|
||||
self.assertEqual(headers.height, 19)
|
||||
|
||||
async def test_connect_from_middle(self):
|
||||
h = Headers(':memory:')
|
||||
h.io.write(HEADERS[:block_bytes(10)])
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
h = Headers(headers_temporary_file)
|
||||
await h.open()
|
||||
self.assertEqual(h.height, 9)
|
||||
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
|
||||
@@ -115,6 +118,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_bounds(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS)
|
||||
self.assertEqual(19, headers.height)
|
||||
with self.assertRaises(IndexError):
|
||||
@@ -126,6 +130,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
|
||||
async def test_repair(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
await headers.connect(0, HEADERS[:block_bytes(11)])
|
||||
self.assertEqual(10, headers.height)
|
||||
await headers.repair()
|
||||
@@ -147,18 +152,39 @@ class TestHeaders(AsyncioTestCase):
|
||||
await headers.repair(start_height=10)
|
||||
self.assertEqual(19, headers.height)
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
async def test_do_not_estimate_unconfirmed(self):
|
||||
headers = Headers(':memory:')
|
||||
headers.io.seek(0)
|
||||
headers.io.write(HEADERS)
|
||||
await headers.open()
|
||||
self.assertIsNone(headers.estimated_timestamp(-1))
|
||||
self.assertIsNone(headers.estimated_timestamp(0))
|
||||
self.assertIsNotNone(headers.estimated_timestamp(1))
|
||||
|
||||
async def test_dont_estimate_whats_there(self):
|
||||
headers = Headers(':memory:')
|
||||
await headers.open()
|
||||
estimated = headers.estimated_timestamp(10)
|
||||
await headers.connect(0, HEADERS)
|
||||
real_time = (await headers.get(10))['timestamp']
|
||||
after_downloading_header_estimated = headers.estimated_timestamp(10)
|
||||
self.assertNotEqual(estimated, after_downloading_header_estimated)
|
||||
self.assertEqual(after_downloading_header_estimated, real_time)
|
||||
|
||||
async def test_misalignment_triggers_repair_on_open(self):
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.write(HEADERS)
|
||||
headers = Headers(headers_temporary_file)
|
||||
with self.assertLogs(level='WARN') as cm:
|
||||
await headers.open()
|
||||
await headers.close()
|
||||
self.assertEqual(cm.output, [])
|
||||
headers.io.seek(0)
|
||||
headers.io.truncate()
|
||||
headers.io.write(HEADERS[:block_bytes(10)])
|
||||
headers.io.write(b'ops')
|
||||
headers.io.write(HEADERS[block_bytes(10):])
|
||||
with open(headers_temporary_file, 'w+b') as headers_file:
|
||||
headers_file.seek(0)
|
||||
headers_file.truncate()
|
||||
headers_file.write(HEADERS[:block_bytes(10)])
|
||||
headers_file.write(b'ops')
|
||||
headers_file.write(HEADERS[block_bytes(10):])
|
||||
await headers.open()
|
||||
self.assertEqual(
|
||||
cm.output, [
|
||||
@@ -186,6 +212,7 @@ class TestHeaders(AsyncioTestCase):
|
||||
reader_task = asyncio.create_task(reader())
|
||||
await writer()
|
||||
await reader_task
|
||||
await headers.close()
|
||||
|
||||
|
||||
HEADERS = unhexlify(
|
||||
|
||||
@@ -48,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase):
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
self.ledger.headers.checkpoints = {}
|
||||
await self.ledger.headers.open()
|
||||
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
|
||||
await self.ledger.db.open()
|
||||
|
||||
|
||||
Reference in new issue
Block a user