Compare commits

..
71 changed files with 1647 additions and 3839 deletions

No files matched your search

-3
View File
@@ -67,9 +67,6 @@ jobs:
run: |
pip install pyinstaller
pip install -e .
# https://stackoverflow.com/a/61693590
# https://github.com/pypa/setuptools/issues/1963
pip install --upgrade 'setuptools<45.0.0'
- if: startsWith(matrix.os, 'windows') == false
name: Build & Run (Unix)
run: |
+1 -6
View File
@@ -6,17 +6,12 @@
/.coverage*
/lbry-venv
/venv
/lbry/blockchain
lbry.egg-info
__pycache__
_trial_temp/
trending*.log
/tests/integration/blockchain/files
/tests/.coverage.*
/lbry/wallet/bin
/.vscode
/.gitignore
/lbry/wallet/bin
+2 -4
View File
@@ -16,8 +16,7 @@ services:
- "lbrycrd:/data/.lbrycrd"
environment:
- RUN_MODE=default
# Curently not snapshot provided
#- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- RPC_ALLOW_IP=0.0.0.0/0
wallet_server:
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
@@ -31,6 +30,5 @@ services:
volumes:
- "wallet_server:/database"
environment:
# Curently not snapshot provided
# - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
+167 -475
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.84.0"
__version__ = "0.74.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+1 -1
View File
@@ -337,9 +337,9 @@ class BlobFile(AbstractBlob):
return self.loop.create_task(write_blob())
def delete(self):
super().delete()
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
+9 -11
View File
@@ -5,8 +5,9 @@ import typing
import logging
from argparse import ArgumentParser
from contextlib import contextmanager
from appdirs import user_data_dir, user_config_dir
import yaml
from appdirs import user_data_dir, user_config_dir
from lbry.error import InvalidCurrencyError
from lbry.dht import constants
from lbry.wallet.coinselection import STRATEGIES
@@ -333,7 +334,7 @@ class ConfigFileAccess:
cls = type(self.configuration)
with open(self.path, 'r') as config_file:
raw = config_file.read()
serialized = yaml.safe_load(raw) or {}
serialized = yaml.load(raw) or {}
for key, value in serialized.items():
attr = getattr(cls, key, None)
if attr is None:
@@ -477,16 +478,16 @@ class TranscodeConfig(BaseConfig):
'', previous_names=['ffmpeg_folder'])
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
r'-maxrate 5500K -bufsize 5000K')
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
@@ -611,7 +612,7 @@ class Config(CLIConfig):
('lbrynet4.lbry.com', 4444) # ASIA
])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
@@ -624,9 +625,6 @@ class Config(CLIConfig):
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
allowed_origin = String(
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
"all hosts; default is to only allow API requests with no `Origin` value.", "")
# media server
streaming_server = String('Host name and port to serve streaming media over range requests',
@@ -636,7 +634,7 @@ class Config(CLIConfig):
coin_selection_strategy = StringChoice(
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
STRATEGIES, "sqlite")
transaction_cache_size = Integer("Transaction cache size", 100_000)
save_resolved_claims = Toggle(
+6 -8
View File
@@ -33,14 +33,12 @@ def is_valid_public_ipv4(address, allow_localhost: bool = False):
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
parsed_ip.is_reserved,
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except ipaddress.AddressValueError:
return False
+7 -17
View File
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
self.response = response.decode()
def _decode_datagram(datagram: bytes):
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
msg_types = {
REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram,
@@ -152,29 +152,19 @@ def _decode_datagram(datagram: bytes):
}
primitive: typing.Dict = bdecode(datagram)
converted = {
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
}
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
else:
raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type]
decoded = {
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
k: primitive[i] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.required_fields)
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
if i in primitive # pylint: disable=unsupported-membership-test
}
for i, _ in enumerate(OPTIONAL_FIELDS):
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
return decoded, datagram_class
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
decoded, datagram_class = _decode_datagram(datagram)
if i + OPTIONAL_ARG_OFFSET in primitive:
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
return datagram_class(**decoded)
+5
View File
@@ -14,6 +14,7 @@ from aiohttp.web import GracefulExit
from docopt import docopt
from lbry import __version__ as lbrynet_version
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.extras.daemon.daemon import Daemon
from lbry.conf import Config, CLIConfig
@@ -254,6 +255,10 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
else:
logger.getChild('lbry').setLevel(logging.DEBUG)
loggly_handler = get_loggly_handler(conf)
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config):
loop = asyncio.get_event_loop()
+5 -15
View File
@@ -22,10 +22,10 @@ def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
@@ -41,9 +41,9 @@ def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id
return False
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
def sign_comment(comment: dict, channel: Output, abandon=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
@@ -52,21 +52,11 @@ def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
'signing_ts': timestamp
})
def sign_reaction(reaction: dict, channel: Output):
timestamp = str(int(time.time()))
signing_field = reaction['channel_name']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
reaction.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
+56 -281
View File
@@ -47,7 +47,6 @@ from lbry.extras.daemon.componentmanager import ComponentManager
from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder
from lbry.extras.daemon import comment_client
from lbry.extras.daemon.undecorated import undecorated
from lbry.extras.daemon.security import ensure_request_allowed
from lbry.file_analysis import VideoFileAnalyzer
from lbry.schema.claim import Claim
from lbry.schema.url import URL
@@ -557,14 +556,6 @@ class Daemon(metaclass=JSONRPCServerType):
await self.component_manager.stop()
else:
self.component_startup_task.cancel()
# the wallet component might have not started
try:
wallet_component = self.component_manager.get_actual_component('wallet')
except NameError:
pass
else:
await wallet_component.stop()
await self.component_manager.stop()
log.info("stopped api components")
await self.rpc_runner.cleanup()
await self.streaming_runner.cleanup()
@@ -575,7 +566,6 @@ class Daemon(metaclass=JSONRPCServerType):
log.info("finished shutting down")
async def handle_old_jsonrpc(self, request):
ensure_request_allowed(request, self.conf)
data = await request.json()
params = data.get('params', {})
include_protobuf = params.pop('include_protobuf', False) if isinstance(params, dict) else False
@@ -994,12 +984,10 @@ class Daemon(metaclass=JSONRPCServerType):
[--include_sent_supports]
[--include_sent_tips]
[--include_received_tips]
[--new_sdk_server=<new_sdk_server>]
Options:
--urls=<urls> : (str, list) one or more urls to resolve
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_purchase_receipt : (bool) lookup and include a receipt if this wallet
has purchased the claim being resolved
--include_is_my_output : (bool) lookup and include a boolean indicating
@@ -1303,9 +1291,8 @@ class Daemon(metaclass=JSONRPCServerType):
'name': SingleKey.name if single_key else HierarchicalDeterministic.name
}
)
await self.ledger._subscribe_accounts([account])
elif wallet.accounts:
await self.ledger._subscribe_accounts(wallet.accounts)
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
wallet.save()
if not skip_on_startup:
with self.conf.update_config() as c:
@@ -1332,9 +1319,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not os.path.exists(wallet_path):
raise Exception(f"Wallet at path '{wallet_path}' was not found.")
wallet = self.wallet_manager.import_wallet(wallet_path)
if not self.ledger.network.is_connected(wallet.id):
await self.ledger._subscribe_accounts(wallet.accounts)
if self.ledger.network.is_connected:
for account in wallet.accounts:
await self.ledger.subscribe_account(account)
return wallet
@requires("wallet")
@@ -1470,7 +1457,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT)
async def jsonrpc_wallet_send(
self, amount, addresses, wallet_id=None,
change_account_id=None, funding_account_ids=None, preview=False, blocking=True):
change_account_id=None, funding_account_ids=None, preview=False):
"""
Send the same number of credits to multiple addresses using all accounts in wallet to
fund the transaction and the default account to receive any change.
@@ -1478,14 +1465,12 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
wallet_send <amount> <addresses>... [--wallet_id=<wallet_id>] [--preview]
[--change_account_id=None] [--funding_account_ids=<funding_account_ids>...]
[--blocking]
Options:
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--change_account_id=<wallet_id> : (str) account where change will go
--funding_account_ids=<funding_account_ids> : (str) accounts to fund the transaction
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until tx has synced
--preview : (bool) do not broadcast the transaction
Returns: {Transaction}
"""
@@ -1511,11 +1496,13 @@ class Daemon(metaclass=JSONRPCServerType):
tx = await Transaction.create(
[], outputs, accounts, account
)
if not preview:
await self.broadcast_or_release(tx, blocking)
await self.ledger.broadcast(tx)
self.component_manager.loop.create_task(self.analytics_manager.send_credits_sent())
else:
await self.ledger.release_tx(tx)
return tx
ACCOUNT_DOC = """
@@ -1620,7 +1607,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
)
wallet.save()
if self.ledger.network.is_connected(wallet.id):
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
return account
@@ -1648,7 +1635,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
)
wallet.save()
if self.ledger.network.is_connected(wallet.id):
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
return account
@@ -1725,7 +1712,7 @@ class Daemon(metaclass=JSONRPCServerType):
change_made = True
if change_made:
account.modified_on = int(time.time())
account.modified_on = time.time()
wallet.save()
return account
@@ -1793,26 +1780,24 @@ class Daemon(metaclass=JSONRPCServerType):
)
@requires(WALLET_COMPONENT)
def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False, blocking=False):
def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False):
"""
Send the same number of credits to multiple addresses from a specific account (or default account).
Usage:
account_send <amount> <addresses>... [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview]
[--blocking]
Options:
--account_id=<account_id> : (str) account to fund the transaction
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until tx has synced
Returns: {Transaction}
"""
return self.jsonrpc_wallet_send(
amount=amount, addresses=addresses, wallet_id=wallet_id,
change_account_id=account_id, funding_account_ids=[account_id] if account_id else [],
preview=preview, blocking=blocking
preview=preview
)
SYNC_DOC = """
@@ -1864,7 +1849,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet_changed = False
if data is not None:
added_accounts = wallet.merge(self.wallet_manager, password, data)
if added_accounts and self.ledger.network.is_connected(wallet.id):
if added_accounts and self.ledger.network.is_connected:
if blocking:
await asyncio.wait([
a.ledger.subscribe_account(a) for a in added_accounts
@@ -2304,41 +2289,6 @@ class Daemon(metaclass=JSONRPCServerType):
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(**kwargs)
async def jsonrpc_support_sum(self, claim_id, new_sdk_server, include_channel_content=False, **kwargs):
"""
List total staked supports for a claim, grouped by the channel that signed the support.
+
+ If claim_id is a channel claim, you can use --include_channel_content to also include supports for
+ content claims in the channel.
!!!! NOTE: PAGINATION DOES NOT DO ANYTHING AT THE MOMENT !!!!!
Usage:
support_sum <claim_id> <new_sdk_server>
[--include_channel_content]
[--page=<page>] [--page_size=<page_size>]
Options:
--claim_id=<claim_id> : (str) claim id
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_channel_content : (bool) if claim_id is for a channel, include supports for claims in
that channel
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
Returns: {Paginated[Dict]}
"""
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
support_sums = await self.ledger.sum_supports(
new_sdk_server, claim_id=claim_id, include_channel_content=include_channel_content, **kwargs
)
return {
"items": support_sums,
"page": page_num,
"page_size": page_size
}
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
"""
@@ -2354,7 +2304,6 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel=<channel> |
[[--channel_ids=<channel_ids>...] [--not_channel_ids=<not_channel_ids>...]]]
[--has_channel_signature] [--valid_channel_signature | --invalid_channel_signature]
[--limit_claims_per_channel=<limit_claims_per_channel>]
[--is_controlling] [--release_time=<release_time>] [--public_key_id=<public_key_id>]
[--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
[--height=<height>] [--creation_height=<creation_height>]
@@ -2374,7 +2323,6 @@ class Daemon(metaclass=JSONRPCServerType):
[--not_locations=<not_locations>...]
[--order_by=<order_by>...] [--page=<page>] [--page_size=<page_size>]
[--wallet_id=<wallet_id>] [--include_purchase_receipt] [--include_is_my_output]
[--new_sdk_server=<new_sdk_server>]
Options:
--name=<name> : (str) claim name (normalized)
@@ -2403,8 +2351,6 @@ class Daemon(metaclass=JSONRPCServerType):
--invalid_channel_signature : (bool) claims with invalid channel signature or no signature,
use in conjunction with --has_channel_signature to
only get claims with invalid signatures
--limit_claims_per_channel=<limit_claims_per_channel>: (int) only return up to the specified
number of claims per channel
--is_controlling : (bool) winning claims of their respective name
--public_key_id=<public_key_id> : (str) only return channels having this public key id, this is
the same key as used in the wallet file to map
@@ -2482,7 +2428,6 @@ class Daemon(metaclass=JSONRPCServerType):
has purchased the claim
--include_is_my_output : (bool) lookup and include a boolean indicating
if claim being resolved is yours
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
Returns: {Paginated[Output]}
"""
@@ -2958,7 +2903,7 @@ class Daemon(metaclass=JSONRPCServerType):
'public_key': data['holding_public_key'],
'address_generator': {'name': 'single-address'}
})
if self.ledger.network.is_connected(wallet.id):
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
await self.ledger._update_tasks.done.wait()
# Case 3: the holding address has changed and we can't create or find an account for it
@@ -3328,7 +3273,7 @@ class Daemon(metaclass=JSONRPCServerType):
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish'))
else:
await account.ledger.release_tx(tx)
log.info("successful publish %s", tx.id)
return tx
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
@@ -4033,9 +3978,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT)
async def jsonrpc_support_create(
self, claim_id, amount, tip=False,
channel_id=None, channel_name=None, channel_account_id=None,
account_id=None, wallet_id=None, funding_account_ids=None,
self, claim_id, amount, tip=False, account_id=None, wallet_id=None, funding_account_ids=None,
preview=False, blocking=False):
"""
Create a support or a tip for name claim.
@@ -4043,30 +3986,23 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
support_create (<claim_id> | --claim_id=<claim_id>) (<amount> | --amount=<amount>)
[--tip] [--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--preview] [--blocking] [--funding_account_ids=<funding_account_ids>...]
Options:
--claim_id=<claim_id> : (str) claim_id of the claim to support
--amount=<amount> : (decimal) amount of support
--tip : (bool) send support to claim owner, default: false.
--channel_id=<channel_id> : (str) claim id of the supporters identity channel
--channel_name=<channel_name> : (str) name of the supporters identity channel
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--account_id=<account_id> : (str) account to use for holding the transaction
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--claim_id=<claim_id> : (str) claim_id of the claim to support
--amount=<amount> : (decimal) amount of support
--tip : (bool) send support to claim owner, default: false.
--account_id=<account_id> : (str) account to use for holding the transaction
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
Returns: {Transaction}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
amount = self.get_dewies_or_error("amount", amount)
claim = await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id)
claim_address = claim.get_address(self.ledger)
@@ -4075,13 +4011,8 @@ class Daemon(metaclass=JSONRPCServerType):
claim_address = await account.receiving.get_or_create_usable_address()
tx = await Transaction.support(
claim.claim_name, claim_id, amount, claim_address, funding_accounts, funding_accounts[0], channel
claim.claim_name, claim_id, amount, claim_address, funding_accounts, funding_accounts[0]
)
new_txo = tx.outputs[0]
if channel:
new_txo.sign(channel)
await tx.sign(funding_accounts)
if not preview:
await self.broadcast_or_release(tx, blocking)
@@ -5040,9 +4971,10 @@ class Daemon(metaclass=JSONRPCServerType):
View, create and abandon comments.
"""
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_list(self, claim_id, parent_id=None, page=1, page_size=50,
include_replies=False, skip_validation=False,
is_channel_signature_valid=False, hidden=False, visible=False):
include_replies=True, is_channel_signature_valid=False,
hidden=False, visible=False):
"""
List comments associated with a claim.
@@ -5050,7 +4982,7 @@ class Daemon(metaclass=JSONRPCServerType):
comment_list (<claim_id> | --claim_id=<claim_id>)
[(--page=<page> --page_size=<page_size>)]
[--parent_id=<parent_id>] [--include_replies]
[--skip_validation] [--is_channel_signature_valid]
[--is_channel_signature_valid]
[--visible | --hidden]
Options:
@@ -5058,7 +4990,6 @@ class Daemon(metaclass=JSONRPCServerType):
--parent_id=<parent_id> : (str) CommentId of a specific thread you'd like to see
--page=<page> : (int) The page you'd like to see in the comment list.
--page_size=<page_size> : (int) The amount of comments that you'd like to retrieve
--skip_validation : (bool) Skip resolving comments to validate channel names
--include_replies : (bool) Whether or not you want to include replies in list
--is_channel_signature_valid : (bool) Only include comments with valid signatures.
[Warning: Paginated total size will not change, even
@@ -5092,9 +5023,8 @@ class Daemon(metaclass=JSONRPCServerType):
if hidden ^ visible:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'comment.List',
'get_claim_hidden_comments',
claim_id=claim_id,
visible=visible,
hidden=hidden,
page=page,
page_size=page_size
@@ -5102,29 +5032,28 @@ class Daemon(metaclass=JSONRPCServerType):
else:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'comment.List',
'get_claim_comments',
claim_id=claim_id,
parent_id=parent_id,
page=page,
page_size=page_size,
top_level=not include_replies
)
if not skip_validation:
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([], [channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
comment['is_channel_signature_valid'] = False
if is_channel_signature_valid:
result['items'] = [
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
]
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([], [channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
comment['is_channel_signature_valid'] = False
if is_channel_signature_valid:
result['items'] = [
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
]
return result
@requires(WALLET_COMPONENT)
@@ -5154,10 +5083,8 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"is_pinned": (boolean) Channel owner has pinned this comment,
"signature": (str) The signature of the comment,
"signing_ts": (str) The timestamp used to sign the comment,
"channel_url": (str) Channel's URI in the ClaimTrie,
@@ -5179,7 +5106,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Create', comment_body)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
@@ -5205,20 +5132,18 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"signature": (str) The signature of the comment,
"signing_ts": (str) Timestamp used to sign the most recent signature,
"channel_url": (str) Channel's URI in the ClaimTrie,
"is_pinned": (boolean) Channel owner has pinned this comment,
"parent_id": (str) Comment this is replying to, (None) if this is the root,
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
}
"""
channel = await comment_client.jsonrpc_post(
self.conf.comment_server,
'comment.GetChannelFromCommentID',
'get_channel_from_comment_id',
comment_id=comment_id
)
if 'error' in channel:
@@ -5235,7 +5160,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(edited_comment, channel_claim)
return await comment_client.jsonrpc_post(
self.conf.comment_server, 'comment.Edit', edited_comment
self.conf.comment_server, 'edit_comment', edited_comment
)
@requires(WALLET_COMPONENT)
@@ -5261,7 +5186,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'comment.GetChannelFromCommentID', comment_id=comment_id
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'abandoned': False}}
@@ -5270,8 +5195,8 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Abandon', abandon_comment_body)
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'abandon_comment', abandon_comment_body)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_hide(self, comment_ids: typing.Union[str, list], wallet_id=None):
@@ -5315,159 +5240,9 @@ class Daemon(metaclass=JSONRPCServerType):
for_signing=True
)
piece = {'comment_id': comment['comment_id']}
comment_client.sign_comment(piece, channel, sign_comment_id=True)
comment_client.sign_comment(piece, channel, abandon=True)
pieces.append(piece)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Hide', pieces=pieces)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_pin(self, comment_id=None, channel_id=None, channel_name=None,
channel_account_id=None, remove=False, wallet_id=None):
"""
Pin a comment published to a claim you control.
Usage:
comment_pin (<comment_id> | --comment_id=<comment_id>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
[--remove]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_id=<comment_id> : (str) Hash identifying the comment to pin
--channel_id=<claim_id> : (str) The ID of channel owning the commented claim
--channel_name=<claim_name> : (str) The name of channel owning the commented claim
--remove : (bool) remove the pin
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--wallet_id=<wallet_id : (str) restrict operation to specific wallet
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_pin_args = {
'comment_id': comment_id,
'channel_name': channel_name,
'channel_id': channel_id,
'remove': remove,
}
comment_client.sign_comment(comment_pin_args, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Pin', comment_pin_args)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, remove=False, clear_types=None, react_type=None, wallet_id=None):
"""
Create and associate a reaction emoji with a comment using your channel identity.
Usage:
comment_react (--comment_ids=<comment_ids>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
(--react_type=<react_type>)
[(--remove) | (--clear_types=<clear_types>)]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_ids=<comment_ids> : (str) one or more comment id reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--react_type=<react_type> : (str) name of reaction type
--remove : (bool) remove specified react_type
--clear_types=<clear_types> : (str) types to clear when adding another type
Returns:
(dict) Reaction object if successfully made, (None) otherwise
{
"Reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction
...
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_body = {
'comment_ids': comment_ids,
'channel_id': channel_id,
'channel_name': channel.claim_name,
'type': react_type,
'remove': remove,
'clear_types': clear_types,
}
comment_client.sign_reaction(react_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.React', react_body)
return response
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react_list(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, react_types=None, wallet_id=None):
"""
List reactions emoji with a claim using your channel identity.
Usage:
comment_react_list (--comment_ids=<comment_ids>)
[(--channel_id=<channel_id>)(--channel_name=<channel_name>)]
[--react_types=<react_types>]
Options:
--comment_ids=<comment_ids> : (str) The comment ids reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--react_types=<react_type> : (str) comma delimited reaction types
Returns:
(dict) Comment object if successfully made, (None) otherwise
{
"my_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
"other_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
react_list_body = {
'comment_ids': comment_ids,
}
if channel_id:
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_list_body['channel_id'] = channel_id
react_list_body['channel_name'] = channel.claim_name
if react_types:
react_list_body['types'] = react_types
if channel_id:
comment_client.sign_reaction(react_list_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.List', react_list_body)
return response
return await comment_client.jsonrpc_post(self.conf.comment_server, 'hide_comments', pieces=pieces)
async def broadcast_or_release(self, tx, blocking=False):
await self.wallet_manager.broadcast_or_release(tx, blocking)
+16 -22
View File
@@ -7,7 +7,6 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError
from lbry.schema.claim import Claim
from lbry.schema.support import Support
from lbry.torrent.torrent_manager import TorrentSource
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
from lbry.wallet.bip32 import PubKey
@@ -136,8 +135,6 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_output(obj)
if isinstance(obj, Claim):
return self.encode_claim(obj)
if isinstance(obj, Support):
return obj.to_dict()
if isinstance(obj, PubKey):
return obj.extended_key_string()
if isinstance(obj, datetime):
@@ -223,28 +220,25 @@ class JSONResponseEncoder(JSONEncoder):
output['claims'] = [self.encode_output(o) for o in txo.claims]
if txo.reposted_claim is not None:
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
try:
output['value'] = txo.signable
if self.include_protobuf:
output['protobuf'] = hexlify(txo.signable.to_bytes())
if txo.purchase_receipt is not None:
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
if txo.script.is_claim_name or txo.script.is_update_claim:
if txo.script.is_claim_name or txo.script.is_update_claim:
try:
output['value'] = txo.claim
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.purchase_receipt is not None:
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
if txo.claim.is_channel:
output['has_signing_key'] = txo.has_private_key
elif txo.script.is_support_claim_data:
output['value_type'] = 'emoji'
if check_signature and txo.signable.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
output['is_channel_signature_valid'] = False
except DecodeError:
pass
if check_signature and txo.claim.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
output['is_channel_signature_valid'] = False
except DecodeError:
pass
return output
def encode_claim_meta(self, meta):
+95
View File
@@ -0,0 +1,95 @@
import asyncio
import json
import logging.handlers
import traceback
import typing
from aiohttp.client_exceptions import ClientError
import aiohttp
from lbry import utils, __version__
if typing.TYPE_CHECKING:
from lbry.conf import Config
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
super().__init__()
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, config: 'Config'):
super().__init__()
self.cookies = {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
self._config = config
@property
def enabled(self):
return self._config.share_usage_data
@staticmethod
def get_full_message(record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
async def _emit(self, record, retry=True):
data = self.format(record).encode()
try:
async with self._session.post(self.url, data=data,
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
if self._loop.is_running() and retry and self.enabled:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
if not self.enabled:
return
try:
asyncio.ensure_future(self._emit(record), loop=self._loop)
except RuntimeError: # TODO: use a second loop
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
f"{self.format(record)}")
def close(self):
super().close()
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(self._session.close())
except RuntimeError:
pass
def get_loggly_handler(config):
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
handler.setFormatter(JsonFormatter())
return handler
-31
View File
@@ -1,31 +0,0 @@
import logging
from aiohttp import web
log = logging.getLogger(__name__)
def ensure_request_allowed(request, conf):
if is_request_allowed(request, conf):
return
if conf.allowed_origin:
log.warning(
"API requests with Origin '%s' are not allowed, "
"configuration 'allowed_origin' limits requests to: '%s'",
request.headers.get('Origin'), conf.allowed_origin
)
else:
log.warning(
"API requests with Origin '%s' are not allowed, "
"update configuration 'allowed_origin' to enable this origin.",
request.headers.get('Origin')
)
raise web.HTTPForbidden()
def is_request_allowed(request, conf) -> bool:
origin = request.headers.get('Origin')
return (
origin is None or
origin == conf.allowed_origin or
conf.allowed_origin == '*'
)
+7 -7
View File
@@ -199,8 +199,6 @@ class VideoFileAnalyzer:
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
if int(stream['sample_rate']) > 48000:
return "Sample rate out of range"
return ""
@@ -306,6 +304,9 @@ class VideoFileAnalyzer:
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
async def _get_volume_filter(self):
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
@staticmethod
def _get_best_container_extension(scan_data, video_encoder):
# the container is chosen by the video format
@@ -409,7 +410,7 @@ class VideoFileAnalyzer:
# the plan for transcoding:
# we have to re-encode the video if it is in a nonstandard format
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
# we also re-encode if our bitrate or sample rate is too high
# we also re-encode if our bitrate is too high
try:
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
@@ -429,10 +430,9 @@ class VideoFileAnalyzer:
if audio_msg or volume_msg:
audio_encoder = await self._get_audio_encoder(extension)
transcode_command.append(audio_encoder)
if volume_msg and self._conf.volume_filter:
transcode_command.append(self._conf.volume_filter)
if audio_msg == "Sample rate out of range":
transcode_command.append(" -ar 48000 ")
if volume_msg:
volume_filter = await self._get_volume_filter()
transcode_command.append(volume_filter)
else:
transcode_command.append("copy")
+5 -1
View File
@@ -30,10 +30,14 @@ class Claim(Signable):
COLLECTION = 'collection'
REPOST = 'repost'
__slots__ = ()
__slots__ = 'version',
message_class = ClaimMessage
def __init__(self, message=None):
super().__init__(message)
self.version = 2
@property
def claim_type(self) -> str:
return self.message.WhichOneof('type')
+1 -3
View File
@@ -47,7 +47,7 @@ types_map = {
'.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'video'),
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
'.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'),
@@ -61,7 +61,6 @@ types_map = {
'.mp3': ('audio/mpeg', 'audio'),
'.mp4': ('video/mp4', 'video'),
'.mpa': ('video/mpeg', 'video'),
'.mpd': ('application/dash+xml', 'video'),
'.mpe': ('video/mpeg', 'video'),
'.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'),
@@ -123,7 +122,6 @@ types_map = {
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.vtt': ('text/vtt', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
+2 -9
View File
@@ -25,14 +25,12 @@ def set_reference(reference, claim_hash, rows):
class Censor:
__slots__ = 'streams', 'channels', 'limit_claims_per_channel', 'censored', 'claims_in_channel', 'total'
__slots__ = 'streams', 'channels', 'censored', 'total'
def __init__(self, streams: dict = None, channels: dict = None, limit_claims_per_channel: int = None):
def __init__(self, streams: dict = None, channels: dict = None):
self.streams = streams or {}
self.channels = channels or {}
self.limit_claims_per_channel = limit_claims_per_channel # doesn't count as censored
self.censored = {}
self.claims_in_channel = {}
self.total = 0
def censor(self, row) -> bool:
@@ -51,11 +49,6 @@ class Censor:
break
if was_censored:
self.total += 1
if not was_censored and self.limit_claims_per_channel is not None and row['channel_hash']:
self.claims_in_channel.setdefault(row['channel_hash'], 0)
self.claims_in_channel[row['channel_hash']] += 1
if self.claims_in_channel[row['channel_hash']] > self.limit_claims_per_channel:
return True
return was_censored
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
+1 -10
View File
@@ -1,15 +1,6 @@
from lbry.schema.base import Signable
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
class Support(Signable):
__slots__ = ()
message_class = SupportMessage
@property
def emoji(self) -> str:
return self.message.emoji
@emoji.setter
def emoji(self, emoji: str):
self.message.emoji = emoji
message_class = None # TODO: add support protobufs
-69
View File
@@ -1,69 +0,0 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: support.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='support.proto',
package='pb',
syntax='proto3',
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\"\x18\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\tb\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_SUPPORT = _descriptor.Descriptor(
name='Support',
full_name='pb.Support',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='emoji', full_name='pb.Support.emoji', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=21,
serialized_end=45,
)
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
DESCRIPTOR = _SUPPORT,
__module__ = 'support_pb2'
# @@protoc_insertion_point(class_scope:pb.Support)
))
_sym_db.RegisterMessage(Support)
# @@protoc_insertion_point(module_scope)
+9 -2
View File
@@ -22,7 +22,8 @@ def _create_url_regex():
return _group(
_named(name+"_name", prefix + invalid_names_regex) +
_oneof(
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
@@ -49,6 +50,7 @@ def normalize_name(name):
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
@@ -59,13 +61,17 @@ class PathSegment(NamedTuple):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
def __str__(self):
if self.claim_id is not None:
return f"{self.name}:{self.claim_id}"
return f"{self.name}#{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
@@ -112,6 +118,7 @@ class URL(NamedTuple):
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
-13
View File
@@ -356,23 +356,10 @@ class ManagedStream(ManagedDownloadSource):
return sent
except ConnectionRefusedError:
return sent
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
raise err
if isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
raise asyncio.CancelledError()
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+5 -27
View File
@@ -60,16 +60,10 @@ class StreamReflectorClient(asyncio.Protocol):
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request
except (AttributeError, asyncio.CancelledError) as err:
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
finally:
self.pending_request = None
@@ -94,16 +88,8 @@ class StreamReflectorClient(asyncio.Protocol):
needed = response.get('needed_blobs', [])
sent_sd = False
if response['send_sd_blob']:
try:
sent = await sd_blob.sendfile(self)
if sent == -1:
log.warning("failed to send sd blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
await sd_blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -126,16 +112,8 @@ class StreamReflectorClient(asyncio.Protocol):
if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
try:
sent = await blob.sendfile(self)
if sent == -1:
log.warning("failed to send blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
await blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+6 -40
View File
@@ -15,9 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -29,25 +27,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.descriptor: typing.Optional['StreamDescriptor'] = None
self.sd_blob: typing.Optional['BlobFile'] = None
self.received = []
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
self.incoming = asyncio.Event(loop=self.loop)
self.chunk_size = response_chunk_size
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
async def wait_for_stop(self):
await self.stop_event.wait()
if self.transport:
self.transport.close()
def connection_made(self, transport):
self.transport = transport
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
def connection_lost(self, exc):
if self.wait_for_stop_task:
self.wait_for_stop_task.cancel()
self.wait_for_stop_task = None
def data_received(self, data: bytes):
if self.incoming.is_set():
@@ -89,7 +73,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set()
self.send_response({"send_sd_blob": True})
try:
@@ -103,7 +86,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.transport.close()
finally:
self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle()
self.writer = None
else:
@@ -111,7 +93,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.not_incoming.set()
if self.writer:
self.writer.close_handle()
self.writer = None
@@ -131,7 +112,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified():
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set()
self.send_response({"send_blob": True})
try:
@@ -140,7 +120,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle()
self.writer = None
else:
@@ -151,18 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.stopped_listening = asyncio.Event(loop=self.loop)
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
self.response_chunk_size = response_chunk_size
self.stop_event = stop_event
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -170,20 +143,13 @@ class ReflectorServer:
async def _start_server():
server = await self.loop.create_server(
lambda: ReflectorServerProtocol(
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
self.not_incoming_event
),
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
interface, port
)
self.started_listening.set()
self.stopped_listening.clear()
log.info("Reflector server listening on TCP %s:%i", interface, port)
try:
async with server:
await server.serve_forever()
finally:
self.stopped_listening.set()
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
-14
View File
@@ -159,18 +159,8 @@ class StreamManager(SourceManager):
))
async def reflect_streams(self):
try:
return await self._reflact_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
async def _reflact_streams(self):
# todo: those debug statements are temporary for #2987 - remove them if its closed
while True:
if self.config.reflect_streams and self.config.reflector_servers:
log.debug("collecting streams to reflect")
sd_hashes = await self.storage.get_streams_to_re_reflect()
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
batch = []
@@ -181,14 +171,10 @@ class StreamManager(SourceManager):
stream.fully_reflected.is_set():
batch.append(self.reflect_stream(stream))
if len(batch) >= self.config.concurrent_reflector_uploads:
log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.gather(*batch, loop=self.loop)
log.debug("done processing %s streams", len(batch))
batch = []
if batch:
log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.gather(*batch, loop=self.loop)
log.debug("done processing %s streams", len(batch))
await asyncio.sleep(300, loop=self.loop)
async def start(self):
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+3 -3
View File
@@ -311,7 +311,7 @@ class Account:
private_key=private_key,
public_key=public_key,
address_generator=d.get('address_generator', {}),
modified_on=int(d.get('modified_on', time.time())),
modified_on=d.get('modified_on', time.time()),
channel_keys=d.get('certificates', {})
)
@@ -343,7 +343,7 @@ class Account:
def merge(self, d: dict):
if d.get('modified_on', 0) > self.modified_on:
self.name = d['name']
self.modified_on = int(d.get('modified_on', time.time()))
self.modified_on = d.get('modified_on', time.time())
assert self.address_generator.name == d['address_generator']['name']
for chain_name in ('change', 'receiving'):
if chain_name in d['address_generator']:
@@ -564,7 +564,7 @@ class Account:
self.change.gap = new_change_gap
gap_changed = True
if gap_changed:
await asyncio.get_event_loop().run_in_executor(None, self.wallet.save)
self.wallet.save()
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False, read_only=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0
+68 -113
View File
@@ -7,6 +7,8 @@ from binascii import hexlify
from collections import defaultdict
from dataclasses import dataclass
from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from prometheus_client import Gauge, Counter, Histogram
@@ -17,12 +19,6 @@ from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Inpu
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
else:
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
log = logging.getLogger(__name__)
sqlite3.enable_callback_tracebacks(True)
@@ -66,6 +62,12 @@ def run_read_only_fetchone(sql, params):
raise
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
ReaderExecutorClass = ThreadPoolExecutor
else:
ReaderExecutorClass = ProcessPoolExecutor
class AIOSQLite:
reader_executor: ReaderExecutorClass
@@ -121,12 +123,7 @@ class AIOSQLite:
if self._closing:
return
self._closing = True
def __checkpoint_and_close(conn: sqlite3.Connection):
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
conn.close()
await asyncio.get_event_loop().run_in_executor(
self.writer_executor, __checkpoint_and_close, self.writer_connection)
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
self.writer_executor.shutdown(wait=True)
self.reader_executor.shutdown(wait=True)
self.read_ready.clear()
@@ -150,7 +147,7 @@ class AIOSQLite:
self.waiting_reads_metric.inc()
self.read_count_metric.inc()
try:
while self.writers and not self._closing: # more writes can come in while we are waiting for the first
while self.writers: # more writes can come in while we are waiting for the first
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
# throttle the writes if they pile up
self.urgent_read_done.clear()
@@ -158,8 +155,6 @@ class AIOSQLite:
# wait until the running writes have finished
await self.read_ready.wait()
still_waiting = True
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.reader_executor, read_only_fn, sql, parameters
)
@@ -202,8 +197,6 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
@@ -239,8 +232,6 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
@@ -476,100 +467,68 @@ def dict_row_factory(cursor, row):
return d
SQLITE_MAX_INTEGER = 9223372036854775807
def get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, reserve_amount: int, floor: int,
fee_per_byte: int):
txs = defaultdict(list)
decoded_transactions = {}
def _get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, decoded_transactions: Dict[str, Transaction],
result: Dict[Tuple[bytes, int, bool], List[int]], reserved: List[Transaction],
amount_to_reserve: int, reserved_amount: int, floor: int, ceiling: int,
fee_per_byte: int) -> int:
accumulated = 0
multiplier = 10
gap_count = 0
accounts_fmt = ",".join(["?"] * len(accounts))
txo_query = f"""
SELECT tx.txid, txo.txoid, tx.raw, tx.height, txo.position as nout, tx.is_verified, txo.amount FROM txo
INNER JOIN account_address USING (address)
LEFT JOIN txi USING (txoid)
INNER JOIN tx USING (txid)
WHERE txo.txo_type=0 AND txi.txoid IS NULL AND tx.txid IS NOT NULL AND NOT txo.is_reserved
AND txo.amount >= ? AND txo.amount < ?
WHERE txo.txo_type=0 AND txi.txoid IS NULL AND tx.txid IS NOT NULL AND tx.is_verified AND NOT txo.is_reserved
AND txo.amount BETWEEN ? AND ?
"""
if accounts:
txo_query += f"""
AND account_address.account {'= ?' if len(accounts_fmt) == 1 else 'IN (' + accounts_fmt + ')'}
"""
txo_query += """
ORDER BY txo.amount ASC, tx.height DESC
"""
# prefer confirmed, but save unconfirmed utxos from this selection in case they are needed
unconfirmed = []
for row in transaction.execute(txo_query, (floor, ceiling, *accounts)):
(txid, txoid, raw, height, nout, verified, amount) = row.values()
# verified or non verified transactions were found- reset the gap count
# multiple txos can come from the same tx, only decode it once and cache
if txid not in decoded_transactions:
# cache the decoded transaction
decoded_transactions[txid] = Transaction(raw)
decoded_tx = decoded_transactions[txid]
# save the unconfirmed txo for possible use later, if still needed
if verified:
# add the txo to the reservation, minus the fee for including it
reserved_amount += amount
reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
# mark it as reserved
result[(raw, height, verified)].append(nout)
reserved.append(txoid)
# if we've reserved enough, return
if reserved_amount >= amount_to_reserve:
return reserved_amount
else:
unconfirmed.append((txid, txoid, raw, height, nout, verified, amount))
# we're popping the items, so to get them in the order they were seen they are reversed
unconfirmed.reverse()
# add available unconfirmed txos if any were previously found
while unconfirmed and reserved_amount < amount_to_reserve:
(txid, txoid, raw, height, nout, verified, amount) = unconfirmed.pop()
# it's already decoded
decoded_tx = decoded_transactions[txid]
# add to the reserved amount
reserved_amount += amount
reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
result[(raw, height, verified)].append(nout)
reserved.append(txoid)
return reserved_amount
def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: List, amount_to_reserve: int, floor: int,
fee_per_byte: int, set_reserved: bool, return_insufficient_funds: bool,
base_multiplier: int = 100):
txs = defaultdict(list)
decoded_transactions = {}
reserved = []
reserved_dewies = 0
multiplier = base_multiplier
gap_count = 0
while reserved_dewies < amount_to_reserve and gap_count < 5 and floor * multiplier < SQLITE_MAX_INTEGER:
previous_reserved_dewies = reserved_dewies
reserved_dewies = _get_spendable_utxos(
transaction, accounts, decoded_transactions, txs, reserved, amount_to_reserve, reserved_dewies,
floor, floor * multiplier, fee_per_byte
)
floor *= multiplier
if previous_reserved_dewies == reserved_dewies:
while accumulated < reserve_amount:
found_txs = False
# prefer confirmed, but save unconfirmed utxos from this selection in case they are needed
unconfirmed = []
for row in transaction.execute(txo_query, (floor, floor * multiplier, *accounts)):
(txid, txoid, raw, height, nout, verified, amount) = row.values()
found_txs = True
if txid not in decoded_transactions:
decoded_transactions[txid] = Transaction(raw)
decoded_tx = decoded_transactions[txid]
if not verified:
unconfirmed.append((txid, txoid, raw, height, nout, verified, amount))
continue
accumulated += amount
accumulated -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
txs[(raw, height, verified)].append(nout)
reserved.append(txoid)
if accumulated >= reserve_amount:
break
unconfirmed.reverse()
while unconfirmed:
(txid, txoid, raw, height, nout, verified, amount) = unconfirmed.pop()
decoded_tx = decoded_transactions[txid]
accumulated += amount
accumulated -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
txs[(raw, height, verified)].append(nout)
reserved.append(txoid)
if accumulated >= reserve_amount:
break
if not found_txs:
gap_count += 1
multiplier **= 2
else:
gap_count = 0
multiplier = base_multiplier
if gap_count == 5:
break
floor *= multiplier
# reserve the accumulated txos if enough were found
if reserved_dewies >= amount_to_reserve:
if set_reserved:
transaction.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?",
[(True, txoid) for txoid in reserved]).fetchall()
return txs
# return_insufficient_funds and set_reserved are used for testing
return txs if return_insufficient_funds else {}
if accumulated >= reserve_amount:
transaction.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?",
[(True, txoid) for txoid in reserved]).fetchall()
return txs
class Database(SQLiteMixin):
@@ -772,12 +731,11 @@ class Database(SQLiteMixin):
# 2. update address histories removing deleted TXs
return True
async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 1,
fee_per_byte: int = 50, set_reserved: bool = True,
return_insufficient_funds: bool = False) -> List:
async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 100000,
fee_per_byte: int = 50) -> List:
to_spend = await self.db.run(
get_and_reserve_spendable_utxos, tuple(account.id for account in accounts), reserve_amount, min_amount,
fee_per_byte, set_reserved, return_insufficient_funds
get_spendable_utxos, tuple(account.id for account in accounts), reserve_amount, min_amount,
fee_per_byte
)
txos = []
for (raw, height, verified), positions in to_spend.items():
@@ -1263,16 +1221,13 @@ class Database(SQLiteMixin):
self.constrain_collections(constraints)
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account=None):
if account is None:
await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1")
else:
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
async def release_all_outputs(self, account):
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
def get_supports_summary(self, read_only=False, **constraints):
return self.get_txos(
+134 -363
View File
@@ -3,6 +3,7 @@ import copy
import time
import asyncio
import logging
from io import StringIO
from datetime import datetime
from functools import partial
from operator import itemgetter
@@ -29,6 +30,7 @@ from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
from .bip32 import PubKey, PrivateKey
from .coinselection import CoinSelector
log = logging.getLogger(__name__)
LedgerType = Type['BaseLedger']
@@ -122,7 +124,7 @@ class Ledger(metaclass=LedgerRegistry):
self.headers.checkpoints = self.checkpoints
self.network: Network = self.config.get('network') or Network(self)
self.network.on_header.listen(self.receive_header)
# self.network.on_status.listen(self.process_status_update)
self.network.on_status.listen(self.process_status_update)
self.network.on_connected.listen(self.join_network)
self.accounts = []
@@ -131,7 +133,7 @@ class Ledger(metaclass=LedgerRegistry):
self._on_transaction_controller = StreamController()
self.on_transaction = self._on_transaction_controller.stream
self.on_transaction.listen(
lambda e: log.info(
lambda e: log.debug(
'(%s) on_transaction: address=%s, height=%s, is_verified=%s, tx.id=%s',
self.get_id(), e.address, e.tx.height, e.tx.is_verified, e.tx.id
)
@@ -162,10 +164,10 @@ class Ledger(metaclass=LedgerRegistry):
self._utxo_reservation_lock = asyncio.Lock()
self._header_processing_lock = asyncio.Lock()
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
self._history_lock = asyncio.Lock()
self.coin_selection_strategy = None
self._known_addresses_out_of_sync = set()
self.went_out_of_sync = asyncio.Queue()
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self._balance_cache = pylru.lrucache(100000)
@@ -243,7 +245,8 @@ class Ledger(metaclass=LedgerRegistry):
def get_address_count(self, **constraints):
return self.db.get_address_count(**constraints)
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1):
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']],
min_amount=100000):
min_amount = min(amount // 10, min_amount)
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
selector = CoinSelector(amount, fee)
@@ -324,16 +327,12 @@ class Ledger(metaclass=LedgerRegistry):
self.db.open(),
self.headers.open()
])
fully_synced = self.on_ready.first
asyncio.create_task(self.network.start())
await self.network.on_connected.first
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
await fully_synced
await self.db.release_all_outputs()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -451,16 +450,10 @@ class Ledger(metaclass=LedgerRegistry):
)
async def subscribe_accounts(self):
return await self._subscribe_accounts(self.accounts)
async def _subscribe_accounts(self, accounts):
wallet_ids = {account.wallet.id for account in accounts}
await self.network.connect_wallets(*wallet_ids)
# if self.network.is_connected and self.accounts:
if accounts:
log.info("Subscribe to %i accounts", len(accounts))
if self.network.is_connected and self.accounts:
log.info("Subscribe to %i accounts", len(self.accounts))
await asyncio.wait([
self.subscribe_account(a) for a in accounts
self.subscribe_account(a) for a in self.accounts
])
async def subscribe_account(self, account: Account):
@@ -469,10 +462,8 @@ class Ledger(metaclass=LedgerRegistry):
await account.ensure_address_gap()
async def unsubscribe_account(self, account: Account):
session = self.network.get_wallet_session(account.wallet)
for address in await account.get_addresses():
await self.network.unsubscribe_address(address, session=session)
await self.network.close_wallet_session(account.wallet)
await self.network.unsubscribe_address(address)
async def announce_addresses(self, address_manager: AddressManager, addresses: List[str]):
await self.subscribe_addresses(address_manager, addresses)
@@ -481,131 +472,96 @@ class Ledger(metaclass=LedgerRegistry):
)
async def subscribe_addresses(self, address_manager: AddressManager, addresses: List[str], batch_size: int = 1000):
if self.network.is_connected(address_manager.account.wallet.id) and addresses:
session = self.network.get_wallet_session(address_manager.account.wallet)
if self.network.is_connected and addresses:
addresses_remaining = list(addresses)
while addresses_remaining:
batch = addresses_remaining[:batch_size]
results = await self.network.subscribe_address(session, *batch)
results = await self.network.subscribe_address(*batch)
for address, remote_status in zip(batch, results):
self._update_tasks.add(self.update_history(session, address, remote_status, address_manager))
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
addresses_remaining = addresses_remaining[batch_size:]
if session and session.server_address_and_port:
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *session.server_address_and_port)
if session and session.server_address_and_port:
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*session.server_address_and_port
)
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *self.network.client.server_address_and_port)
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*self.network.client.server_address_and_port
)
def process_status_update(self, client, update):
def process_status_update(self, update):
address, remote_status = update
self._update_tasks.add(self.update_history(client, address, remote_status))
self._update_tasks.add(self.update_history(address, remote_status))
async def update_history(self, client, address, remote_status, address_manager: AddressManager = None,
reattempt_update: bool = True):
async def update_history(self, address, remote_status, address_manager: AddressManager = None):
async with self._address_update_locks[address]:
self._known_addresses_out_of_sync.discard(address)
local_status, local_history = await self.get_local_status_and_history(address)
if local_status == remote_status:
log.info("no new txs needed for %s, remote matches local", address)
return True
# remote_history = await self.network.retriable_call(self.network.get_history, address, client)
remote_history = await self.network.get_history(address, session=client)
remote_history = await self.network.retriable_call(self.network.get_history, address)
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
we_need = set(remote_history) - set(local_history)
if not we_need:
remote_missing = set(local_history) - set(remote_history)
if remote_missing:
log.warning(
"%i transactions we have for %s are not in the remote address history",
len(remote_missing), address
)
log.info(
"no new txs needed for %s, local status %s, remote status %s", address, local_status, remote_status
)
return True
acquire_lock_tasks = []
cache_tasks: List[asyncio.Task[Transaction]] = []
synced_history = StringIO()
loop = asyncio.get_running_loop()
for i, (txid, remote_height) in enumerate(remote_history):
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
synced_history.write(f'{txid}:{remote_height}:')
else:
check_local = (txid, remote_height) not in we_need
cache_tasks.append(loop.create_task(
self.cache_transaction(txid, remote_height, check_local=check_local)
))
synced_txs = []
to_request = {}
pending_synced_history = {}
updated_cached_items = {}
already_synced = set()
for task in cache_tasks:
tx = await task
already_synced_offset = 0
for i, (txid, remote_height) in enumerate(remote_history):
if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height):
pending_synced_history[i] = f'{txid}:{remote_height}:'
already_synced.add((txid, remote_height))
already_synced_offset += 1
continue
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
check_db_for_txos = []
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
if cache_item is not None:
if cache_item.tx is None:
await cache_item.has_tx.wait()
assert cache_item.tx is not None
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos.append(txi.txo_ref.id)
for txid, remote_height in remote_history[already_synced_offset:]:
cache_item = self._tx_cache[txid]
acquire_lock_tasks.append(asyncio.create_task(cache_item.lock.acquire()))
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
)
}
if acquire_lock_tasks:
await asyncio.wait(acquire_lock_tasks)
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
referenced_txo = referenced_txos.get(txi.txo_ref.id)
if referenced_txo is not None:
txi.txo_ref = referenced_txo.ref
tx_indexes = {}
for i, (txid, remote_height) in enumerate(remote_history):
tx_indexes[txid] = i
if (txid, remote_height) in already_synced:
continue
cache_item = self._tx_cache.get(txid)
cache_item.pending_verifications += 1
updated_cached_items[txid] = cache_item
assert cache_item is not None, 'cache item is none'
assert cache_item.lock.locked(), 'cache lock is not held?'
# tx = cache_item.tx
# if cache_item.tx is not None and \
# cache_item.tx.height >= remote_height and \
# (cache_item.tx.is_verified or remote_height < 1):
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
# continue
to_request[i] = (txid, remote_height)
log.debug(
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
len(remote_history), address
)
requested_txes = await self._request_transaction_batch(to_request, len(remote_history), address, client)
for tx in requested_txes:
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
synced_history.write(f'{tx.id}:{tx.height}:')
synced_txs.append(tx)
assert len(pending_synced_history) == len(remote_history), \
f"{len(pending_synced_history)} vs {len(remote_history)}"
synced_history = ""
for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())):
assert i == remote_i, f"{i} vs {remote_i}"
txid, height = remote_history[remote_i]
if f"{txid}:{height}:" != pending_synced_history[i]:
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
synced_history += pending_synced_history[i]
cache_size = self.config.get("tx_cache_size", 100_000)
for txid, cache_item in updated_cached_items.items():
cache_item.pending_verifications -= 1
if cache_item.pending_verifications < 0:
log.warning("config value tx cache size %i needs to be increased", cache_size)
cache_item.pending_verifications = 0
try:
cache_item.lock.release()
except RuntimeError:
log.warning("lock was already released?")
await self.db.save_transaction_io_batch(
[], address, self.address_to_hash160(address), synced_history
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
)
await asyncio.wait([
self._on_transaction_controller.add(TransactionEvent(address, tx))
for tx in synced_txs
])
if address_manager is None:
address_manager = await self.get_address_manager_for_address(address)
@@ -613,16 +569,8 @@ class Ledger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
for txid, cache_item in updated_cached_items.items():
if self._tx_cache.get(txid) is not cache_item:
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
if reattempt_update:
return await self.update_history(client, address, remote_status, address_manager, False)
return False
local_status, local_history = \
await self.get_local_status_and_history(address, synced_history)
await self.get_local_status_and_history(address, synced_history.getvalue())
if local_status != remote_status:
if local_history == remote_history:
log.warning(
@@ -642,13 +590,18 @@ class Ledger(metaclass=LedgerRegistry):
"******",
address, remote_status, len(remote_history), len(remote_set),
local_status, len(local_history), len(local_set), len(remote_set.symmetric_difference(local_set)),
"\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)]),
"\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)])
"\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)]),
"\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)])
)
# log.warning("local: %s", local_history)
# log.warning("remote: %s", remote_history)
self.went_out_of_sync.put_nowait(address)
self._known_addresses_out_of_sync.add(address)
return False
else:
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
log.warning(
"%s is synced", address
)
return True
async def cache_transaction(self, txid, remote_height, check_local=True):
@@ -660,39 +613,41 @@ class Ledger(metaclass=LedgerRegistry):
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
cache_item.pending_verifications += 1
try:
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
cache_item.tx = tx # make sure it's saved before caching it
tx.height = remote_height
if merkle and 0 < remote_height < len(self.headers):
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
cache_item.pending_verifications += 1
return await self._update_cache_item(cache_item, txid, remote_height, check_local)
finally:
cache_item.pending_verifications -= 1
async def _update_cache_item(self, cache_item, txid, remote_height, check_local=True):
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle.get('block_height'))
cache_item.tx = tx # make sure it's saved before caching it
await self.maybe_verify_transaction(tx, remote_height, merkle)
return tx
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx.height = remote_height
cached = self._tx_cache.get(tx.id)
if not cached:
# cache txs looked up by transaction_show too
cached = TransactionCacheItem()
cached.tx = tx
self._tx_cache[tx.id] = cached
cached.tx = tx
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
if not merkle:
@@ -702,154 +657,6 @@ class Ledger(metaclass=LedgerRegistry):
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
async def request_transactions_for_inflate(self, to_request: Tuple[Tuple[str, int], ...], session_override=None):
header_cache = {}
batches = [[]]
remote_heights = {}
transactions = []
heights_in_batch = 0
last_height = 0
for txid, height in sorted(to_request, key=lambda x: x[1]):
remote_heights[txid] = height
if height != last_height:
heights_in_batch += 1
last_height = height
if len(batches[-1]) == 100 or heights_in_batch == 20:
batches.append([])
heights_in_batch = 1
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
async def _single_batch(batch):
if session_override:
batch_result = await self.network.get_transaction_batch(
batch, restricted=False, session=session_override
)
else:
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
merkle_height = merkle['block_height']
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx = cache_item.tx or Transaction(bytes.fromhex(raw), height=remote_height)
tx.height = remote_height
cache_item.tx = tx
if 'merkle' in merkle and remote_heights[txid] > 0:
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
try:
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
except IndexError:
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
else:
header_cache[remote_heights[txid]] = header
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
transactions.append(tx)
for batch in batches:
await _single_batch(batch)
return transactions
async def _request_transaction_batch(self, to_request, remote_history_size, address, session):
header_cache = {}
batches = [[]]
remote_heights = {}
synced_txs = []
heights_in_batch = 0
last_height = 0
for idx in sorted(to_request):
txid = to_request[idx][0]
height = to_request[idx][1]
remote_heights[txid] = height
if height != last_height:
heights_in_batch += 1
last_height = height
if len(batches[-1]) == 100 or heights_in_batch == 20:
batches.append([])
heights_in_batch = 1
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
last_showed_synced_count = 0
async def _single_batch(batch):
this_batch_synced = []
# batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, session)
batch_result = await self.network.get_transaction_batch(batch, session)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
merkle_height = merkle['block_height']
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx = cache_item.tx or Transaction(unhexlify(raw), height=remote_height)
tx.height = remote_height
cache_item.tx = tx
if 'merkle' in merkle and remote_heights[txid] > 0:
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
try:
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
except IndexError:
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
else:
header_cache[remote_heights[txid]] = header
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
check_db_for_txos = []
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
if cache_item is not None:
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos.append(txi.txo_ref.id)
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
)
}
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
referenced_txo = referenced_txos.get(txi.txo_ref.id)
if referenced_txo is not None:
txi.txo_ref = referenced_txo.ref
continue
cache_item = self._tx_cache.get(txi.txo_ref.id)
if cache_item is None:
cache_item = self._tx_cache[txi.txo_ref.id] = TransactionCacheItem()
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.ref
synced_txs.append(tx)
this_batch_synced.append(tx)
await self.db.save_transaction_io_batch(
this_batch_synced, address, self.address_to_hash160(address), ""
)
await asyncio.wait([
self._on_transaction_controller.add(TransactionEvent(address, tx))
for tx in this_batch_synced
])
nonlocal last_showed_synced_count
if last_showed_synced_count + 100 < len(synced_txs):
log.info("synced %i/%i transactions for %s", len(synced_txs), remote_history_size, address)
last_showed_synced_count = len(synced_txs)
for batch in batches:
await _single_batch(batch)
return synced_txs
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
details = await self.db.get_address(address=address)
for account in self.accounts:
@@ -886,26 +693,20 @@ class Ledger(metaclass=LedgerRegistry):
address_record['address']
)) for address_record in records
], timeout=1)
if not pending:
return True
records = await self.db.get_addresses(address__in=addresses)
for record in records:
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id:
if local_height >= height:
return True
log.warning(
"local history has higher height than remote for %s (%i vs %i)", txid,
local_height, height
)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
found = False
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id and local_height >= height:
found = True
if not found:
log.debug("timeout: %s, %s, %s", record['history'], addresses, tx.id)
return False
log.warning(
"local history does not contain %s, requested height %i", tx.id, height
)
return False
return True
async def _inflate_outputs(
self, query, accounts,
@@ -913,17 +714,14 @@ class Ledger(metaclass=LedgerRegistry):
include_is_my_output=False,
include_sent_supports=False,
include_sent_tips=False,
include_received_tips=False,
session_override=None) -> Tuple[List[Output], dict, int, int]:
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
if len(outputs.txs) > 0:
txs: List[Transaction] = []
if session_override:
txs.extend((await self.request_transactions_for_inflate(tuple(outputs.txs), session_override)))
else:
txs.extend((await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))))
txs: List[Transaction] = await asyncio.gather(*(
self.cache_transaction(*tx) for tx in outputs.txs
))
_txos, blocked = outputs.inflate(txs)
@@ -995,29 +793,13 @@ class Ledger(metaclass=LedgerRegistry):
txo.received_tips = tips
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
txos = []
async def resolve(self, accounts, urls, **kwargs):
resolve = partial(self.network.retriable_call, self.network.resolve)
urls_copy = list(urls)
if new_sdk_server:
resolve = partial(self.network.new_resolve, new_sdk_server)
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, **kwargs
))[0]
)
else:
async with self.network.single_call_context(self.network.resolve) as (resolve, session):
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, session_override=session, **kwargs
))[0]
)
txos = []
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -1030,25 +812,14 @@ class Ledger(metaclass=LedgerRegistry):
result[url] = txo
return result
async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]:
return await self.network.sum_supports(new_sdk_server, **kwargs)
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
if new_sdk_server:
claim_search = partial(self.network.new_claim_search, new_sdk_server)
return await self._inflate_outputs(
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output,
)
async with self.network.single_call_context(self.network.claim_search) as (claim_search, session):
return await self._inflate_outputs(
claim_search(**kwargs), accounts, session_override=session,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output,
)
**kwargs) -> Tuple[List[Output], dict, int, int]:
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
)
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
+5 -2
View File
@@ -300,8 +300,11 @@ class WalletManager:
async def broadcast_or_release(self, tx, blocking=False):
try:
await self.ledger.broadcast(tx)
if blocking:
await self.ledger.wait(tx, timeout=None)
except CodeMessageError as err:
log.warning("transaction rejected, leaving reserved")
raise
except:
await self.ledger.release_tx(tx)
raise
if blocking:
await self.ledger.wait(tx, timeout=None)
+51 -190
View File
@@ -1,19 +1,16 @@
import logging
import asyncio
import json
import time
from time import perf_counter
from operator import itemgetter
from contextlib import asynccontextmanager
from functools import partial
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
from lbry.wallet.stream import StreamController
log = logging.getLogger(__name__)
@@ -33,26 +30,11 @@ class ClientSession(BaseClientSession):
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
self._connected = asyncio.Event()
self._disconnected = asyncio.Event()
self._disconnected.set()
self._on_status_controller = StreamController(merge_repeated_events=True)
self.on_status = self._on_status_controller.stream
self.on_status.listen(partial(self.network.ledger.process_status_update, self))
self.subscription_controllers = {
'blockchain.headers.subscribe': self.network._on_header_controller,
'blockchain.address.subscribe': self._on_status_controller,
}
@property
def available(self):
return not self.is_closing() and self.response_time is not None
@property
def is_connected(self) -> bool:
return self._connected.is_set()
@property
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
if not self.transport:
@@ -73,8 +55,8 @@ class ClientSession(BaseClientSession):
return result
async def send_request(self, method, args=()):
log.info("%i in flight, send %s%s to %s:%i ", self.pending_amount, method, tuple(args), *self.server)
self.pending_amount += 1
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
try:
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
@@ -82,11 +64,11 @@ class ClientSession(BaseClientSession):
while not request.done():
done, pending = await asyncio.wait([request], timeout=self.timeout)
if pending:
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
log.info("Time since last packet: %s", perf_counter() - self.last_packet_received)
if (perf_counter() - self.last_packet_received) < self.timeout:
continue
log.info("timeout sending %s to %s:%i", method, *self.server)
raise asyncio.TimeoutError
log.warning("timeout sending %s(%s) to %s:%i", method, str(args), *self.server)
raise asyncio.TimeoutError()
if done:
try:
return request.result()
@@ -110,9 +92,11 @@ class ClientSession(BaseClientSession):
raise
finally:
self.pending_amount -= 1
log.info("%i in flight, finished %s%s ", self.pending_amount, method, tuple(args))
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
# TODO: change to 'ping' on newer protocol (above 1.2)
retry_delay = default_delay = 1.0
while True:
try:
@@ -158,23 +142,17 @@ class ClientSession(BaseClientSession):
self.connection_latency = perf_counter() - start
async def handle_request(self, request):
controller = self.subscription_controllers[request.method]
controller = self.network.subscription_controllers[request.method]
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
log.warning("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
# self.pending_amount = 0
self._on_disconnect_controller.add(True)
self._connected.clear()
self._disconnected.set()
def connection_made(self, transport):
super(ClientSession, self).connection_made(transport)
self._disconnected.clear()
self._connected.set()
class Network:
@@ -186,7 +164,6 @@ class Network:
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.clients: Dict[str, ClientSession] = {}
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
self.running = False
@@ -199,34 +176,25 @@ class Network:
self._on_header_controller = StreamController(merge_repeated_events=True)
self.on_header = self._on_header_controller.stream
self._on_status_controller = StreamController(merge_repeated_events=True)
self.on_status = self._on_status_controller.stream
self.subscription_controllers = {
'blockchain.headers.subscribe': self._on_header_controller,
'blockchain.address.subscribe': self._on_status_controller,
}
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
def get_wallet_session(self, wallet):
return self.clients[wallet.id]
async def close_wallet_session(self, wallet):
session = self.clients.pop(wallet.id)
ensure_connect_task = self.session_pool.wallet_session_tasks.pop(session)
if ensure_connect_task and not ensure_connect_task.done():
ensure_connect_task.cancel()
await session.close()
@property
def config(self):
return self.ledger.config
async def switch_forever(self):
while self.running:
if self.is_connected():
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
@@ -242,42 +210,25 @@ class Network:
async def start(self):
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
await self.ledger.subscribe_accounts()
async def connect_wallets(self, *wallet_ids):
if wallet_ids:
await asyncio.wait([self.session_pool.connect_wallet(wallet_id) for wallet_id in wallet_ids])
async def stop(self):
if self.running:
self.running = False
await self.aiohttp_session.close()
self._switch_task.cancel()
self.session_pool.stop()
def is_connected(self, wallet_id: str = None):
if wallet_id is None:
if not self.client:
return False
return self.client.is_connected
if wallet_id not in self.clients:
return False
return self.clients[wallet_id].is_connected
@property
def is_connected(self):
return self.client and not self.client.is_closing()
def connect_wallet_client(self, wallet_id: str, server: Tuple[str, int]):
client = ClientSession(network=self, server=server)
self.clients[wallet_id] = client
return client
def rpc(self, list_or_method, args, restricted=True, session=None):
session = session or (self.client if restricted else self.session_pool.fastest_session)
def rpc(self, list_or_method, args, restricted=True):
session = self.client if restricted else self.session_pool.fastest_session
if session and not session.is_closing():
return session.send_request(list_or_method, args)
else:
@@ -299,148 +250,76 @@ class Network:
pass
raise asyncio.CancelledError() # if we got here, we are shutting down
@asynccontextmanager
async def fastest_connection_context(self):
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
server = self.session_pool.fastest_session.server
session = ClientSession(network=self, server=server)
async def call_with_reconnect(function, *args, **kwargs):
nonlocal session
while self.running:
if not session.is_connected:
try:
await session.create_connection()
except asyncio.TimeoutError:
if not session.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
server = self.session_pool.fastest_session.server
session = ClientSession(network=self, server=server)
try:
return await partial(function, *args, session_override=session, **kwargs)(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("'%s' failed, retrying", function.__name__)
try:
yield call_with_reconnect
finally:
await session.close()
@asynccontextmanager
async def single_call_context(self, function, *args, **kwargs):
if not self.is_connected():
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
server = self.session_pool.fastest_session.server
session = ClientSession(network=self, server=server)
async def call_with_reconnect(*a, **kw):
while self.running:
if not session.available:
await session.create_connection()
try:
return await partial(function, *args, session_override=session, **kwargs)(*a, **kw)
except asyncio.TimeoutError:
log.warning("'%s' failed, retrying", function.__name__)
try:
yield (call_with_reconnect, session)
finally:
await session.close()
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
def get_transaction(self, tx_hash, known_height=None, session=None):
def get_transaction(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted, session=session)
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_batch(self, txids, restricted=True, session=None):
# use any server if its old, otherwise restrict to who gave us the history
return self.rpc('blockchain.transaction.get_batch', txids, restricted, session=session)
def get_transaction_and_merkle(self, tx_hash, known_height=None, session=None):
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.info', [tx_hash], restricted, session=session)
return self.rpc('blockchain.transaction.info', [tx_hash], restricted)
def get_transaction_height(self, tx_hash, known_height=None, session=None):
def get_transaction_height(self, tx_hash, known_height=None):
restricted = not known_height or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted, session=session)
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
def get_merkle(self, tx_hash, height, session=None):
def get_merkle(self, tx_hash, height):
restricted = 0 > height > self.remote_height - 10
return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height], restricted, session=session)
return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height], restricted)
def get_headers(self, height, count=10000, b64=False):
restricted = height >= self.remote_height - 100
return self.rpc('blockchain.block.headers', [height, count, 0, b64], restricted)
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
def get_history(self, address, session=None):
return self.rpc('blockchain.address.get_history', [address], True, session=session)
async def get_history(self, address):
log.info("get history %s", address)
start = time.perf_counter()
try:
return await self.rpc('blockchain.address.get_history', [address], True)
finally:
log.info("%s history took %s", address, time.perf_counter() - start)
def broadcast(self, raw_transaction, session=None):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], True, session=session)
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], True)
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True], True)
async def subscribe_address(self, session, address, *addresses):
async def subscribe_address(self, address, *addresses):
addresses = list((address, ) + addresses)
server_addr_and_port = session.server_address_and_port # on disconnect client will be None
server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None
try:
return await self.rpc('blockchain.address.subscribe', addresses, True, session=session)
return await self.rpc('blockchain.address.subscribe', addresses, True)
except asyncio.TimeoutError:
log.warning(
"timed out subscribing to addresses from %s:%i",
*server_addr_and_port
)
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
if session:
session.abort()
if self.client:
self.client.abort()
log.warning("raise cancelled")
raise asyncio.CancelledError()
def unsubscribe_address(self, address, session=None):
return self.rpc('blockchain.address.unsubscribe', [address], True, session=session)
def unsubscribe_address(self, address):
return self.rpc('blockchain.address.unsubscribe', [address], True)
def get_server_features(self, session=None):
return self.rpc('server.features', (), restricted=True, session=session)
def get_server_features(self):
return self.rpc('server.features', (), restricted=True)
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def resolve(self, urls, session_override=None):
return self.rpc('blockchain.claimtrie.resolve', urls, False, session=session_override)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def claim_search(self, session_override=None, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs, False, session=session_override)
async def new_resolve(self, server, urls):
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def new_claim_search(self, server, **kwargs):
kwargs['protobuf'] = True
message = {"method": "claim_search", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def sum_supports(self, server, **kwargs):
message = {"method": "support_sum", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
class SessionPool:
@@ -448,7 +327,6 @@ class SessionPool:
def __init__(self, network: Network, timeout: float):
self.network = network
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.wallet_session_tasks: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.timeout = timeout
self.new_connection_event = asyncio.Event()
@@ -508,23 +386,6 @@ class SessionPool:
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
async def connect_wallet(self, wallet_id: str):
fastest = await self.wait_for_fastest_session()
connected = asyncio.Event()
if not self.network.is_connected(wallet_id):
session = self.network.connect_wallet_client(wallet_id, fastest.server)
session._on_connect_cb = connected.set
# session._on_connect_cb = self._get_session_connect_callback(session)
else:
connected.set()
session = self.network.clients[wallet_id]
task = self.wallet_session_tasks.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
# task.add_done_callback(lambda _: self.ensure_connections())
self.wallet_session_tasks[session] = task
await connected.wait()
def start(self, default_servers):
for server in default_servers:
self._connect_session(server)
+4 -9
View File
@@ -206,7 +206,7 @@ class SessionBase(asyncio.Protocol):
"""
return self._address
def peer_address_str(self, for_log=True):
def peer_address_str(self):
"""Returns the peer's IP address and port as a human-readable
string."""
if not self._address:
@@ -480,20 +480,15 @@ class RPCSession(SessionBase):
await event.wait()
result = event.result
if isinstance(result, Exception):
self.logger.error("%s sending %s(%s)", result, method, "" if not args else ",".join([str(a) for a in args]))
raise result
return result
async def send_notification(self, method, args=()) -> bool:
async def send_notification(self, method, args=()):
"""Send an RPC notification over the network."""
message = self.connection.send_notification(Notification(method, args))
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
try:
await self._send_message(message)
return True
except asyncio.TimeoutError:
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
self.abort()
return False
await self._send_message(message)
def send_batch(self, raise_errors=False):
"""Return a BatchRequest. Intended to be used like so:
-27
View File
@@ -438,17 +438,6 @@ class OutputScript(Script):
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
))
SUPPORT_CLAIM_DATA_OPCODES = (
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
OP_2DROP, OP_2DROP
)
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
))
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
))
UPDATE_CLAIM_OPCODES = (
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
OP_2DROP, OP_2DROP
@@ -485,8 +474,6 @@ class OutputScript(Script):
CLAIM_NAME_SCRIPT,
SUPPORT_CLAIM_PUBKEY,
SUPPORT_CLAIM_SCRIPT,
SUPPORT_CLAIM_DATA_PUBKEY,
SUPPORT_CLAIM_DATA_SCRIPT,
UPDATE_CLAIM_PUBKEY,
UPDATE_CLAIM_SCRIPT,
SELL_CLAIM, SELL_SCRIPT,
@@ -540,16 +527,6 @@ class OutputScript(Script):
'pubkey_hash': pubkey_hash
})
@classmethod
def pay_support_data_pubkey_hash(
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
'claim_name': claim_name,
'claim_id': claim_id,
'support': support,
'pubkey_hash': pubkey_hash
})
@classmethod
def sell_script(cls, price):
return cls(template=cls.SELL_SCRIPT, values={
@@ -598,10 +575,6 @@ class OutputScript(Script):
def is_support_claim(self):
return self.template.name.startswith('support_claim+')
@property
def is_support_claim_data(self):
return self.template.name.startswith('support_claim+data+')
@property
def is_sell_claim(self):
return self.template.name.startswith('sell_claim+')
+38 -31
View File
@@ -171,8 +171,7 @@ class BlockProcessor:
# Caches of unflushed items.
self.headers = []
self.block_hashes = []
self.block_txs = []
self.tx_hashes = []
self.undo_infos = []
# UTXO cache
@@ -252,7 +251,7 @@ class BlockProcessor:
async def get_raw_blocks(last_height, hex_hashes):
heights = range(last_height, last_height - len(hex_hashes), -1)
try:
blocks = [await self.db.read_raw_block(height) for height in heights]
blocks = [self.db.read_raw_block(height) for height in heights]
self.logger.info(f'read {len(blocks)} blocks from disk')
return blocks
except FileNotFoundError:
@@ -337,8 +336,8 @@ class BlockProcessor:
def flush_data(self):
"""The data for a flush. The lock must be taken."""
assert self.state_lock.locked()
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
self.block_txs, self.undo_infos, self.utxo_cache,
return FlushData(self.height, self.tx_count, self.headers,
self.tx_hashes, self.undo_infos, self.utxo_cache,
self.db_deletes, self.tip)
async def flush(self, flush_utxos):
@@ -352,9 +351,11 @@ class BlockProcessor:
# performed on the DB.
if self._caught_up_event.is_set():
await self.flush(True)
elif time.perf_counter() > self.next_cache_check:
await self.flush(True)
self.next_cache_check = time.perf_counter() + 30
elif time.time() > self.next_cache_check:
flush_arg = self.check_cache_size()
if flush_arg is not None:
await self.flush(flush_arg)
self.next_cache_check = time.time() + 30
def check_cache_size(self):
"""Flush a cache if it gets too big."""
@@ -393,8 +394,7 @@ class BlockProcessor:
for block in blocks:
height += 1
undo_info = self.advance_txs(
height, block.transactions, self.coin.electrum_header(block.header, height),
self.coin.header_hash(block.header)
height, block.transactions, self.coin.electrum_header(block.header, height)
)
if height >= min_height:
self.undo_infos.append((undo_info, height))
@@ -405,26 +405,25 @@ class BlockProcessor:
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
def advance_txs(self, height, txs, header, block_hash):
self.block_hashes.append(block_hash)
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
undo_info = []
tx_num = self.tx_count
hashXs_by_tx = []
def advance_txs(self, height, txs, header):
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
# Use local vars for speed in the loops
undo_info = []
tx_num = self.tx_count
script_hashX = self.coin.hashX_from_script
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
update_touched = self.touched.update
append_hashX_by_tx = hashXs_by_tx.append
hashX_from_script = self.coin.hashX_from_script
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
for tx, tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
tx_numb = pack('<I', tx_num)
tx_numb = s_pack('<I', tx_num)
# Spend the inputs
for txin in tx.inputs:
@@ -437,16 +436,18 @@ class BlockProcessor:
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hashX. Ignore unspendable outputs
hashX = hashX_from_script(txout.pk_script)
hashX = script_hashX(txout.pk_script)
if hashX:
append_hashX(hashX)
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
put_utxo(tx_hash + s_pack('<H', idx),
hashX + tx_numb + s_pack('<Q', txout.value))
append_hashX_by_tx(hashXs)
append_hashXs(hashXs)
update_touched(hashXs)
tx_num += 1
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
self.tx_count = tx_num
self.db.tx_counts.append(tx_num)
@@ -488,16 +489,20 @@ class BlockProcessor:
# Use local vars for speed in the loops
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 12 + HASHX_LEN
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
hashX = self.coin.hashX_from_script(txout.pk_script)
hashX = script_hashX(txout.pk_script)
if hashX:
cache_value = self.spend_utxo(tx_hash, idx)
self.touched.add(cache_value[:-12])
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-12])
# Restore the inputs
for txin in reversed(tx.inputs):
@@ -505,8 +510,9 @@ class BlockProcessor:
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
self.touched.add(undo_item[:-12])
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
undo_item)
touched.add(undo_item[:-12])
assert n == 0
self.tx_count -= len(txs)
@@ -750,6 +756,8 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.block_notify = asyncio.Event()
self.block_notify.set()
def advance_blocks(self, blocks):
self.sql.begin()
@@ -764,11 +772,10 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
def advance_txs(self, height, txs, header, block_hash):
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
self.timer.show(height=height)
+200 -398
View File
@@ -13,407 +13,209 @@ STREAM_TYPES = {
'model': 6
}
# 9/21/2020
MOST_USED_TAGS = {
"gaming",
"people & blogs",
"entertainment",
"music",
"pop culture",
"education",
"technology",
"blockchain",
"news",
"funny",
"science & technology",
"learning",
"gameplay",
"news & politics",
"comedy",
"bitcoin",
"beliefs",
"nature",
"art",
"economics",
"film & animation",
"lets play",
"games",
"sports",
"howto & style",
"game",
"cryptocurrency",
"playstation 4",
"automotive",
"crypto",
"mature",
"sony interactive entertainment",
"walkthrough",
"tutorial",
"video game",
"weapons",
"playthrough",
"pc",
"anime",
"how to",
"btc",
"fun",
"ethereum",
"food",
"travel & events",
"minecraft",
"science",
"autos & vehicles",
"play",
"politics",
"commentary",
"twitch",
"ps4live",
"love",
"ps4",
"nonprofits & activism",
"ps4share",
"fortnite",
"xbox",
"porn",
"video games",
"trump",
"español",
"money",
"music video",
"nintendo",
"movie",
"coronavirus",
"donald trump",
"steam",
"trailer",
"android",
"podcast",
"xbox one",
"survival",
"audio",
"linux",
"travel",
"funny moments",
"litecoin",
"animation",
"gamer",
"lets",
"playstation",
"bitcoin news",
"history",
"xxx",
"fox news",
"dance",
"god",
"adventure",
"liberal",
"2020",
"horror",
"government",
"freedom",
"reaction",
"meme",
"photography",
"truth",
"health",
"lbry",
"family",
"online",
"eth",
"crypto news",
"diy",
"trading",
"gold",
"memes",
"world",
"space",
"lol",
"covid-19",
"rpg",
"humor",
"democrat",
"film",
"call of duty",
"tech",
"religion",
"conspiracy",
"rap",
"cnn",
"hangoutsonair",
"unboxing",
"fiction",
"conservative",
"cars",
"hoa",
"epic",
"programming",
"progressive",
"cryptocurrency news",
"classical",
"jesus",
"movies",
"book",
"ps3",
"republican",
"fitness",
"books",
"multiplayer",
"animals",
"pokemon",
"bitcoin price",
"facebook",
"sharefactory",
"criptomonedas",
"cod",
"bible",
"business",
"stream",
"comics",
"how",
"fail",
"nsfw",
"new music",
"satire",
"pets & animals",
"computer",
"classical music",
"indie",
"musica",
"msnbc",
"fps",
"mod",
"sport",
"sony",
"ripple",
"auto",
"rock",
"marvel",
"complete",
"mining",
"political",
"mobile",
"pubg",
"hip hop",
"flat earth",
"xbox 360",
"reviews",
"vlogging",
"latest news",
"hack",
"tarot",
"iphone",
"media",
"cute",
"christian",
"free speech",
"trap",
"war",
"remix",
"ios",
"xrp",
"spirituality",
"song",
"league of legends",
"cat"
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
def normalize_tag(tag):
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
COMMON_TAGS = {
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
INDEXED_LANGUAGES = [
'none',
'en',
'aa',
'ab',
'ae',
'af',
'ak',
'am',
'an',
'ar',
'as',
'av',
'ay',
'az',
'ba',
'be',
'bg',
'bh',
'bi',
'bm',
'bn',
'bo',
'br',
'bs',
'ca',
'ce',
'ch',
'co',
'cr',
'cs',
'cu',
'cv',
'cy',
'da',
'de',
'dv',
'dz',
'ee',
'el',
'eo',
'es',
'et',
'eu',
'fa',
'ff',
'fi',
'fj',
'fo',
'fr',
'fy',
'ga',
'gd',
'gl',
'gn',
'gu',
'gv',
'ha',
'he',
'hi',
'ho',
'hr',
'ht',
'hu',
'hy',
'hz',
'ia',
'id',
'ie',
'ig',
'ii',
'ik',
'io',
'is',
'it',
'iu',
'ja',
'jv',
'ka',
'kg',
'ki',
'kj',
'kk',
'kl',
'km',
'kn',
'ko',
'kr',
'ks',
'ku',
'kv',
'kw',
'ky',
'la',
'lb',
'lg',
'li',
'ln',
'lo',
'lt',
'lu',
'lv',
'mg',
'mh',
'mi',
'mk',
'ml',
'mn',
'mr',
'ms',
'mt',
'my',
'na',
'nb',
'nd',
'ne',
'ng',
'nl',
'nn',
'no',
'nr',
'nv',
'ny',
'oc',
'oj',
'om',
'or',
'os',
'pa',
'pi',
'pl',
'ps',
'pt',
'qu',
'rm',
'rn',
'ro',
'ru',
'rw',
'sa',
'sc',
'sd',
'se',
'sg',
'si',
'sk',
'sl',
'sm',
'sn',
'so',
'sq',
'sr',
'ss',
'st',
'su',
'sv',
'sw',
'ta',
'te',
'tg',
'th',
'ti',
'tk',
'tl',
'tn',
'to',
'tr',
'ts',
'tt',
'tw',
'ty',
'ug',
'uk',
'ur',
'uz',
've',
'vi',
'vo',
'wa',
'wo',
'xh',
'yi',
'yo',
'za',
'zh',
'zu'
]
+5 -18
View File
@@ -18,7 +18,7 @@ from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs, Censor
from lbry.wallet import Ledger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .full_text_search import FTS_ORDER_BY
@@ -39,7 +39,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
'tx_position', 'channel_join', 'reposted',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
@@ -96,8 +96,8 @@ class ReaderState:
def get_resolve_censor(self) -> Censor:
return Censor(self.blocked_streams, self.blocked_channels)
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels, limit_claims_per_channel)
def get_search_censor(self) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
@@ -421,13 +421,12 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
total = None
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
if not constraints.pop('no_totals', False):
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
context = ctx.get()
search_censor = context.get_search_censor(limit_claims_per_channel)
search_censor = context.get_search_censor()
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -536,18 +535,6 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
AND tag IN ({values})
)
"""
elif attr == 'language':
indexed_languages = any_items & set(INDEXED_LANGUAGES)
if indexed_languages:
any_items -= indexed_languages
for language in indexed_languages:
any_queries[f'#_any_common_languages_{language}'] = f"""
EXISTS(
SELECT 1 FROM language INDEXED BY language_{language}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
AND language = '{language}'
)
"""
if any_items:
+269 -323
View File
@@ -1,17 +1,18 @@
"""
AR-like trending with a delayed effect and a faster
decay rate for high valued claims.
Delayed AR with variable decay rate.
The spike height function is also simpler.
"""
import math
import copy
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold, in LBC (higher -> less DB writing)
WHALE_THRESHOLD = 10000.0
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
@@ -37,7 +38,6 @@ def install(connection):
Install the trending algorithm.
"""
check_trending_values(connection)
trending_data.initialise(connection.cursor())
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
@@ -46,6 +46,7 @@ def install(connection):
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
@@ -59,304 +60,19 @@ def check_trending_values(connection):
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True,
end="")
print("Resetting some columns. This might take a while...", flush=True, end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def trending_log(s):
def spike_height(x, x_old):
"""
Log a string to the log file
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
def trending_unit(height):
"""
Return the trending score unit at a given height.
"""
# Round to the beginning of a SAVE_INTERVAL batch of blocks.
_height = height - (height % SAVE_INTERVAL)
return 1.0/DECAY**(height % RENORM_INTERVAL)
class TrendingDB:
"""
An in-memory database of trending scores
"""
def __init__(self):
self.conn = apsw.Connection(":memory:")
self.cursor = self.conn.cursor()
self.initialised = False
self.write_needed = set()
def execute(self, query, *args, **kwargs):
return self.cursor.execute(query, *args, **kwargs)
def executemany(self, query, *args, **kwargs):
return self.cursor.executemany(query, *args, **kwargs)
def begin(self):
self.execute("BEGIN;")
def commit(self):
self.execute("COMMIT;")
def initialise(self, db):
"""
Pass in claims.db
"""
if self.initialised:
return
trending_log("Initialising trending database...")
# The need for speed
self.execute("PRAGMA JOURNAL_MODE=OFF;")
self.execute("PRAGMA SYNCHRONOUS=0;")
self.begin()
# Create the tables
self.execute("""
CREATE TABLE IF NOT EXISTS claims
(claim_hash BYTES PRIMARY KEY,
lbc REAL NOT NULL DEFAULT 0.0,
trending_score REAL NOT NULL DEFAULT 0.0)
WITHOUT ROWID;""")
self.execute("""
CREATE TABLE IF NOT EXISTS spikes
(id INTEGER PRIMARY KEY,
claim_hash BYTES NOT NULL,
height INTEGER NOT NULL,
mass REAL NOT NULL,
FOREIGN KEY (claim_hash)
REFERENCES claims (claim_hash));""")
# Clear out any existing data
self.execute("DELETE FROM claims;")
self.execute("DELETE FROM spikes;")
# Create indexes
self.execute("CREATE INDEX idx1 ON spikes (claim_hash, height, mass);")
self.execute("CREATE INDEX idx2 ON spikes (claim_hash, height, mass DESC);")
self.execute("CREATE INDEX idx3 on claims (lbc DESC, claim_hash, trending_score);")
# Import data from claims.db
for row in db.execute("""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc,
trending_mixed
FROM claim;
"""):
self.execute("INSERT INTO claims VALUES (?, ?, ?);", row)
self.commit()
self.initialised = True
trending_log("done.\n")
def apply_spikes(self, height):
"""
Apply spikes that are due. This occurs inside a transaction.
"""
spikes = []
unit = trending_unit(height)
for row in self.execute("""
SELECT SUM(mass), claim_hash FROM spikes
WHERE height = ?
GROUP BY claim_hash;
""", (height, )):
spikes.append((row[0]*unit, row[1]))
self.write_needed.add(row[1])
self.executemany("""
UPDATE claims
SET trending_score = (trending_score + ?)
WHERE claim_hash = ?;
""", spikes)
self.execute("DELETE FROM spikes WHERE height = ?;", (height, ))
def decay_whales(self, height):
"""
Occurs inside transaction.
"""
if height % SAVE_INTERVAL != 0:
return
whales = self.execute("""
SELECT trending_score, lbc, claim_hash
FROM claims
WHERE lbc >= ?;
""", (WHALE_THRESHOLD, )).fetchall()
whales2 = []
for whale in whales:
trending, lbc, claim_hash = whale
# Overall multiplication factor for decay rate
# At WHALE_THRESHOLD, this is 1
# At 10*WHALE_THRESHOLD, it is 3
decay_rate_factor = 1.0 + 2.0*math.log10(lbc/WHALE_THRESHOLD)
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# Decay
trending *= factor
whales2.append((trending, claim_hash))
self.write_needed.add(claim_hash)
self.executemany("UPDATE claims SET trending_score=? WHERE claim_hash=?;",
whales2)
def renorm(self, height):
"""
Renormalise trending scores. Occurs inside a transaction.
"""
if height % RENORM_INTERVAL == 0:
threshold = 1.0E-3/DECAY_PER_RENORM
for row in self.execute("""SELECT claim_hash FROM claims
WHERE ABS(trending_score) >= ?;""",
(threshold, )):
self.write_needed.add(row[0])
self.execute("""UPDATE claims SET trending_score = ?*trending_score
WHERE ABS(trending_score) >= ?;""",
(DECAY_PER_RENORM, threshold))
def write_to_claims_db(self, db, height):
"""
Write changed trending scores to claims.db.
"""
if height % SAVE_INTERVAL != 0:
return
rows = self.execute(f"""
SELECT trending_score, claim_hash
FROM claims
WHERE claim_hash IN
({','.join('?' for _ in self.write_needed)});
""", self.write_needed).fetchall()
db.executemany("""UPDATE claim SET trending_mixed = ?
WHERE claim_hash = ?;""", rows)
# Clear list of claims needing to be written to claims.db
self.write_needed = set()
def update(self, db, height, recalculate_claim_hashes):
"""
Update trending scores.
Input is a cursor to claims.db, the block height, and the list of
claims that changed.
"""
assert self.initialised
self.begin()
self.renorm(height)
# Fetch changed/new claims from claims.db
for row in db.execute(f"""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
claim_hash, lbc = row
# Insert into trending db if it does not exist
self.execute("""
INSERT INTO claims (claim_hash)
VALUES (?)
ON CONFLICT (claim_hash) DO NOTHING;""",
(claim_hash, ))
# See if it was an LBC change
old = self.execute("SELECT * FROM claims WHERE claim_hash=?;",
(claim_hash, )).fetchone()
lbc_old = old[1]
# Save new LBC value into trending db
self.execute("UPDATE claims SET lbc = ? WHERE claim_hash = ?;",
(lbc, claim_hash))
if lbc > lbc_old:
# Schedule a future spike
delay = min(int((lbc + 1E-8)**0.4), HALF_LIFE)
spike = (claim_hash, height + delay, spike_mass(lbc, lbc_old))
self.execute("""INSERT INTO spikes
(claim_hash, height, mass)
VALUES (?, ?, ?);""", spike)
elif lbc < lbc_old:
# Subtract from future spikes
penalty = spike_mass(lbc_old, lbc)
spikes = self.execute("""
SELECT * FROM spikes
WHERE claim_hash = ?
ORDER BY height ASC, mass DESC;
""", (claim_hash, )).fetchall()
for spike in spikes:
spike_id, mass = spike[0], spike[3]
if mass > penalty:
# The entire penalty merely reduces this spike
self.execute("UPDATE spikes SET mass=? WHERE id=?;",
(mass - penalty, spike_id))
penalty = 0.0
else:
# Removing this spike entirely accounts for some (or
# all) of the penalty, then move on to other spikes
self.execute("DELETE FROM spikes WHERE id=?;",
(spike_id, ))
penalty -= mass
# If penalty remains, that's a negative spike to be applied
# immediately.
if penalty > 0.0:
self.execute("""
INSERT INTO spikes (claim_hash, height, mass)
VALUES (?, ?, ?);""",
(claim_hash, height, -penalty))
self.apply_spikes(height)
self.decay_whales(height)
self.commit()
self.write_to_claims_db(db, height)
# The "global" instance to work with
# pylint: disable=C0103
trending_data = TrendingDB()
def spike_mass(x, x_old):
"""
Compute the mass of a trending spike (normed - constant units).
x_old = old LBC value
x = new LBC value
Compute the size of a trending spike (normed - constant units).
"""
# Sign of trending spike
@@ -373,16 +89,155 @@ def spike_mass(x, x_old):
return sign*mag
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log(f"Skipping trending calculations at block {height}.\n")
return
start = time.time()
trending_log(f"Calculating variable_decay trending at block {height}.\n")
trending_data.update(db, height, recalculate_claim_hashes)
end = time.time()
trending_log(f"Trending operations took {end - start} seconds.\n\n")
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
def test_trending():
"""
@@ -405,12 +260,12 @@ def test_trending():
COMMIT;
""")
# Initialise trending data before anything happens with the claims
trending_data.initialise(db)
# Insert initial states of claims
everything = {"huge_whale": 0.01, "medium_whale": 0.01, "small_whale": 0.01,
"huge_whale_botted": 0.01, "minnow": 0.01}
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
@@ -422,17 +277,13 @@ def test_trending():
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
# Process block zero
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for row in trending_data.execute("""
SELECT claim_hash, trending_score
FROM claims;
"""):
trajectories[row[0]] = [row[1]/trending_unit(height)]
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
# Main loop
for height in range(1, 1000):
@@ -461,24 +312,119 @@ def test_trending():
# Call run()
run(db, height, height, everything.keys())
# Append current trending scores to trajectories
for row in db.execute("""
SELECT claim_hash, trending_mixed
FROM claim;
"""):
trajectories[row[0]].append(row[1]/trending_unit(height))
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trajectories:
for key in trending_data.claims:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
+4 -31
View File
@@ -6,6 +6,7 @@ from decimal import Decimal
from collections import namedtuple
from multiprocessing import Manager
from binascii import unhexlify
from lbry.wallet.server.leveldb import LevelDB
from lbry.wallet.server.util import class_logger
from lbry.wallet.database import query, constraints_to_sql
@@ -18,7 +19,7 @@ from lbry.wallet.server.db.canonical import register_canonical_functions
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -116,15 +117,6 @@ class SQLDB:
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_LANGUAGE_TABLE = """
create table if not exists language (
language text not null,
claim_hash bytes not null,
height integer not null
);
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
"""
CREATE_CLAIMTRIE_TABLE = """
create table if not exists claimtrie (
normalized text primary key,
@@ -182,18 +174,12 @@ class SQLDB:
for tag_value, tag_key in COMMON_TAGS.items()
)
LANGUAGE_INDEXES = '\n'.join(
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
for language in INDEXED_LANGUAGES
)
CREATE_TABLES_QUERY = (
CREATE_CLAIM_TABLE +
CREATE_FULL_TEXT_SEARCH +
CREATE_SUPPORT_TABLE +
CREATE_CLAIMTRIE_TABLE +
CREATE_TAG_TABLE +
CREATE_LANGUAGE_TABLE
CREATE_TAG_TABLE
)
def __init__(
@@ -319,7 +305,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags, languages = set(), [], {}, {}
claim_hashes, claims, tags = set(), [], {}
for txo in txos:
tx = txo.tx_ref.tx
@@ -330,13 +316,6 @@ class SQLDB:
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
continue
language = 'none'
try:
if txo.claim.is_stream and txo.claim.stream.languages:
language = txo.claim.stream.languages[0].language
except:
pass
claim_hash = txo.claim_hash
claim_hashes.add(claim_hash)
claim_record = {
@@ -394,8 +373,6 @@ class SQLDB:
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
@@ -406,10 +383,6 @@ class SQLDB:
self.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
if languages:
self.executemany(
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
)
return claims
+95 -200
View File
@@ -18,7 +18,6 @@ from asyncio import sleep
from bisect import bisect_right
from collections import namedtuple
from functools import partial
from binascii import unhexlify, hexlify
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
@@ -33,13 +32,6 @@ from lbry.wallet.server.history import History
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
HEADER_PREFIX = b'H'
TX_COUNT_PREFIX = b'T'
TX_HASH_PREFIX = b'X'
TX_PREFIX = b'B'
TX_NUM_PREFIX = b'N'
BLOCK_HASH_PREFIX = b'C'
@attr.s(slots=True)
@@ -47,8 +39,7 @@ class FlushData:
height = attr.ib()
tx_count = attr.ib()
headers = attr.ib()
block_hashes = attr.ib()
block_txs = attr.ib()
block_tx_hashes = attr.ib()
# The following are flushed to the UTXO DB if undo_infos is not None
undo_infos = attr.ib()
adds = attr.ib()
@@ -72,7 +63,15 @@ class LevelDB:
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.executor = None
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset
self.header_len = self.coin.static_header_len
else:
self.header_offset = self.dynamic_header_offset
self.header_len = self.dynamic_header_len
self.logger.info(f'switching current directory to {env.db_dir}')
@@ -88,64 +87,51 @@ class LevelDB:
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.headers_db = None
self.tx_db = None
path = partial(os.path.join, self.env.db_dir)
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
path('meta/headers_offsets'), 2, 16000000)
async def _read_tx_counts(self):
if self.tx_counts is not None:
return
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
def get_counts():
return tuple(
util.unpack_be_uint64(tx_count)
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
)
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
size = (self.db_height + 1) * 4
tx_counts = self.tx_counts_file.read(0, size)
assert len(tx_counts) == size
self.tx_counts = array.array('I', tx_counts)
if self.tx_counts:
assert self.db_tx_count == self.tx_counts[-1], \
f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)"
assert self.db_tx_count == self.tx_counts[-1]
else:
assert self.db_tx_count == 0
async def _open_dbs(self, for_sync, compacting):
if self.executor is None:
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
coin_path = os.path.join(self.env.db_dir, 'COIN')
if not os.path.isfile(coin_path):
with util.open_file(coin_path, create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
assert self.headers_db is None
self.headers_db = self.db_class('headers', for_sync)
if self.headers_db.is_new:
self.logger.info('created new headers db')
self.logger.info(f'opened headers DB (for sync: {for_sync})')
assert self.tx_db is None
self.tx_db = self.db_class('tx', for_sync)
if self.tx_db.is_new:
self.logger.info('created new tx db')
self.logger.info(f'opened tx DB (for sync: {for_sync})')
assert self.utxo_db is None
# First UTXO DB
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new utxo db')
self.logger.info(f'opened utxo db (for sync: {for_sync})')
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
with util.open_file(coin_path, create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file.write(0, bytes(8))
else:
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
self.read_utxo_state()
# Then history DB
self.utxo_flush_count = self.history.open_db(
self.db_class, for_sync, self.utxo_flush_count, compacting
)
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
self.utxo_flush_count,
compacting)
self.clear_excess_undo_info()
# Read TX counts (requires meta directory)
@@ -154,10 +140,7 @@ class LevelDB:
def close(self):
self.utxo_db.close()
self.history.close_db()
self.headers_db.close()
self.tx_db.close()
self.executor.shutdown(wait=True)
self.executor = None
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -169,28 +152,18 @@ class LevelDB:
synchronization. When serving clients we want the open files for
serving network connections.
"""
self.logger.info("opened for sync")
await self._open_dbs(True, False)
async def open_for_serving(self):
"""Open the databases for serving. If they are already open they are
closed first.
"""
self.logger.info('closing DBs to re-open for serving')
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
if self.headers_db:
self.headers_db.close()
self.headers_db = None
if self.tx_db:
self.tx_db.close()
self.tx_db = None
await self._open_dbs(False, False)
self.logger.info("opened for serving")
# Header merkle cache
@@ -212,7 +185,7 @@ class LevelDB:
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.block_txs
assert not flush_data.block_tx_hashes
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
@@ -270,43 +243,36 @@ class LevelDB:
"""
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_txs) == len(flush_data.headers)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
assert len(
b''.join(hashes for hashes, _ in flush_data.block_txs)
) // 32 == flush_data.tx_count - prior_tx_count
hashes = b''.join(flush_data.block_tx_hashes)
flush_data.block_tx_hashes.clear()
assert len(hashes) % 32 == 0
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
start_time = time.perf_counter()
start_time = time.time()
height_start = self.fs_height + 1
tx_num = prior_tx_count
offset = self.header_offset(height_start)
self.headers_file.write(offset, b''.join(flush_data.headers))
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
flush_data.headers.clear()
for header, block_hash, (tx_hashes, txs) in zip(
flush_data.headers, flush_data.block_hashes, flush_data.block_txs):
tx_count = self.tx_counts[height_start]
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
self.tx_db.put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
self.tx_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
self.tx_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
self.tx_db.put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
self.tx_db.put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
tx_num += 1
offset += 32
flush_data.block_txs.clear()
flush_data.block_hashes.clear()
offset = height_start * self.tx_counts.itemsize
self.tx_counts_file.write(offset,
self.tx_counts[height_start:].tobytes())
offset = prior_tx_count * 32
self.hashes_file.write(offset, hashes)
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
flush_data.headers.clear()
elapsed = time.perf_counter() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
if self.utxo_db.for_sync:
elapsed = time.time() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
def flush_history(self):
self.history.flush()
@@ -365,7 +331,7 @@ class LevelDB:
def flush_backup(self, flush_data, touched):
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
assert not flush_data.headers
assert not flush_data.block_txs
assert not flush_data.block_tx_hashes
assert flush_data.height < self.db_height
self.history.assert_flushed()
@@ -384,6 +350,28 @@ class LevelDB:
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
def fs_update_header_offsets(self, offset_start, height_start, headers):
if self.coin.STATIC_BLOCK_HEADERS:
return
offset = offset_start
offsets = []
for h in headers:
offset += len(h)
offsets.append(pack("<Q", offset))
# For each header we get the offset of the next header, hence we
# start writing from the next height
pos = (height_start + 1) * 8
self.headers_offsets_file.write(pos, b''.join(offsets))
def dynamic_header_offset(self, height):
assert not self.coin.STATIC_BLOCK_HEADERS
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
return offset
def dynamic_header_len(self, height):
return self.dynamic_header_offset(height + 1)\
- self.dynamic_header_offset(height)
def backup_fs(self, height, tx_count):
"""Back up during a reorg. This just updates our pointers."""
self.fs_height = height
@@ -415,13 +403,9 @@ class LevelDB:
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(
self.headers_db.iterator(
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
include_key=False
)
), disk_count
offset = self.header_offset(start_height)
size = self.header_offset(start_height + disk_count) - offset
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
@@ -434,54 +418,9 @@ class LevelDB:
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.tx_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
tx_hash = self.hashes_file.read(tx_num * 32, 32)
return tx_hash, tx_height
def _fs_transactions(self, txids):
def _iter_transactions():
block_txs = {}
branch_and_root = self.merkle.branch_and_root
tx_iterator = self.tx_db.iterator
tx_counts = self.tx_counts
tx_db_get = self.tx_db.get
unpack_be_uint64 = util.unpack_be_uint64
pack_be_uint64 = util.pack_be_uint64
for tx_hash in txids:
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
if tx_num is not None:
tx_num = unpack_be_uint64(tx_num)
tx_height = bisect_right(tx_counts, tx_num)
else:
yield tx_hash, (None, {'block_height': -1})
continue
if tx_height >= self.db_height:
yield tx_hash, (None, {'block_height': -1})
continue
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
if tx_height not in block_txs:
block_txs[tx_height] = list(tx_iterator(
start=TX_HASH_PREFIX + pack_be_uint64(tx_counts[tx_height - 1]),
stop=None if tx_height + 1 == len(tx_counts) else
TX_HASH_PREFIX + pack_be_uint64(tx_counts[tx_height]),
include_key=False
))
tx_pos = tx_counts[tx_height] - tx_num
branch, root = branch_and_root(block_txs[tx_height], tx_pos)
merkle = {
'block_height': tx_height,
'merkle': [hash_to_hex_str(hash) for hash in branch],
'pos': tx_pos
}
yield tx_hash, (None if not tx else tx.hex(), merkle)
return {
_tx_hash: _val for (_tx_hash, _val) in _iter_transactions()
}
async def fs_transactions(self, txids):
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
async def fs_block_hashes(self, height, count):
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
@@ -489,8 +428,9 @@ class LevelDB:
offset = 0
headers = []
for n in range(count):
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
offset += self.coin.BASIC_HEADER_SIZE
hlen = self.header_len(height + n)
headers.append(headers_concat[offset:offset + hlen])
offset += hlen
return [self.coin.header_hash(header) for header in headers]
@@ -501,54 +441,14 @@ class LevelDB:
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
"""
# def read_history():
# hashx_history = []
# for key, hist in self.history.db.iterator(prefix=hashX):
# a = array.array('I')
# a.frombytes(hist)
# for tx_num in a:
# tx_height = bisect_right(self.tx_counts, tx_num)
# if tx_height > self.db_height:
# tx_hash = None
# else:
# tx_hash = self.tx_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
#
# hashx_history.append((tx_hash, tx_height))
# if limit and len(hashx_history) >= limit:
# return hashx_history
# return hashx_history
def iter_tx_heights():
db_height = self.db_height
tx_counts = self.tx_counts
tx_db_get = self.tx_db.get
pack_be_uint64 = util.pack_be_uint64
cnt = 0
for key, hist in self.history.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(tx_counts, tx_num)
if tx_height > db_height:
yield None, tx_height
return
yield tx_db_get(TX_HASH_PREFIX + pack_be_uint64(tx_num)), tx_height
cnt += 1
if limit and cnt >= limit:
return
if limit and cnt >= limit:
return
def read_history():
return [
(tx_num, tx_height) for (tx_num, tx_height) in iter_tx_heights()
]
tx_nums = list(self.history.get_txnums(hashX, limit))
fs_tx_hash = self.fs_tx_hash
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if not history or history[-1][0] is not None:
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
@@ -574,20 +474,16 @@ class LevelDB:
batch_put(self.undo_key(height), b''.join(undo_info))
def raw_block_prefix(self):
return 'block'
return 'meta/block'
def raw_block_path(self, height):
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
async def read_raw_block(self, height):
def read_raw_block(self, height):
"""Returns a raw block read from disk. Raises FileNotFoundError
if the block isn't on-disk."""
def read():
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
def write_raw_block(self, block, height):
"""Write a raw block to disk."""
@@ -706,14 +602,13 @@ class LevelDB:
utxos = []
utxos_append = utxos.append
s_unpack = unpack
fs_tx_hash = self.fs_tx_hash
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
value, = unpack('<Q', db_value)
tx_hash, height = fs_tx_hash(tx_num)
tx_hash, height = self.fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
return utxos
+8 -29
View File
@@ -13,7 +13,6 @@ import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
from prometheus_client import Histogram
import attr
@@ -80,16 +79,6 @@ class MemPoolAPI(ABC):
daemon's height at the time the mempool was obtained."""
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
mempool_process_time_metric = Histogram(
"processed_mempool", "Time to process mempool and notify touched addresses",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
class MemPool:
"""Representation of the daemon's mempool.
@@ -104,7 +93,7 @@ class MemPool:
hashXs: hashX -> set of all hashes of txs touching the hashX
"""
def __init__(self, coin, api, refresh_secs=1.0, log_status_secs=120.0):
def __init__(self, coin, api, refresh_secs=5.0, log_status_secs=120.0):
assert isinstance(api, MemPoolAPI)
self.coin = coin
self.api = api
@@ -118,7 +107,6 @@ class MemPool:
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
self.mempool_process_time_metric = mempool_process_time_metric
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
@@ -219,19 +207,24 @@ class MemPool:
async def _refresh_hashes(self, synchronized_event):
"""Refresh our view of the daemon's mempool."""
while True:
start = time.perf_counter()
height = self.api.cached_height()
hex_hashes = await self.api.mempool_hashes()
if height != await self.api.height():
continue
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
async with self.lock:
start = time.perf_counter()
touched = await self._process_mempool(hashes)
duration = time.perf_counter() - start
if duration >= 1:
self.logger.info("processed mempool in %s", duration)
synchronized_event.set()
synchronized_event.clear()
start = time.perf_counter()
await self.api.on_mempool(touched, height)
duration = time.perf_counter() - start
self.mempool_process_time_metric.observe(duration)
if duration >= 1:
self.logger.info("sent history notifications in %s", duration)
try:
# we wait up to `refresh_secs` but go early if a broadcast happens (which triggers wakeup event)
await asyncio.wait_for(self.wakeup.wait(), timeout=self.refresh_secs)
@@ -389,17 +382,3 @@ class MemPool:
if hX == hashX:
utxos.append(UTXO(-1, pos, tx_hash, 0, value))
return utxos
def get_mempool_height(self, tx_hash):
# Height Progression
# -2: not broadcast
# -1: in mempool but has unconfirmed inputs
# 0: in mempool and all inputs confirmed
# +num: confirmed in a specific block (height)
if tx_hash not in self.txs:
return -2
tx = self.txs[tx_hash]
unspent_inputs = sum(1 if hash in self.txs else 0 for hash, idx in tx.prevouts)
if unspent_inputs:
return -1
return 0
+1 -1
View File
@@ -63,7 +63,7 @@ class Merkle:
raise TypeError('index must be an integer')
# This also asserts hashes is not empty
if not 0 <= index < len(hashes):
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
raise ValueError('index out of range')
natural_length = self.branch_length(len(hashes))
if length is None:
length = natural_length
+130 -89
View File
@@ -17,7 +17,7 @@ from asyncio import Event, sleep
from collections import defaultdict
from functools import partial
from binascii import hexlify, unhexlify
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
@@ -123,7 +123,6 @@ HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class SessionManager:
"""Holds global state about all sessions."""
@@ -141,11 +140,7 @@ class SessionManager:
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
labelnames=("method", "version"))
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
@@ -165,18 +160,6 @@ class SessionManager:
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
address_history_metric = Histogram(
"address_history", "Time to fetch an address history",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
notifications_in_flight_metric = Gauge(
"notifications_in_flight", "Count of notifications in flight",
namespace=NAMESPACE
)
notifications_sent_metric = Histogram(
"notifications_sent", "Time to send an address notification",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
shutdown_event: asyncio.Event):
@@ -619,6 +602,7 @@ class SessionManager:
async def broadcast_transaction(self, raw_tx):
hex_hash = await self.daemon.broadcast_transaction(raw_tx)
self.mempool.wakeup.set()
self.txs_sent += 1
return hex_hash
@@ -931,48 +915,64 @@ class LBRYElectrumX(SessionBase):
def sub_count(self):
return len(self.hashX_subs)
UGLY_COUNT = 0
async def notify(self, touched, height_changed):
"""Notify the client about changes to touched addresses (from mempool
updates or new blocks) and height.
"""
self.UGLY_COUNT += 1
if height_changed and self.subscribe_headers:
args = (await self.subscribe_headers_result(), )
if not (await self.send_notification('blockchain.headers.subscribe', args)):
return
async def send_history_notification(alias, hashX):
start = time.perf_counter()
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
start = time.perf_counter()
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
await self.send_notification('blockchain.headers.subscribe', args)
except asyncio.TimeoutError:
self.logger.info("timeout sending headers notification to %s", self.peer_address_str(for_log=True))
self.abort()
return
touched = touched.intersection(self.hashX_subs)
if touched or (height_changed and self.mempool_statuses):
notified = set()
mempool_addrs = tuple(self.mempool_statuses.keys())
changed = {}
for hashX in touched:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
for hashX in mempool_addrs:
if hashX not in notified:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
if self.UGLY_COUNT == 25:
print('sleeping for ', hashX)
if not self.bp.block_notify.is_set():
await self.bp.block_notify.wait()
await asyncio.sleep(3)
status = await self.address_status(hashX)
changed[alias] = status
if touched:
es = '' if len(touched) == 1 else 'es'
self.logger.info(f'notified {len(notified)} mempool/{len(touched):,d} touched address{es}')
# Check mempool hashXs - the status is a function of the
# confirmed state of other transactions. Note: we cannot
# iterate over mempool_statuses as it changes size.
for hashX in tuple(self.mempool_statuses):
# Items can be evicted whilst await-ing status; False
# ensures such hashXs are notified
old_status = self.mempool_statuses.get(hashX, False)
status = await self.address_status(hashX)
if status != old_status:
alias = self.hashX_subs[hashX]
changed[alias] = status
for alias, status in changed.items():
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
start = time.perf_counter()
if not self.bp.block_notify.is_set():
await self.bp.block_notify.wait()
t = asyncio.create_task(self.send_notification(method, (alias, status)))
t.add_done_callback(lambda _: self.logger.warning("sent notification to %s in %s", alias, time.perf_counter() - start))
if changed:
es = '' if len(changed) == 1 else 'es'
self.logger.info(f'notified of {len(changed):,d} address{es}')
def get_metrics_or_placeholder_for_api(self, query_name):
""" Do not hold on to a reference to the metrics
@@ -1049,12 +1049,7 @@ class LBRYElectrumX(SessionBase):
async def claimtrie_resolve(self, *urls):
if urls:
count = len(urls)
try:
self.session_mgr.urls_to_resolve_count_metric.inc(count)
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
finally:
self.session_mgr.resolved_url_count_metric.inc(count)
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
async def get_server_height(self):
return self.bp.height
@@ -1206,6 +1201,7 @@ class LBRYElectrumX(SessionBase):
self.mempool_statuses[hashX] = status
else:
self.mempool_statuses.pop(hashX, None)
return status
async def hashX_listunspent(self, hashX):
@@ -1227,7 +1223,10 @@ class LBRYElectrumX(SessionBase):
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
self.hashX_subs.pop(hashX, None)
try:
del self.hashX_subs[hashX]
except ValueError:
pass
def address_to_hashX(self, address):
try:
@@ -1518,7 +1517,6 @@ class LBRYElectrumX(SessionBase):
try:
hex_hash = await self.session_mgr.broadcast_transaction(raw_tx)
self.txs_sent += 1
self.mempool.wakeup.set()
self.logger.info(f'sent tx: {hex_hash}')
return hex_hash
except DaemonError as e:
@@ -1529,45 +1527,51 @@ class LBRYElectrumX(SessionBase):
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
return (await self.transaction_get_batch(tx_hash))[tx_hash]
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon_request('deserialised_block', block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
async def transaction_get_batch(self, *tx_hashes):
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = await self.db.fs_transactions(tx_hashes)
needed_merkles = {}
batch_result = {}
height = None
block_hash = None
block = None
for tx_hash in tx_hashes:
if tx_hash in batch_result and batch_result[tx_hash][0]:
continue
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if block_hash:
block = await self.daemon.deserialised_block(block_hash)
height = block['height']
try:
pos = block['tx'].index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
if height is None:
if 'blockhash' in tx_info:
block_hash = tx_info['blockhash']
block = await self.daemon_request('deserialised_block', block_hash)
height = block['height']
else:
height = -1
if block_hash != tx_info.get('blockhash'):
raise RPCError(BAD_REQUEST, f'request contains a mix of transaction heights')
else:
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
def threaded_get_merkle():
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
batch_result[tx_hash] = raw_tx, {
'merkle': self._get_merkle_branch(block_txs, pos),
'pos': pos,
'block_height': block_height
}
if needed_merkles:
await asyncio.get_running_loop().run_in_executor(self.db.executor, threaded_get_merkle)
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
if not block_hash:
merkle = {'block_height': -1}
else:
try:
pos = block['tx'].index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
merkle = {
"merkle": self._get_merkle_branch(block['tx'], pos),
"pos": pos
}
batch_result[tx_hash] = [raw_tx, merkle]
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
@@ -1582,6 +1586,19 @@ class LBRYElectrumX(SessionBase):
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def _block_hash_and_tx_hashes(self, height):
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
the given height.
block_hash is a hexadecimal string, and tx_hashes is an
ordered list of hexadecimal strings.
"""
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon_request('deserialised_block', block_hash)
return block_hash, block['tx']
def _get_merkle_branch(self, tx_hashes, tx_pos):
"""Return a merkle branch to a transaction.
@@ -1601,11 +1618,35 @@ class LBRYElectrumX(SessionBase):
height: the height of the block it is in
"""
assert_tx_hash(tx_hash)
result = await self.transaction_get_batch(tx_hash)
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block at height {height:,d}')
return result[tx_hash][1]
f'block {block_hash} at height {height:,d}')
branch = self._get_merkle_branch(tx_hashes, pos)
return {"block_height": height, "merkle": branch, "pos": pos}
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
"""Return the txid and optionally a merkle proof, given
a block height and position in the block.
"""
tx_pos = non_negative_integer(tx_pos)
if merkle not in (True, False):
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
tx_hash = tx_hashes[tx_pos]
except IndexError:
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
f'block {block_hash} at height {height:,d}')
if merkle:
branch = self._get_merkle_branch(tx_hashes, tx_pos)
return {"tx_hash": tx_hash, "merkle": branch}
else:
return tx_hash
class LocalRPC(SessionBase):
+6 -4
View File
@@ -77,16 +77,18 @@ class LevelDB(Storage):
import plyvel
cls.module = plyvel
def open(self, name, create, lru_cache_size=None):
mof = 10000
def open(self, name, create):
mof = 512 if self.for_sync else 128
path = os.path.join(self.db_dir, name)
# Use snappy compression (the default)
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
self.db = self.module.DB(path, create_if_missing=create,
max_open_files=mof)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
self.iterator = self.db.iterator
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
self.write_batch = partial(self.db.write_batch, transaction=True,
sync=True)
class RocksDB(Storage):
+1 -3
View File
@@ -333,13 +333,11 @@ unpack_le_uint64_from = struct_le_Q.unpack_from
unpack_be_uint16_from = struct_be_H.unpack_from
unpack_be_uint32_from = struct_be_I.unpack_from
unpack_be_uint64 = lambda x: int.from_bytes(x, byteorder='big')
pack_le_int32 = struct_le_i.pack
pack_le_int64 = struct_le_q.pack
pack_le_uint16 = struct_le_H.pack
pack_le_uint32 = struct_le_I.pack
pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big')
pack_le_uint64 = struct_le_Q.pack
pack_be_uint16 = struct_be_H.pack
pack_be_uint32 = struct_be_I.pack
pack_byte = structB.pack
+18 -65
View File
@@ -19,9 +19,7 @@ from lbry.crypto.hash import hash160, sha256
from lbry.crypto.base58 import Base58
from lbry.schema.url import normalize_name
from lbry.schema.claim import Claim
from lbry.schema.base import Signable
from lbry.schema.purchase import Purchase
from lbry.schema.support import Support
from .script import InputScript, OutputScript
from .constants import COIN, NULL_HASH32
@@ -213,7 +211,7 @@ class Output(InputOutput):
'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input',
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims', '_signable'
'reposted_claim', 'claims',
)
def __init__(self, amount: int, script: OutputScript,
@@ -241,7 +239,6 @@ class Output(InputOutput):
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
self.reposted_claim: 'Output' = None # txo representing claim being reposted
self.claims: List['Output'] = None # resolved claims for collection
self._signable: Optional[Signable] = None
self.meta = {}
def update_annotations(self, annotated: 'Output'):
@@ -315,10 +312,6 @@ class Output(InputOutput):
def is_support(self) -> bool:
return self.script.is_support_claim
@property
def is_support_data(self) -> bool:
return self.script.is_support_claim_data
@property
def claim_hash(self) -> bytes:
if self.script.is_claim_name:
@@ -354,33 +347,9 @@ class Output(InputOutput):
def can_decode_claim(self):
try:
return self.claim
except Exception:
except: # pylint: disable=bare-except
return False
@property
def support(self) -> Support:
if self.is_support_data:
if not isinstance(self.script.values['support'], Support):
self.script.values['support'] = Support.from_bytes(self.script.values['support'])
return self.script.values['support']
raise ValueError('Only supports with data can be represented as Supports.')
@property
def can_decode_support(self):
try:
return self.support
except Exception:
return False
@property
def signable(self) -> Signable:
if self._signable is None:
if self.is_claim:
self._signable = self.claim
elif self.is_support_data:
self._signable = self.support
return self._signable
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
@@ -392,22 +361,22 @@ class Output(InputOutput):
return self.private_key is not None
def get_signature_digest(self, ledger):
if self.signable.unsigned_payload:
if self.claim.unsigned_payload:
pieces = [
Base58.decode(self.get_address(ledger)),
self.signable.unsigned_payload,
self.signable.signing_channel_hash[::-1]
self.claim.unsigned_payload,
self.claim.signing_channel_hash[::-1]
]
else:
pieces = [
self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]
return sha256(b''.join(pieces))
def get_encoded_signature(self):
signature = hexlify(self.signable.signature)
signature = hexlify(self.claim.signature)
r = int(signature[:int(len(signature)/2)], 16)
s = int(signature[int(len(signature)/2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
@@ -431,18 +400,18 @@ class Output(InputOutput):
def sign(self, channel: 'Output', first_input_id=None):
self.channel = channel
self.signable.signing_channel_hash = channel.claim_hash
self.claim.signing_channel_hash = channel.claim_hash
digest = sha256(b''.join([
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]))
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.claim.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.script.generate()
def clear_signature(self):
self.channel = None
self.signable.clear_signature()
self.claim.clear_signature()
async def generate_channel_private_key(self):
self.private_key = await asyncio.get_event_loop().run_in_executor(
@@ -477,14 +446,6 @@ class Output(InputOutput):
)
return cls(amount, script)
@classmethod
def pay_support_data_pubkey_hash(
cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_support_data_pubkey_hash(
claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash
)
return cls(amount, script)
@classmethod
def add_purchase_data(cls, purchase: Purchase) -> 'Output':
script = OutputScript.return_data(purchase)
@@ -899,20 +860,12 @@ class Transaction:
@classmethod
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None):
funding_accounts: List['Account'], change_account: 'Account'):
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
if signing_channel is not None:
support = Support()
support.emoji = '👍'
support_output = Output.pay_support_data_pubkey_hash(
amount, claim_name, claim_id, support, ledger.address_to_hash160(holding_address)
)
support_output.sign(signing_channel, b'placeholder txid:nout')
else:
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
return cls.create([], [support_output], funding_accounts, change_account, sign=False)
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
return cls.create([], [support_output], funding_accounts, change_account)
@classmethod
def purchase(cls, claim_id: str, amount: int, merchant_address: bytes,
+4 -9
View File
@@ -35,7 +35,7 @@ class TimestampedPreferences(UserDict):
def __setitem__(self, key, value):
self.data[key] = {
'value': value,
'ts': int(time.time())
'ts': time.time()
}
def __repr__(self):
@@ -145,15 +145,14 @@ class Wallet:
elif not self.is_locked:
log.warning(
"Disk encryption requested but no password available for encryption. "
"Resetting encryption preferences and saving wallet in an unencrypted state."
"Saving wallet in an unencrypted state."
)
self.preferences[ENCRYPT_ON_DISK] = False
return self.storage.write(self.to_dict())
@property
def hash(self) -> bytes:
h = sha256()
if self.is_encrypted:
if self.preferences.get(ENCRYPT_ON_DISK, False):
assert self.encryption_password is not None, \
"Encryption is enabled but no password is available, cannot generate hash."
h.update(self.encryption_password.encode())
@@ -220,11 +219,7 @@ class Wallet:
@property
def is_encrypted(self) -> bool:
# either its locked or it was unlocked using a password.
# if its set to encrypt on preferences but isnt encrypted and no password was given so far,
# then its not encrypted
return self.is_locked or (
self.preferences.get(ENCRYPT_ON_DISK, False) and self.encryption_password is not None)
return self.is_locked or self.preferences.get(ENCRYPT_ON_DISK, False)
def decrypt(self):
assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first."
-1
View File
@@ -1,7 +1,6 @@
import curses
import time
import asyncio
import lbry.wallet
from lbry.conf import Config
from lbry.extras.daemon.client import daemon_rpc
+1 -2
View File
@@ -25,8 +25,7 @@ RETURN_DOCS = {
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
'Address': 'an address in base58',
'Dict': 'glorious data in dictionary',
'Address': 'an address in base58'
}
-1
View File
@@ -35,7 +35,6 @@ disable=
too-many-statements,
too-many-nested-blocks,
too-many-public-methods,
too-many-return-statements,
too-many-instance-attributes,
protected-access,
unused-argument
+2 -2
View File
@@ -34,7 +34,7 @@ setup(
},
install_requires=[
'aiohttp==3.5.4',
'aioupnp==0.0.18',
'aioupnp==0.0.17',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
@@ -46,7 +46,7 @@ setup(
'msgpack==0.6.1',
'prometheus_client==0.7.1',
'ecdsa==0.13.3',
'pyyaml==5.3.1',
'pyyaml==4.2b1',
'docopt==0.6.2',
'hachoir',
'multidict==4.6.1',
@@ -40,6 +40,8 @@ class ClaimTestCase(CommandTestCase):
class ClaimSearchCommand(ClaimTestCase):
VERBOSITY = logging.WARNING
async def create_channel(self):
self.channel = await self.channel_create('@abc', '1.0')
self.channel_id = self.get_claim_id(self.channel)
@@ -89,7 +91,7 @@ class ClaimSearchCommand(ClaimTestCase):
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
] * 33829
] * 23829
with self.assertRaises(ConnectionResetError):
await self.claim_search(claim_ids=claim_ids)
@@ -157,6 +159,60 @@ class ClaimSearchCommand(ClaimTestCase):
await self.stream_abandon(txid=signed2['txid'], nout=0)
await self.assertFindsClaims([], channel_ids=[channel_id2])
async def test_break_it(self):
await self.generate(5)
address = await self.account.receiving.get_or_create_usable_address()
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid)
address = await self.account.receiving.get_or_create_usable_address()
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid)
address = await self.account.receiving.get_or_create_usable_address()
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid)
address = await self.account.receiving.get_or_create_usable_address()
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid)
await self.generate(7)
async def _doit(n):
try:
await self.daemon.jsonrpc_channel_create(
name=f'@arena{n}', bid='0.1', blocking=True
)
except InsufficientFundsError:
pass
def doit(n):
asyncio.create_task(_doit(n))
async def break_it():
count = 0
for _ in range(4):
for _ in range(10):
count += 1
if not count % 7:
asyncio.create_task(self.generate(1))
doit(count)
if self.ledger._known_addresses_out_of_sync:
print('out of sync', self.ledger._known_addresses_out_of_sync)
await asyncio.sleep(1)
await self.generate(1)
bp = self.conductor.spv_node.server.bp
break_task = asyncio.create_task(break_it())
address = await self.ledger.went_out_of_sync.get()
bp.block_notify.clear()
print('%s is out of sync' % address)
with self.assertRaises(InsufficientFundsError):
await self.daemon.jsonrpc_channel_create(
name=f'@derp', bid='0.1', blocking=True
)
self.assertTrue(False)
print("woohoo")
if not break_task.done():
break_task.cancel()
async def test_pagination(self):
await self.create_channel()
await self.create_lots_of_streams()
@@ -262,19 +318,6 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc')
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es')
await self.assertFindsClaims([claim4, claim3], any_languages=['en'])
await self.assertFindsClaims([claim2, claim1], any_languages=['none'])
await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_languages=['none', 'en'])
await self.assertFindsClaims([claim5], any_languages=['es'])
await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es'])
await self.assertFindsClaims([], fee_currency='foo')
async def test_search_by_channel(self):
@@ -332,25 +375,6 @@ class ClaimSearchCommand(ClaimTestCase):
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
async def test_limit_claims_per_channel(self):
match = self.assertFindsClaims
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
claim3 = await self.stream_create('claim3', channel_id=chan1_id)
claim4 = await self.stream_create('claim4', channel_id=chan1_id)
claim5 = await self.stream_create('claim5', channel_id=chan2_id)
claim6 = await self.stream_create('claim6', channel_id=chan2_id)
await match(
[claim6, claim5, claim4, claim3, claim1],
limit_claims_per_channel=2, claim_type='stream'
)
await match(
[claim6, claim5, claim4, claim3, claim2, claim1],
limit_claims_per_channel=3, claim_type='stream'
)
async def test_claim_type_and_media_type_search(self):
# create an invalid/unknown claim
address = await self.account.receiving.get_or_create_usable_address()
@@ -1860,8 +1884,7 @@ class SupportCommands(CommandTestCase):
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2',
funding_account_ids=[account2.id])
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(tip['txid'])
@@ -1892,8 +1915,7 @@ class SupportCommands(CommandTestCase):
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2',
funding_account_ids=[account2.id])
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(support['txid'])
@@ -1920,12 +1942,6 @@ class SupportCommands(CommandTestCase):
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
async def test_signed_supports(self):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
tx = await self.support_create(stream_id, '0.3', channel_id=channel_id)
self.assertTrue(tx['outputs'][0]['is_channel_signature_valid'])
class CollectionCommands(CommandTestCase):
+6 -12
View File
@@ -75,7 +75,7 @@ class ReconnectTests(IntegrationTestCase):
session.trigger_urgent_reconnect.set()
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
self.assertTrue(self.ledger.network.is_connected())
self.assertTrue(self.ledger.network.is_connected)
switch_event = self.ledger.network.on_connected.first
await node2.stop(True)
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
@@ -99,7 +99,7 @@ class ReconnectTests(IntegrationTestCase):
address1 = await self.account.receiving.get_or_create_usable_address()
# disconnect and send a new tx, should reconnect and get it
self.ledger.network.client.connection_lost(Exception())
self.assertFalse(self.ledger.network.is_connected())
self.assertFalse(self.ledger.network.is_connected)
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
await self.blockchain.generate(1)
@@ -122,7 +122,7 @@ class ReconnectTests(IntegrationTestCase):
sendtxid = await self.blockchain.send_to_address(address1, 42)
await self.blockchain.generate(1)
# (this is just so the test doesn't hang forever if it doesn't reconnect)
if not self.ledger.network.is_connected():
if not self.ledger.network.is_connected:
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
# omg, the burned cable still works! torba is fire proof!
await self.ledger.network.get_transaction(sendtxid)
@@ -130,11 +130,11 @@ class ReconnectTests(IntegrationTestCase):
async def test_timeout_then_reconnect(self):
# tests that it connects back after some failed attempts
await self.conductor.spv_node.stop()
self.assertFalse(self.ledger.network.is_connected())
self.assertFalse(self.ledger.network.is_connected)
await asyncio.sleep(0.2) # let it retry and fail once
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected())
self.assertTrue(self.ledger.network.is_connected)
async def test_online_but_still_unavailable(self):
# Edge case. See issue #2445 for context
@@ -179,18 +179,12 @@ class ServerPickingTestCase(AsyncioTestCase):
],
'connect_timeout': 3
})
ledger.accounts = []
async def subscribe_accounts():
pass
ledger.subscribe_accounts = subscribe_accounts
network = Network(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
self.assertTrue(network.is_connected())
self.assertTrue(network.is_connected)
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
# ensure we are connected to all of them after a while
@@ -117,6 +117,11 @@ class ResolveCommand(BaseResolveTestCase):
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo:1', claim_id1)
await self.assertResolvesToClaimId('foo:2', claim_id2)
await self.assertResolvesToClaimId('foo:3', claim_id3)
await self.assertResolvesToClaimId('foo:4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
@@ -5,7 +5,6 @@ from itertools import chain
from lbry.wallet.transaction import Transaction, Output, Input
from lbry.testcase import IntegrationTestCase
from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis
from lbry.wallet.manager import WalletManager
class BasicTransactionTests(IntegrationTestCase):
@@ -152,11 +151,8 @@ class BasicTransactionTests(IntegrationTestCase):
for batch in range(0, len(sends), 10):
txids = await asyncio.gather(*sends[batch:batch + 10])
await asyncio.wait([self.on_transaction_id(txid) for txid in txids])
client = self.ledger.network.get_wallet_session(self.account.wallet)
remote_status = await self.ledger.network.subscribe_address(client, address)
self.assertTrue(await self.ledger.update_history(client, address, remote_status))
remote_status = await self.ledger.network.subscribe_address(address)
self.assertTrue(await self.ledger.update_history(address, remote_status))
# 20 unconfirmed txs, 10 from blockchain, 10 from local to local
utxos = await self.account.get_utxos()
txs = []
@@ -169,138 +165,11 @@ class BasicTransactionTests(IntegrationTestCase):
await self.broadcast(tx)
txs.append(tx)
await asyncio.wait([self.on_transaction_address(tx, address) for tx in txs], timeout=1)
remote_status = await self.ledger.network.subscribe_address(client, address)
self.assertTrue(await self.ledger.update_history(client, address, remote_status))
remote_status = await self.ledger.network.subscribe_address(address)
self.assertTrue(await self.ledger.update_history(address, remote_status))
# server history grows unordered
txid = await self.blockchain.send_to_address(address, 1)
await self.on_transaction_id(txid)
self.assertTrue(await self.ledger.update_history(client, address, remote_status))
self.assertTrue(await self.ledger.update_history(address, remote_status))
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
def wait_for_txid(self, txid, address):
return self.ledger.on_transaction.where(
lambda e: e.tx.id == txid and e.address == address
)
async def _test_transaction(self, send_amount, address, inputs, change):
tx = await Transaction.create(
[], [Output.pay_pubkey_hash(send_amount, self.ledger.address_to_hash160(address))], [self.account],
self.account
)
await self.ledger.broadcast(tx)
input_amounts = [txi.amount for txi in tx.inputs]
self.assertListEqual(inputs, input_amounts)
self.assertEqual(len(inputs), len(tx.inputs))
self.assertEqual(2, len(tx.outputs))
self.assertEqual(send_amount, tx.outputs[0].amount)
self.assertEqual(change, tx.outputs[1].amount)
return tx
async def assertSpendable(self, amounts):
spendable = await self.ledger.db.get_spendable_utxos(
self.ledger, 2000000000000, [self.account], set_reserved=False, return_insufficient_funds=True
)
got_amounts = [estimator.effective_amount for estimator in spendable]
self.assertListEqual(amounts, got_amounts)
async def test_sqlite_coin_chooser(self):
wallet_manager = WalletManager([self.wallet], {self.ledger.get_id(): self.ledger})
await self.blockchain.generate(300)
await self.assertBalance(self.account, '0.0')
address = await self.account.receiving.get_or_create_usable_address()
other_account = self.wallet.generate_account(self.ledger)
other_address = await other_account.receiving.get_or_create_usable_address()
self.ledger.coin_selection_strategy = 'sqlite'
await self.ledger.subscribe_account(self.account)
txids = []
txids.append(await self.blockchain.send_to_address(address, 1.0))
txids.append(await self.blockchain.send_to_address(address, 1.0))
txids.append(await self.blockchain.send_to_address(address, 3.0))
txids.append(await self.blockchain.send_to_address(address, 5.0))
txids.append(await self.blockchain.send_to_address(address, 10.0))
await asyncio.wait([self.wait_for_txid(txid, address) for txid in txids], timeout=1)
await self.assertBalance(self.account, '20.0')
await self.assertSpendable([99992600, 99992600, 299992600, 499992600, 999992600])
# send 1.5 lbc
first_tx = await Transaction.create(
[], [Output.pay_pubkey_hash(150000000, self.ledger.address_to_hash160(other_address))], [self.account],
self.account
)
self.assertEqual(2, len(first_tx.inputs))
self.assertEqual(2, len(first_tx.outputs))
self.assertEqual(100000000, first_tx.inputs[0].amount)
self.assertEqual(100000000, first_tx.inputs[1].amount)
self.assertEqual(150000000, first_tx.outputs[0].amount)
self.assertEqual(49980200, first_tx.outputs[1].amount)
await self.assertBalance(self.account, '18.0')
await self.assertSpendable([299992600, 499992600, 999992600])
await wallet_manager.broadcast_or_release(first_tx, blocking=True)
await self.assertSpendable([49972800, 299992600, 499992600, 999992600])
# 0.499, 3.0, 5.0, 10.0
await self.assertBalance(self.account, '18.499802')
# send 1.5lbc again
second_tx = await self._test_transaction(150000000, other_address, [49980200, 300000000], 199960400)
await self.assertSpendable([499992600, 999992600])
# replicate cancelling the api call after the tx broadcast while ledger.wait'ing it
e = asyncio.Event()
real_broadcast = self.ledger.broadcast
async def broadcast(tx):
try:
return await real_broadcast(tx)
finally:
e.set()
self.ledger.broadcast = broadcast
broadcast_task = asyncio.create_task(wallet_manager.broadcast_or_release(second_tx, blocking=True))
# wait for the broadcast to finish
await e.wait()
# cancel the api call
broadcast_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await broadcast_task
# test if sending another 1.5 lbc will try to double spend the inputs from the cancelled tx
tx1 = await self._test_transaction(150000000, other_address, [500000000], 349987600)
await self.ledger.wait(tx1, timeout=1)
# wait for the cancelled transaction too, so that it's in the database
# needed to keep everything deterministic
await self.ledger.wait(second_tx, timeout=1)
await self.assertSpendable([199953000, 349980200, 999992600])
# spend deep into the mempool and see what else breaks
tx2 = await self._test_transaction(150000000, other_address, [199960400], 49948000)
await self.assertSpendable([349980200, 999992600])
await self.ledger.wait(tx2, timeout=1)
await self.assertSpendable([49940600, 349980200, 999992600])
tx3 = await self._test_transaction(150000000, other_address, [49948000, 349987600], 249915800)
await self.assertSpendable([999992600])
await self.ledger.wait(tx3, timeout=1)
await self.assertSpendable([249908400, 999992600])
tx4 = await self._test_transaction(150000000, other_address, [249915800], 99903400)
await self.assertSpendable([999992600])
await self.ledger.wait(tx4, timeout=1)
await self.assertBalance(self.account, '10.999034')
await self.assertSpendable([99896000, 999992600])
# spend more
tx5 = await self._test_transaction(100000000, other_address, [99903400, 1000000000], 999883600)
await self.assertSpendable([])
await self.ledger.wait(tx5, timeout=1)
await self.assertSpendable([999876200])
await self.assertBalance(self.account, '9.998836')
@@ -8,14 +8,16 @@ from lbry.wallet.dewies import dict_values_to_lbc
class WalletCommands(CommandTestCase):
async def test_wallet_create_and_add_subscribe(self):
self.assertSetEqual({0, 27}, {len(session.hashX_subs) for session in self.conductor.spv_node.server.session_mgr.sessions})
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions))
self.assertEqual(len(session.hashX_subs), 27)
wallet = await self.daemon.jsonrpc_wallet_create('foo', create_account=True, single_key=True)
self.assertSetEqual({0, 27, 1}, {len(session.hashX_subs) for session in self.conductor.spv_node.server.session_mgr.sessions})
self.assertEqual(len(session.hashX_subs), 28)
await self.daemon.jsonrpc_wallet_remove(wallet.id)
self.assertSetEqual({0, 27}, {len(session.hashX_subs) for session in self.conductor.spv_node.server.session_mgr.sessions})
self.assertEqual(len(session.hashX_subs), 27)
await self.daemon.jsonrpc_wallet_add(wallet.id)
self.assertSetEqual({0, 27, 1}, {len(session.hashX_subs) for session in self.conductor.spv_node.server.session_mgr.sessions})
self.assertEqual(len(session.hashX_subs), 28)
async def test_wallet_syncing_status(self):
address = await self.daemon.jsonrpc_address_unused()
@@ -2,33 +2,26 @@ import asyncio
import lbry
import lbry.wallet
import unittest
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.testcase import CommandTestCase
from lbry.testcase import IntegrationTestCase, CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
class MockNetwork:
def __init__(self, ledger):
self.ledger = ledger
self._on_header_controller = None
class TestSessions(CommandTestCase):
class TestSessions(IntegrationTestCase):
"""
Tests that server cleans up stale connections after session timeout and client times out too.
"""
LEDGER = lbry.wallet
async def test_session_bloat_from_socket_timeout(self):
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
await self.conductor.start_spv()
session = ClientSession(
network=MockNetwork(self.ledger), server=(self.conductor.spv_node.hostname, self.conductor.spv_node.port),
timeout=0.2
network=None, server=(self.conductor.spv_node.hostname, self.conductor.spv_node.port), timeout=0.2
)
await session.create_connection()
await session.send_request('server.banner', ())
@@ -53,7 +46,6 @@ class TestSessions(CommandTestCase):
class TestUsagePayment(CommandTestCase):
@unittest.skip('freezes ci')
async def test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
wallet_pay_service.payment_period = 1
-7
View File
@@ -4,7 +4,6 @@ from lbry.testcase import CommandTestCase
class EpicAdventuresOfChris45(CommandTestCase):
async def test_no_this_is_not_a_test_its_an_adventure(self):
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
# and everything in between; it's also the setting of some record setting
# integration tests.
@@ -195,9 +194,3 @@ class EpicAdventuresOfChris45(CommandTestCase):
}},
await self.resolve(uri)
)
# He closes and opens the wallet server databases to see how horribly they break
db = self.conductor.spv_node.server.db
db.close()
await db.open_for_serving()
# They didn't! (error would be AssertionError: 276 vs 266 (264 counts) on startup)
+23 -300
View File
@@ -1,4 +1,5 @@
import re
import time
import typing
from math import ceil
@@ -29,17 +30,6 @@ class MockedCommentServer:
'timestamp': None,
'channel_url': None,
'is_hidden': False,
'is_pinned': False,
}
REACT_SCHEMA = {
'react_id': None,
'comment_id': None,
'reaction_type': None,
'timestamp': None,
'signature': None,
'channel_id': None,
'channel_name': None
}
def __init__(self, port=2903):
@@ -49,9 +39,7 @@ class MockedCommentServer:
self.runner = None
self.server = None
self.comments = []
self.reacts = {}
self.comment_id = 0
self.react_id = 0
@classmethod
def _create_comment(cls, **kwargs):
@@ -59,12 +47,6 @@ class MockedCommentServer:
schema.update(**kwargs)
return schema
@classmethod
def _react(cls, **kwargs):
schema = cls.REACT_SCHEMA.copy()
schema.update(**kwargs)
return schema
def create_comment(self, claim_id=None, parent_id=None, channel_name=None, channel_id=None, **kwargs):
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
@@ -123,21 +105,6 @@ class MockedCommentServer:
return True
return False
def pin_comment(
self,
comment_id: typing.Union[int, str],
channel_name: str, channel_id: str, remove: bool,
signing_ts: str, signature: str
):
comment_id = self.get_comment_id(comment_id)
if self.is_signable(signature, signing_ts):
if remove:
self.comments[comment_id]['is_pinned'] = False
else:
self.comments[comment_id]['is_pinned'] = True
return self.comments[comment_id]
return False
def hide_comments(self, pieces: list):
hidden = []
for p in pieces:
@@ -150,18 +117,27 @@ class MockedCommentServer:
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, hidden=False, visible=False, page=1, page_size=50,**kwargs):
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
if hidden ^ visible:
comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(comments)/page_size),
'total_items': len(comments),
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], self.comments)))
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
}
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(select_comments) / page_size),
'total_items': len(select_comments),
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
@@ -181,99 +157,15 @@ class MockedCommentServer:
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
def react(
self,
comment_ids=None,
channel_name=None,
channel_id=None,
remove=None,
clear_types=None,
type=None,
signing_ts=None,
**kwargs
):
comment_batch = comment_ids.split(',')
for item in comment_batch:
react_id = self.react_id
c_id = item.strip()
reacts_for_comment_id = [r for r in list(self.reacts.values()) if r['comment_id'] == c_id]
channels_reacts_for_comment_id = [r for r in reacts_for_comment_id if r['channel_id'] == channel_id]
if remove:
matching_react = None
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == type:
matching_react = reaction
break
if matching_react:
self.reacts.pop(int(matching_react['react_id']), None)
else:
if clear_types:
for r_type in clear_types.split(','):
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == r_type:
x = self.reacts.pop(int(reaction['react_id']), None)
react = self._react(
react_id=str(react_id),
comment_id=str(c_id),
channel_name=channel_name,
channel_id=channel_id,
reaction_type=type,
timestamp=str(int(time.time())),
**kwargs
)
self.reacts[self.react_id] = react
self.react_id += 1
self.clean(react)
return True
def list_reacts(self, comment_ids, channel_id, channel_name, types=None, **kwargs):
all_types = list(set([r['reaction_type'] for r in list(self.reacts.values())]))
comment_id_list = comment_ids.split(',')
# _reacts: {'a1': {'like': 0, 'dislike': 0}, 'a2': {'like': 0, 'dislike': 0}}
own_reacts = {}
other_reacts = {}
# for each comment_id
# add comment_id: {} to own_reacts and other_reacts
# for each react in own_reacts
# for each react in other_reacts
for cid in comment_id_list:
own_reacts[cid] = {}
other_reacts[cid] = {}
for r_type in all_types:
own_reacts[cid][r_type] = 0
other_reacts[cid][r_type] = 0
# process that comment ids reactions for own and other categories
reacts_for_comment = list(filter(lambda c: c['comment_id'] == cid, list(self.reacts.values())))
if types:
reacts_for_comment = list(filter(lambda c: c['reaction_type'] in types.split(','), reacts_for_comment))
own_reacts_for_comment = list(filter(lambda c: c['channel_id'] == channel_id, reacts_for_comment))
other_reacts_for_comment = list(filter(lambda c: c['channel_id'] != channel_id, reacts_for_comment))
if own_reacts_for_comment:
for react in own_reacts_for_comment:
own_reacts[cid][react['reaction_type']] += 1
if other_reacts_for_comment:
for react in other_reacts_for_comment:
other_reacts[cid][react['reaction_type']] += 1
return {
'my_reactions': own_reacts,
'others_reactions': other_reacts,
}
methods = {
'comment.List': get_claim_comments,
'get_claim_comments': get_claim_comments,
'get_comments_by_id': get_comments_by_id,
'comment.Create': create_comment,
'comment.Abandon': abandon_comment,
'comment.GetChannelFromCommentID': get_comment_channel_by_id,
'comment.Hide': hide_comments,
'comment.Pin': pin_comment,
'comment.Edit': edit_comment,
'reaction.React': react,
'reaction.List': list_reacts,
'create_comment': create_comment,
'abandon_comment': abandon_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
'get_claim_hidden_comments': get_claim_hidden_comments,
'hide_comments': hide_comments,
'edit_comment': edit_comment,
}
def process_json(self, body) -> dict:
@@ -535,56 +427,6 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test05_comment_pin(self):
# wherein a bee makes a sick burn on moth's channel and moth pins it
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
moth_stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
moth_claim_id = moth_stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=moth_claim_id,
channel_id=bee['claim_id']
)
self.assertFalse(comment1['is_pinned'])
comment2 = await self.daemon.jsonrpc_comment_create(
comment='sick burn, brah',
claim_id=moth_claim_id,
channel_id=moth_id,
)
self.assertFalse(comment2['is_pinned'])
comments = await self.daemon.jsonrpc_comment_list(moth_claim_id)
comments_items = comments['items']
self.assertIn('is_pinned', comments_items[0])
self.assertFalse(comments_items[0]['is_pinned'])
pinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name']
)
self.assertTrue(pinned['is_pinned'])
comments_after_pinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_pin = comments_after_pinning['items']
self.assertTrue(items_after_pin[1]['is_pinned'])
unpinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
remove=True
)
self.assertFalse(unpinned['is_pinned'])
comments_after_unpinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_unpin = comments_after_unpinning['items']
self.assertFalse(items_after_unpin[1]['is_pinned'])
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
@@ -608,6 +450,7 @@ class CommentCommands(CommandTestCase):
channel_id=bee['claim_id']
)
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
@@ -689,123 +532,3 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
async def test06_reactions(self):
# wherein a bee insulted by a rude moth accidentally
# 1) likes an insult
# 2) clicks dislike instead, testing "clear_types"
# 3) then clicks dislike again, testing "remove"
# calls from the point of view of the moth
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
first_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth['claim_id'],
)
second_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
all_comments = [second_comment, first_comment]
comment_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertEqual(comment_list['total_items'], 2)
bee_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='like',
)
moth_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_type='like',
)
reactions = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1}}, 'others_reactions': {'0': {'like': 1}}}
self.assertEqual(reactions['my_reactions']['0']['like'], 1)
self.assertEqual(reactions['others_reactions']['0']['like'], 1)
bee_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='dislike',
clear_types='like',
)
reactions_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1, 'dislike', 0}}, 'others_reactions': {'0': {'like': 0, 'dislike': 1}}}
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(reactions_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['dislike'], 1)
self.assertEqual(reactions_after_bee_dislikes['others_reactions']['0']['like'], 0)
only_likes_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_types='like',
)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['like'], 1)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions']['0']['like'], 0)
bee_un_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
remove=True,
react_type='dislike',
)
reactions_after_bee_absconds = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_bee_absconds['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_bee_absconds['my_reactions']['0'])
self.assertEqual(reactions_after_bee_absconds['others_reactions']['0']['like'], 0)
self.assertNotIn('dislike', reactions_after_bee_absconds['others_reactions']['0'])
bee_reacts_to_both_comments = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='frozen_tom',
)
reactions_after_double_frozen_tom = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment['comment_id'] + ',' + second_comment['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_double_frozen_tom['my_reactions']['0']['like'], 1)
self.assertNotIn('dislike', reactions_after_double_frozen_tom['my_reactions']['0'])
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['0']['frozen_tom'], 1)
self.assertEqual(reactions_after_double_frozen_tom['others_reactions']['1']['frozen_tom'], 1)
@@ -1,3 +1,4 @@
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.testcase import CommandTestCase
@@ -28,9 +29,14 @@ class SettingsManagement(CommandTestCase):
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
# tests that changing share_usage_data propagates to the relevant properties
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
loggly = get_loggly_handler(self.daemon.conf)
self.addCleanup(loggly.close)
self.assertFalse(self.daemon.analytics_manager.enabled)
self.assertFalse(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', True)
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
self.assertTrue(self.daemon.analytics_manager.enabled)
self.assertTrue(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', False)
+5 -5
View File
@@ -37,8 +37,8 @@ class TestSigningComments(AsyncioTestCase):
rswanson = await get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, sign_comment_id=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, sign_comment_id=True))
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
@@ -53,7 +53,7 @@ class TestSigningComments(AsyncioTestCase):
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, sign_comment_id=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, sign_comment_id=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, sign_comment_id=True))
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
@@ -1,9 +1,7 @@
import binascii
import unittest
from lbry.dht.error import DecodeError
from lbry.dht.serialization.bencoding import _bencode
from lbry.dht.serialization.datagram import RequestDatagram, ResponseDatagram, decode_datagram, ErrorDatagram
from lbry.dht.serialization.datagram import _decode_datagram
from lbry.dht.serialization.datagram import REQUEST_TYPE, RESPONSE_TYPE, ERROR_TYPE
from lbry.dht.serialization.datagram import make_compact_address, decode_compact_address
@@ -141,38 +139,6 @@ class TestDatagram(unittest.TestCase):
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_str_or_int_keys(self):
datagram = decode_datagram(_bencode({
b'0': 0,
b'1': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
b'2': b'111111111111111111111111111111111111111111111111',
b'3': b'ping',
b'4': [{b'protocolVersion': 1}],
b'5': b'should not error'
}))
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_mixed_str_or_int_keys(self):
# datagram, _ = _bencode({
# b'0': 0,
# 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
# b'2': b'111111111111111111111111111111111111111111111111',
# 3: b'ping',
# b'4': [{b'protocolVersion': 1}],
# b'5': b'should not error'
# }))
encoded = binascii.unhexlify(b"64313a3069306569316532303a0abcb5269d6cfc1e87a08e920bf39fe9df8e92fc313a3234383a313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131693365343a70696e67313a346c6431353a70726f746f636f6c56657273696f6e6931656565313a3531363a73686f756c64206e6f74206572726f7265")
self.assertDictEqual(
{
'packet_type': 0,
'rpc_id': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
'node_id': b'111111111111111111111111111111111111111111111111',
'method': b'ping',
'args': [{b'protocolVersion': 1}]
}, _decode_datagram(encoded)[0]
)
class TestCompactAddress(unittest.TestCase):
def test_encode_decode(self, address='1.2.3.4', port=4444, node_id=b'1' * 48):
+1 -26
View File
@@ -1,7 +1,7 @@
import asyncio
import unittest
from lbry.utils import generate_id
from lbry.dht.peer import PeerManager, make_kademlia_peer, is_valid_public_ipv4
from lbry.dht.peer import PeerManager, make_kademlia_peer
from lbry.testcase import AsyncioTestCase
@@ -50,31 +50,6 @@ class PeerTest(AsyncioTestCase):
for i in range(32):
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], f"{224 + i}.0.0.0", 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '255.255.255.255', 1000)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], 'beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', 1000
)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], '2001:db8::ff00:42:8329', 1000
)
def test_is_valid_ipv4(self):
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef'))
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', True))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329'))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329', True))
self.assertFalse(is_valid_public_ipv4('127.0.0.1'))
self.assertTrue(is_valid_public_ipv4('127.0.0.1', True))
self.assertFalse(is_valid_public_ipv4('172.16.0.1'))
self.assertFalse(is_valid_public_ipv4('172.16.0.1', True))
self.assertTrue(is_valid_public_ipv4('1.2.3.4'))
self.assertTrue(is_valid_public_ipv4('1.2.3.4', True))
self.assertFalse(is_valid_public_ipv4('derp'))
self.assertFalse(is_valid_public_ipv4('derp', True))
def test_boolean(self):
self.assertNotEqual(self.first_contact, self.second_contact)
@@ -1,53 +0,0 @@
import unittest
from aiohttp.test_utils import make_mocked_request as request
from aiohttp.web import HTTPForbidden
from lbry.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.extras.daemon.security import is_request_allowed as allowed, ensure_request_allowed as ensure
class TestAllowedOrigin(unittest.TestCase):
def test_allowed_origin_default(self):
conf = Config()
# lack of Origin is always allowed
self.assertTrue(allowed(request('GET', '/'), conf))
# deny all other Origins
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_star(self):
conf = Config(allowed_origin='*')
# everything is allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_specified(self):
conf = Config(allowed_origin='localhost')
# no origin and only localhost are allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_ensure_default(self):
conf = Config()
ensure(request('GET', '/'), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
self.assertIn("'localhost' are not allowed", log.output[0])
def test_ensure_specific(self):
conf = Config(allowed_origin='localhost')
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)
self.assertIn("'hackers.com' are not allowed", log.output[0])
self.assertIn("'allowed_origin' limits requests to: 'localhost'", log.output[0])
+15 -16
View File
@@ -9,15 +9,14 @@ claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
class TestURLParsing(unittest.TestCase):
segments = 'stream', 'channel'
fields = 'name', 'claim_id', 'amount_order'
fields = 'name', 'claim_id', 'sequence', 'amount_order'
def _assert_url(self, url_string, strictly=True, **kwargs):
def _assert_url(self, url_string, **kwargs):
url = URL.parse(url_string)
if strictly:
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
present = {}
for key in kwargs:
for segment_name in self.segments:
@@ -43,22 +42,19 @@ class TestURLParsing(unittest.TestCase):
url = self._assert_url
# stream
url('test', stream_name='test')
url('test*1', stream_name='test*1')
url('test:1', stream_name='test', stream_sequence='1')
url('test$1', stream_name='test', stream_amount_order='1')
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False)
url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id)
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id)
# channel
url('@test', channel_name='@test')
url('@test:1', channel_name='@test', channel_sequence='1')
url('@test$1', channel_name='@test', channel_amount_order='1')
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False)
url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id)
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id)
# channel/stream
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False)
url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# combined legacy and new
url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\uD799')
@@ -108,8 +104,10 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://test@")
fail("lbry://tes@t")
fail(f"lbry://test:1#{claim_id}")
fail("lbry://test:0")
fail("lbry://test$0")
fail("lbry://test/path")
fail("lbry://@test1:1ab/fakepath")
fail("lbry://test:1:1:1")
fail("whatever/lbry://test")
fail("lbry://lbry://test")
@@ -117,4 +115,5 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://abc:0x123")
fail("lbry://abc:0x123/page")
fail("lbry://@test1#ABCDEF/fakepath")
fail("test:0001")
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
+2 -121
View File
@@ -10,7 +10,7 @@ from lbry.stream.stream_manager import StreamManager
from lbry.stream.reflector.server import ReflectorServer
class TestReflector(AsyncioTestCase):
class TestStreamAssembler(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
@@ -22,7 +22,6 @@ class TestReflector(AsyncioTestCase):
self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf)
self.addCleanup(self.blob_manager.stop)
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
server_tmp_dir = tempfile.mkdtemp()
@@ -31,7 +30,6 @@ class TestReflector(AsyncioTestCase):
self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf)
self.addCleanup(self.server_blob_manager.stop)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
@@ -40,7 +38,7 @@ class TestReflector(AsyncioTestCase):
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream_manager.config.reflect_streams = False
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self, response_chunk_size):
@@ -56,7 +54,6 @@ class TestReflector(AsyncioTestCase):
set(map(lambda b: b.blob_hash,
self.stream.descriptor.blobs[:-1] + [self.blob_manager.get_blob(self.stream.sd_hash)]))
)
self.assertTrue(self.stream.is_fully_reflected)
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
self.assertTrue(server_sd_blob.get_is_verified())
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
@@ -78,119 +75,3 @@ class TestReflector(AsyncioTestCase):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
async def test_result_from_disconnect_mid_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
stop.set()
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_after_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await not_incoming.wait()
stop.set()
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_mid_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
stop.set()
with self.assertRaises(asyncio.CancelledError):
await reflect_task
# self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
)
self.assertFalse(self.stream.is_fully_reflected)
async def test_delete_file_during_reflector_upload(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
with self.assertRaises(asyncio.CancelledError):
await reflect_task
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
)
self.assertFalse(self.stream.is_fully_reflected)
+24 -3
View File
@@ -15,6 +15,7 @@ from lbry.testcase import AsyncioTestCase
from lbry.extras.cli import normalize_value, main, setup_logging
from lbry.extras.system_info import get_platform
from lbry.extras.daemon.daemon import Daemon
from lbry.extras.daemon.loggly_handler import HTTPSLogglyHandler
from lbry.conf import Config
from lbry.extras import cli
@@ -62,8 +63,10 @@ class CLILoggingTest(AsyncioTestCase):
log = log.getChild("lbry")
self.assertTrue(log.isEnabledFor(logging.INFO))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 1)
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 2)
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
self.assertFalse(log.handlers[1].enabled)
async with get_logger(["start", "--verbose"]) as log:
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
@@ -77,14 +80,32 @@ class CLILoggingTest(AsyncioTestCase):
self.assertFalse(log.getChild("lbry").isEnabledFor(logging.DEBUG))
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
async def test_loggly(self):
async with get_logger(["start"]) as log: # default share_usage_data=False
log = log.getChild("lbry")
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=True) as log:
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertTrue(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async def test_quiet(self):
async with get_logger(["start"]) as log: # default is loud
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 2)
self.assertEqual(len(log.handlers), 3)
self.assertIs(type(log.handlers[1]), logging.StreamHandler)
async with get_logger(["start", "--quiet"]) as log:
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 1)
self.assertEqual(len(log.handlers), 2)
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
+4 -4
View File
@@ -132,7 +132,7 @@ class TestAccount(AsyncioTestCase):
async def test_load_and_save_account(self):
account_data = {
'name': 'Main Account',
'modified_on': 123,
'modified_on': 123.456,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
@@ -183,7 +183,7 @@ class TestAccount(AsyncioTestCase):
def test_merge_diff(self):
account_data = {
'name': 'My Account',
'modified_on': 123,
'modified_on': 123.456,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
@@ -203,7 +203,7 @@ class TestAccount(AsyncioTestCase):
account = Account.from_dict(self.ledger, Wallet(), account_data)
self.assertEqual(account.name, 'My Account')
self.assertEqual(account.modified_on, 123)
self.assertEqual(account.modified_on, 123.456)
self.assertEqual(account.change.gap, 5)
self.assertEqual(account.change.maximum_uses_per_address, 2)
self.assertEqual(account.receiving.gap, 5)
@@ -366,7 +366,7 @@ class TestSingleKeyAccount(AsyncioTestCase):
async def test_load_and_save_account(self):
account_data = {
'name': 'My Account',
'modified_on': 123,
'modified_on': 123.456,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
+8 -15
View File
@@ -16,12 +16,12 @@ class MockNetwork:
self.address = None
self.get_history_called = []
self.get_transaction_called = []
self.is_connected = lambda _: False
self.is_connected = False
def retriable_call(self, function, *args, **kwargs):
return function(*args, **kwargs)
async def get_history(self, address, session=None):
async def get_history(self, address):
self.get_history_called.append(address)
self.address = address
return self.history
@@ -35,17 +35,11 @@ class MockNetwork:
async def get_transaction_and_merkle(self, tx_hash, known_height=None):
tx = await self.get_transaction(tx_hash)
merkle = {'block_height': -1}
merkle = {}
if known_height:
merkle = await self.get_merkle(tx_hash, known_height)
return tx, merkle
async def get_transaction_batch(self, txids, session=None):
return {
txid: await self.get_transaction_and_merkle(txid)
for txid in txids
}
class LedgerTestCase(AsyncioTestCase):
@@ -111,7 +105,7 @@ class TestSynchronization(LedgerTestCase):
txid2: hexlify(get_transaction(get_output(2)).raw),
txid3: hexlify(get_transaction(get_output(3)).raw),
})
await self.ledger.update_history(None, address, '')
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid1, txid2, txid3])
@@ -126,10 +120,9 @@ class TestSynchronization(LedgerTestCase):
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
await self.ledger.update_history(None, address, '')
for cache_item in self.ledger._tx_cache.values():
cache_item.tx.is_verified = True
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])
@@ -137,7 +130,7 @@ class TestSynchronization(LedgerTestCase):
self.ledger.network.transaction[txid4] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
await self.ledger.update_history(None, address, '')
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid4])
address_details = await self.ledger.db.get_address(address=address)
+16 -18
View File
@@ -427,40 +427,34 @@ class TransactionIOBalancing(AsyncioTestCase):
async def test_basic_use_cases_sqlite(self):
self.ledger.coin_selection_strategy = 'sqlite'
self.ledger.fee_per_byte = int(0.01*CENT)
self.ledger.fee_per_byte = int(.01*CENT)
# available UTXOs for filling missing inputs
utxos = await self.create_utxos([
1, 1, 3, 5, 10
])
self.assertEqual(5, len(await self.ledger.get_utxos()))
# pay 3 coins (3.07 w/ fees)
tx = await self.tx(
[], # inputs
[self.txo(3)] # outputs
)
# there are 4x 1.0 utxos available
self.assertListEqual(self.inputs(tx), [1.0, 1.0, 1.0, 1.0])
await self.ledger.db.db.run(self.ledger.db._transaction_io, tx, tx.outputs[0].get_address(self.ledger), tx.id)
# a change of 0.93 is added to reach balance
self.assertListEqual(self.outputs(tx), [3, 0.93])
self.assertListEqual(self.inputs(tx), [1.0, 1.0, 3.0])
# a change of 1.95 is added to reach balance
self.assertListEqual(self.outputs(tx), [3, 1.95])
# utxos: 1.95, 3, 5, 10
self.assertEqual(2, len(await self.ledger.get_utxos()))
# pay 4.946 coins (5.00 w/ fees)
await self.ledger.release_outputs(utxos)
# pay 6.917 coins (7.00 w/ fees)
tx = await self.tx(
[], # inputs
[self.txo(4.946)] # outputs
[], # inputs
[self.txo(6.917)] # outputs
)
self.assertEqual(1, len(await self.ledger.get_utxos()))
self.assertListEqual(self.inputs(tx), [5.0])
self.assertEqual(2, len(tx.outputs))
self.assertEqual(494600000, tx.outputs[0].amount)
self.assertListEqual(self.inputs(tx), [1.0, 1.0, 1.0, 1.0, 3.0])
self.assertListEqual(self.outputs(tx), [6.92])
# utxos: 3, 1.95, 4.946, 10
await self.ledger.release_outputs(utxos)
# supplied input and output, but input is not enough to cover output
@@ -472,7 +466,9 @@ class TransactionIOBalancing(AsyncioTestCase):
self.assertListEqual([10, 1.0, 1.0], self.inputs(tx))
# change is now needed to consume extra input
self.assertListEqual([11, 0.95], self.outputs(tx))
await self.ledger.release_outputs(utxos)
# liquidating a UTXO
tx = await self.tx(
[self.txi(self.txo(10))], # inputs
@@ -481,7 +477,9 @@ class TransactionIOBalancing(AsyncioTestCase):
self.assertListEqual([10], self.inputs(tx))
# missing change added to consume the amount
self.assertListEqual([9.98], self.outputs(tx))
await self.ledger.release_outputs(utxos)
# liquidating at a loss, requires adding extra inputs
tx = await self.tx(
[self.txi(self.txo(0.01))], # inputs
+2 -44
View File
@@ -38,7 +38,7 @@ class TestWalletCreation(AsyncioTestCase):
'certificates': {},
'name': 'An Account',
'ledger': 'lbc_mainnet',
'modified_on': 123,
'modified_on': 123.456,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
@@ -62,7 +62,7 @@ class TestWalletCreation(AsyncioTestCase):
wallet = Wallet.from_storage(storage, self.manager)
self.assertEqual(wallet.name, 'Main Wallet')
self.assertEqual(
hexlify(wallet.hash), b'869acc4660dde0f13784ed743796adf89562cdf79fdfc9e5c6dbea98d62ccf90'
hexlify(wallet.hash), b'a75913d2e7339c1a9ac0c89d621a4e10fd3a40dc3560dc01f4cf4ada0a0b05b8'
)
self.assertEqual(len(wallet.accounts), 1)
account = wallet.default_account
@@ -74,48 +74,6 @@ class TestWalletCreation(AsyncioTestCase):
decrypted = Wallet.unpack('password', encrypted)
self.assertEqual(decrypted['accounts'][0]['name'], 'An Account')
def test_no_password_but_encryption_preferred(self):
wallet_dict = {
'version': 1,
'name': 'Main Wallet',
'preferences': {
"encrypt-on-disk": {
"ts": 1571762543.351794,
"value": True
},
},
'accounts': [
{
'certificates': {},
'name': 'An Account',
'ledger': 'lbc_mainnet',
'modified_on': 123,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
'encrypted': False,
'private_key':
'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7'
'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe',
'public_key':
'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm'
'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9',
'address_generator': {
'name': 'deterministic-chain',
'receiving': {'gap': 17, 'maximum_uses_per_address': 3},
'change': {'gap': 10, 'maximum_uses_per_address': 3}
}
}
]
}
storage = WalletStorage(default=wallet_dict)
wallet = Wallet.from_storage(storage, self.manager)
self.assertEqual(
hexlify(wallet.hash), b'8cc6341885e6ad46f72a17364c65f8441f09e79996c55202196b399c75f8d751'
)
self.assertFalse(wallet.is_encrypted)
def test_read_write(self):
manager = WalletManager()
config = {'data_path': '/tmp/wallet'}