Compare commits

..
Author SHA1 Message Date
Victor Shyba 24d749ed2b lbry-libtorrent is now on pypi 2020-03-10 21:46:20 -03:00
Victor Shyba bdd843196d use ubuntu 18 on gitlab, temporarly 2020-03-10 20:25:16 -03:00
Victor Shyba dd456347c6 calculate total bytes outside of dict 2020-03-10 20:25:16 -03:00
Victor Shyba e31323fa9d tell progress, stop trying to read first piece 2020-03-10 20:25:16 -03:00
Victor Shyba ed8572a961 wait started event 2020-03-10 20:25:16 -03:00
Victor Shyba 9b9e55d07d find and show largest file 2020-03-10 20:25:16 -03:00
Victor Shyba eb9e913788 fix and test delete with torrents 2020-03-10 20:25:16 -03:00
Victor Shyba 539655292a fix moving to a new btih 2020-03-10 20:25:16 -03:00
Victor Shyba 5a7312cbb8 fix not knowing a torrent exists 2020-03-10 20:25:16 -03:00
Victor Shyba a5a4b224e9 gitlab: use ubuntu on datanetwork tests 2020-03-10 20:25:16 -03:00
Victor Shyba 7f88fcd1b5 add boost on gitlab, fix failing test, add libtorrent to linux build 2020-03-10 20:25:16 -03:00
Victor Shyba 7e2381816d correct wheel with boost 1.65 2020-03-10 20:25:16 -03:00
Victor Shyba 3f2346eb01 linting and minor refactor 2020-03-10 20:25:16 -03:00
Victor Shyba 260a713b08 fixes from rebase, install libtorrent from s3 2020-03-10 20:25:16 -03:00
Victor Shyba f6b4fee814 pylint 2020-03-10 20:25:16 -03:00
Victor Shyba e9623fae13 working file list after torrent get 2020-03-10 20:25:16 -03:00
Victor Shyba a1090679a5 torrent test and misc fixes 2020-03-10 20:25:16 -03:00
Victor Shyba ad66358d18 fix torrent and stream manager reference leftovers 2020-03-10 20:25:16 -03:00
Victor Shyba 34c6e09e6f adds more torrent parts 2020-03-10 20:25:16 -03:00
Victor Shyba 507db5f79a torrent manager and torrent source 2020-03-10 20:25:16 -03:00
Victor Shyba e6663603a6 fix unit tests 2020-03-10 20:25:16 -03:00
Victor Shyba 899852d8ba pylint 2020-03-10 20:25:16 -03:00
Victor Shyba 41175a814b fix save from resolve 2020-03-10 20:25:16 -03:00
Victor Shyba 78faf4e691 stream manager component becomes file manager component 2020-03-10 20:25:15 -03:00
Victor Shyba 7e1795b7b1 wip 2020-03-10 20:25:15 -03:00
Victor Shyba 837dc1b078 add torrent component 2020-03-10 20:25:15 -03:00
Jack Robison 8d0f31e466 add lbry.torrent 2020-03-10 20:25:15 -03:00
Jack Robison 404f0dcfe0 file manager refactor 2020-03-10 20:25:15 -03:00
Jack Robison c9b868a238 ManagedDownloadSource and SourceManager refactor 2020-03-10 20:25:15 -03:00
97 changed files with 1654 additions and 6266 deletions

No files matched your search

+5 -10
View File
@@ -6,7 +6,7 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -17,7 +17,7 @@ jobs:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -37,14 +37,12 @@ jobs:
- blockchain
- other
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
run: sudo apt install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
@@ -59,7 +57,7 @@ jobs:
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -67,9 +65,6 @@ jobs:
run: |
pip install pyinstaller
pip install -e .
# https://stackoverflow.com/a/61693590
# https://github.com/pypa/setuptools/issues/1963
pip install --upgrade 'setuptools<45.0.0'
- if: startsWith(matrix.os, 'windows') == false
name: Build & Run (Unix)
run: |
+1 -4
View File
@@ -14,7 +14,4 @@ _trial_temp/
/tests/integration/blockchain/files
/tests/.coverage.*
/lbry/wallet/bin
/.vscode
/.gitignore
/lbry/wallet/bin
+1 -1
View File
@@ -9,7 +9,7 @@ install:
pip install -e .
tools:
pip install mypy==0.701 pylint==2.4.4
pip install mypy==0.701
pip install coverage astroid pylint
lint:
+172 -783
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.79.0"
__version__ = "0.64.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+9 -23
View File
@@ -110,7 +110,7 @@ class AbstractBlob:
if reader in self.readers:
self.readers.remove(reader)
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
def set_length(self, length) -> None:
@@ -198,17 +198,11 @@ class AbstractBlob:
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
def update_events(_):
self.verified.set()
self.writing.clear()
if self.is_writeable():
self.writing.set()
task = self._write_blob(verified_bytes)
task.add_done_callback(update_events)
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
task.add_done_callback(lambda _: self.blob_completed_callback(self))
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
@@ -267,11 +261,9 @@ class BlobBuffer(AbstractBlob):
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
async def write():
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
return self.loop.create_task(write())
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
@@ -327,14 +319,8 @@ class BlobFile(AbstractBlob):
handle.close()
def _write_blob(self, blob_bytes: bytes):
def _write_blob():
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
async def write_blob():
await self.loop.run_in_executor(None, _write_blob)
return self.loop.create_task(write_blob())
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def delete(self):
if os.path.isfile(self.file_path):
-2
View File
@@ -152,8 +152,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
# wait for the io to finish
await self.blob.verified.wait()
log.info("%s at %fMB/s", msg,
round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0, 2))
+20 -38
View File
@@ -5,8 +5,9 @@ import typing
import logging
from argparse import ArgumentParser
from contextlib import contextmanager
from appdirs import user_data_dir, user_config_dir
import yaml
from appdirs import user_data_dir, user_config_dir
from lbry.error import InvalidCurrencyError
from lbry.dht import constants
from lbry.wallet.coinselection import STRATEGIES
@@ -276,23 +277,14 @@ class Strings(ListSetting):
class EnvironmentAccess:
PREFIX = 'LBRY_'
def __init__(self, config: 'BaseConfig', environ: dict):
self.configuration = config
self.data = {}
if environ:
self.load(environ)
def load(self, environ):
for setting in self.configuration.get_settings():
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
self.data[setting.name] = setting.deserialize(value)
def __init__(self, environ: dict):
self.environ = environ
def __contains__(self, item: str):
return item in self.data
return f'{self.PREFIX}{item.upper()}' in self.environ
def __getitem__(self, item: str):
return self.data[item]
return self.environ[f'{self.PREFIX}{item.upper()}']
class ArgumentAccess:
@@ -333,7 +325,7 @@ class ConfigFileAccess:
cls = type(self.configuration)
with open(self.path, 'r') as config_file:
raw = config_file.read()
serialized = yaml.safe_load(raw) or {}
serialized = yaml.load(raw) or {}
for key, value in serialized.items():
attr = getattr(cls, key, None)
if attr is None:
@@ -451,7 +443,7 @@ class BaseConfig:
self.arguments = ArgumentAccess(self, args)
def set_environment(self, environ=None):
self.environment = EnvironmentAccess(self, environ or os.environ)
self.environment = EnvironmentAccess(environ or os.environ)
def set_persisted(self, config_file_path=None):
if config_file_path is None:
@@ -472,21 +464,19 @@ class BaseConfig:
class TranscodeConfig(BaseConfig):
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
'', previous_names=['ffmpeg_folder'])
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
r'-maxrate 5500K -bufsize 5000K')
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
@@ -585,14 +575,9 @@ class Config(CLIConfig):
)
# servers
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.com', 5566)
])
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
('cdn.reflector.lbry.com', 5567)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv11.lbry.com', 50001),
('spv12.lbry.com', 50001),
@@ -615,6 +600,8 @@ class Config(CLIConfig):
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
@@ -624,9 +611,6 @@ class Config(CLIConfig):
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
allowed_origin = String(
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
"all hosts; default is to only allow API requests with no `Origin` value.", "")
# media server
streaming_server = String('Host name and port to serve streaming media over range requests',
@@ -638,7 +622,6 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
transaction_cache_size = Integer("Transaction cache size", 100_000)
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
@@ -708,10 +691,9 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
if down_dir:
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
download_dir = re.sub('\"', '', down_dir)
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
+6 -8
View File
@@ -33,14 +33,12 @@ def is_valid_public_ipv4(address, allow_localhost: bool = False):
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
parsed_ip.is_reserved,
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except ipaddress.AddressValueError:
return False
+1 -2
View File
@@ -10,7 +10,6 @@ from asyncio.protocols import DatagramProtocol
from asyncio.transports import DatagramTransport
from lbry.dht import constants
from lbry.dht.serialization.bencoding import DecodeError
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
from lbry.dht.error import RemoteException, TransportNotConnected
@@ -555,7 +554,7 @@ class KademliaProtocol(DatagramProtocol):
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
try:
message = decode_datagram(datagram)
except (ValueError, TypeError, DecodeError):
except (ValueError, TypeError):
self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
+5
View File
@@ -14,6 +14,7 @@ from aiohttp.web import GracefulExit
from docopt import docopt
from lbry import __version__ as lbrynet_version
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.extras.daemon.daemon import Daemon
from lbry.conf import Config, CLIConfig
@@ -254,6 +255,10 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
else:
logger.getChild('lbry').setLevel(logging.DEBUG)
loggly_handler = get_loggly_handler(conf)
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config):
loop = asyncio.get_event_loop()
+1 -1
View File
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
"node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.fixed_peers) > 0,
"use_fixed_peers": len(conf.reflector_servers) > 0,
"fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count,
+2 -5
View File
@@ -158,14 +158,11 @@ class ComponentManager:
for component in self.components
}
def get_actual_component(self, component_name):
def get_component(self, component_name):
for component in self.components:
if component.component_name == component_name:
return component
return component.component
raise NameError(component_name)
def get_component(self, component_name):
return self.get_actual_component(component_name).component
def has_component(self, component_name):
return any(component for component in self.components if component_name == component.component_name)
+1 -1
View File
@@ -150,7 +150,7 @@ class WalletComponent(Component):
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
else:
progress = 100
best_hash = await self.wallet_manager.get_best_blockhash()
best_hash = self.wallet_manager.get_best_blockhash()
result.update({
'headers_synchronization_progress': progress,
'blocks': max(local_height, 0),
File diff suppressed because it is too large. Load diff
+45 -86
View File
@@ -7,7 +7,6 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError
from lbry.schema.claim import Claim
from lbry.schema.support import Support
from lbry.torrent.torrent_manager import TorrentSource
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
from lbry.wallet.bip32 import PubKey
@@ -110,9 +109,7 @@ def encode_file_doc():
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
'claim_name': '(str) None if claim is not found else the claim name'
}
@@ -128,7 +125,9 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_account(obj)
if isinstance(obj, Wallet):
return self.encode_wallet(obj)
if isinstance(obj, (ManagedStream, TorrentSource)):
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, TorrentSource):
return self.encode_file(obj)
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
@@ -136,8 +135,6 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_output(obj)
if isinstance(obj, Claim):
return self.encode_claim(obj)
if isinstance(obj, Support):
return obj.to_dict()
if isinstance(obj, PubKey):
return obj.extended_key_string()
if isinstance(obj, datetime):
@@ -172,22 +169,16 @@ class JSONResponseEncoder(JSONEncoder):
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger) if txo.has_address else None,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_received is not None:
output['is_received'] = txo.is_received
if txo.is_spent is not None:
output['is_spent'] = txo.is_spent
if txo.is_my_output is not None:
output['is_my_output'] = txo.is_my_output
if txo.is_my_input is not None:
output['is_my_input'] = txo.is_my_input
if txo.sent_supports is not None:
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
if txo.sent_tips is not None:
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
if txo.received_tips is not None:
output['received_tips'] = dewies_to_lbc(txo.received_tips)
if txo.is_internal_transfer is not None:
output['is_internal_transfer'] = txo.is_internal_transfer
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_name:
output['type'] = 'claim'
@@ -223,28 +214,25 @@ class JSONResponseEncoder(JSONEncoder):
output['claims'] = [self.encode_output(o) for o in txo.claims]
if txo.reposted_claim is not None:
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
try:
output['value'] = txo.signable
if self.include_protobuf:
output['protobuf'] = hexlify(txo.signable.to_bytes())
if txo.purchase_receipt is not None:
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
if txo.script.is_claim_name or txo.script.is_update_claim:
if txo.script.is_claim_name or txo.script.is_update_claim:
try:
output['value'] = txo.claim
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.purchase_receipt is not None:
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
if txo.claim.is_channel:
output['has_signing_key'] = txo.has_private_key
elif txo.script.is_support_claim_data:
output['value_type'] = 'emoji'
if check_signature and txo.signable.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
output['is_channel_signature_valid'] = False
except DecodeError:
pass
if check_signature and txo.claim.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
output['is_channel_signature_valid'] = False
except DecodeError:
pass
return output
def encode_claim_meta(self, meta):
@@ -253,7 +241,7 @@ class JSONResponseEncoder(JSONEncoder):
if isinstance(value, int):
meta[key] = dewies_to_lbc(value)
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
return meta
def encode_input(self, txi):
@@ -286,26 +274,26 @@ class JSONResponseEncoder(JSONEncoder):
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
else:
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
result = {
'streaming_url': None,
return {
'streaming_url': managed_stream.stream_url if is_stream else f'file://{managed_stream.full_path}',
'completed': managed_stream.completed,
'file_name': None,
'download_directory': None,
'download_path': None,
'file_name': managed_stream.file_name if output_exists else None,
'download_directory': managed_stream.download_directory if output_exists else None,
'download_path': managed_stream.full_path if output_exists else None,
'points_paid': 0.0,
'stopped': not managed_stream.running,
'stream_hash': None,
'stream_name': None,
'suggested_file_name': None,
'sd_hash': None,
'mime_type': None,
'key': None,
'stream_hash': managed_stream.stream_hash if is_stream else None,
'stream_name': managed_stream.descriptor.stream_name if is_stream else None,
'suggested_file_name': managed_stream.descriptor.suggested_file_name if is_stream else None,
'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None,
'mime_type': managed_stream.mime_type if is_stream else None,
'key': managed_stream.descriptor.key if is_stream else None,
'total_bytes_lower_bound': total_bytes_lower_bound,
'total_bytes': total_bytes,
'written_bytes': managed_stream.written_bytes,
'blobs_completed': None,
'blobs_in_stream': None,
'blobs_remaining': None,
'written_bytes': managed_stream.written_bytes if is_stream else managed_stream.written_bytes,
'blobs_completed': managed_stream.blobs_completed if is_stream else None,
'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None,
'blobs_remaining': managed_stream.blobs_remaining if is_stream else None,
'status': managed_stream.status,
'claim_id': managed_stream.claim_id,
'txid': managed_stream.txid,
@@ -321,38 +309,9 @@ class JSONResponseEncoder(JSONEncoder):
'added_on': managed_stream.added_on,
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': False,
'reflector_progress': False,
'uploading_to_reflector': False
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
'is_fully_reflected': managed_stream.is_fully_reflected if is_stream else False
}
if is_stream:
result.update({
'streaming_url': managed_stream.stream_url,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.descriptor.stream_name,
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
})
else:
result.update({
'streaming_url': f'file://{managed_stream.full_path}',
})
if output_exists:
result.update({
'file_name': managed_stream.file_name,
'download_directory': managed_stream.download_directory,
'download_path': managed_stream.full_path,
})
return result
def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict()
+95
View File
@@ -0,0 +1,95 @@
import asyncio
import json
import logging.handlers
import traceback
import typing
from aiohttp.client_exceptions import ClientError
import aiohttp
from lbry import utils, __version__
if typing.TYPE_CHECKING:
from lbry.conf import Config
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
super().__init__()
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, config: 'Config'):
super().__init__()
self.cookies = {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
self._config = config
@property
def enabled(self):
return self._config.share_usage_data
@staticmethod
def get_full_message(record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
async def _emit(self, record, retry=True):
data = self.format(record).encode()
try:
async with self._session.post(self.url, data=data,
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
if self._loop.is_running() and retry and self.enabled:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
if not self.enabled:
return
try:
asyncio.ensure_future(self._emit(record), loop=self._loop)
except RuntimeError: # TODO: use a second loop
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
f"{self.format(record)}")
def close(self):
super().close()
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(self._session.close())
except RuntimeError:
pass
def get_loggly_handler(config):
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
handler.setFormatter(JsonFormatter())
return handler
-31
View File
@@ -1,31 +0,0 @@
import logging
from aiohttp import web
log = logging.getLogger(__name__)
def ensure_request_allowed(request, conf):
if is_request_allowed(request, conf):
return
if conf.allowed_origin:
log.warning(
"API requests with Origin '%s' are not allowed, "
"configuration 'allowed_origin' limits requests to: '%s'",
request.headers.get('Origin'), conf.allowed_origin
)
else:
log.warning(
"API requests with Origin '%s' are not allowed, "
"update configuration 'allowed_origin' to enable this origin.",
request.headers.get('Origin')
)
raise web.HTTPForbidden()
def is_request_allowed(request, conf) -> bool:
origin = request.headers.get('Origin')
return (
origin is None or
origin == conf.allowed_origin or
conf.allowed_origin == '*'
)
+1 -5
View File
@@ -85,7 +85,7 @@ class FileManager:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(
self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
self.wallet_manager.ledger.resolve(wallet.accounts, [uri]),
resolve_timeout
)
except asyncio.TimeoutError:
@@ -142,8 +142,6 @@ class FileManager:
log.info("claim contains an update to a stream we have, downloading it")
if save_file and existing_for_claim_id[0].output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
existing_for_claim_id[0].downloader.node = source_manager.node
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
if not existing_for_claim_id[0].output_file_exists and (
save_file or file_name or download_directory):
@@ -157,8 +155,6 @@ class FileManager:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
updated_stream.downloader.node = source_manager.node
await updated_stream.start(timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
+11 -34
View File
@@ -27,7 +27,6 @@ class SourceManager:
'status',
'file_name',
'added_on',
'download_path',
'claim_name',
'claim_height',
'claim_id',
@@ -35,14 +34,7 @@ class SourceManager:
'txid',
'nout',
'channel_claim_id',
'channel_name',
'completed'
}
set_filter_fields = {
"claim_ids": "claim_id",
"channel_claim_ids": "channel_claim_id",
"outpoints": "outpoint"
'channel_name'
}
source_class = ManagedDownloadSource
@@ -103,36 +95,21 @@ class SourceManager:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in self.filter_fields:
raise ValueError(f"'{search}' is not a valid search operation")
compare_sets = {}
if isinstance(search_by.get('claim_id'), list):
compare_sets['claim_ids'] = search_by.pop('claim_id')
if isinstance(search_by.get('outpoint'), list):
compare_sets['outpoints'] = search_by.pop('outpoint')
if isinstance(search_by.get('channel_claim_id'), list):
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
if search_by or compare_sets:
if search_by:
comparison = comparison or 'eq'
streams = []
sources = []
for stream in self._sources.values():
if compare_sets and not all(
getattr(stream, self.set_filter_fields[set_search]) in val
for set_search, val in compare_sets.items()):
continue
if search_by and not all(
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
for search, val in search_by.items()):
continue
streams.append(stream)
for search, val in search_by.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
sources.append(stream)
break
else:
streams = list(self._sources.values())
sources = list(self._sources.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
sources.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
sources.reverse()
return sources
+72 -148
View File
@@ -3,17 +3,16 @@ import json
import logging
import os
import pathlib
import platform
import re
import shlex
import shutil
import subprocess
from math import ceil
import platform
import lbry.utils
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
DISABLED = platform.system() == "Windows"
class VideoFileAnalyzer:
@@ -27,95 +26,63 @@ class VideoFileAnalyzer:
def __init__(self, conf: TranscodeConfig):
self._conf = conf
self._available_encoders = ""
self._ffmpeg_installed = None
self._which_ffmpeg = None
self._which_ffprobe = None
self._env_copy = dict(os.environ)
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
self._env_copy = dict(os.environ)
if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment:
self._replace_or_pop_env('LD_LIBRARY_PATH')
@staticmethod
def _execute(command, environment):
# log.debug("Executing: %s", command)
async def _execute(self, command, arguments):
if DISABLED:
return "Disabled on Windows", -1
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
async def _verify_executable(self, name):
try:
with subprocess.Popen(
shlex.split(command) if platform.system() != 'Windows' else command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
) as process:
(stdout, stderr) = process.communicate() # blocks until the process exits
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
except subprocess.SubprocessError as e:
return str(e), -1
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
async def _execute_ffmpeg(self, arguments):
arguments = self._which_ffmpeg + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _execute_ffprobe(self, arguments):
arguments = self._which_ffprobe + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _verify_executables(self):
try:
await self._execute_ffprobe("-version")
version, code = await self._execute_ffmpeg("-version")
version, code = await self._execute(name, "-version")
except Exception as e:
code = -1
version = str(e)
if code != 0 or not version.startswith("ffmpeg"):
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
raise FileNotFoundError(f"Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_path")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
if code != 0 or not version.startswith(name):
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
return version
@staticmethod
def _which_ffmpeg_and_ffmprobe(path):
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
async def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed:
return
self._ffmpeg_installed = False
path = self._conf.ffmpeg_path
if hasattr(self._conf, "data_dir"):
path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
path += os.path.pathsep + self._env_copy.get("PATH", "")
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
None, self._which_ffmpeg_and_ffmprobe, path
)
if not self._which_ffmpeg:
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
if not self._which_ffprobe:
log.warning("Unable to locate ffprobe executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
log.warning("ffmpeg and ffprobe are in different folders!")
await self._verify_executables()
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def status(self, reset=False, recheck=False):
if reset:
self._available_encoders = ""
self._ffmpeg_installed = None
self._ffmpeg_installed = False
self._which = None
if self._checked_ffmpeg and not recheck:
pass
elif self._ffmpeg_installed is None:
installed = self._ffmpeg_installed
else:
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
pass
installed = False
self._checked_ffmpeg = True
return {
"available": self._ffmpeg_installed,
"which": self._which_ffmpeg,
"available": installed,
"which": self._which,
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
}
@@ -123,24 +90,9 @@ class VideoFileAnalyzer:
def _verify_container(scan_data: json):
container = scan_data["format"]["format_name"]
log.debug(" Detected container is %s", container)
splits = container.split(",")
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
return "Container format is not in the approved list of WebM, MP4. " \
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
if "matroska" in splits:
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
codec = stream["codec_name"]
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
return "WebM format requires VP8/9 or AV1 video. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
elif stream["codec_type"] == "audio":
codec = stream["codec_name"]
if not {"vorbis", "opus"}.intersection(codec.split(",")):
return "WebM format requires Vorbis or Opus audio. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
return ""
@staticmethod
@@ -169,12 +121,12 @@ class VideoFileAnalyzer:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
return ""
@@ -183,7 +135,7 @@ class VideoFileAnalyzer:
if {"webm", "ogg"}.intersection(container.split(",")):
return ""
result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
if match and int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
@@ -199,8 +151,6 @@ class VideoFileAnalyzer:
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
if int(stream['sample_rate']) > 48000:
return "Sample rate out of range"
return ""
@@ -213,8 +163,8 @@ class VideoFileAnalyzer:
if not validate_volume:
return ""
result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
try:
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
@@ -249,7 +199,7 @@ class VideoFileAnalyzer:
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
if not self._available_encoders:
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
encoder = self._conf.video_encoder.split(" ", 1)[0]
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
@@ -283,7 +233,7 @@ class VideoFileAnalyzer:
wants_opus = extension != "mp4"
if not self._available_encoders:
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
encoder = self._conf.audio_encoder.split(" ", 1)[0]
if wants_opus and 'opus' in encoder:
@@ -306,6 +256,9 @@ class VideoFileAnalyzer:
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
async def _get_volume_filter(self):
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
@staticmethod
def _get_best_container_extension(scan_data, video_encoder):
# the container is chosen by the video format
@@ -313,27 +266,25 @@ class VideoFileAnalyzer:
# if we are vp8/vp9/av1 we want webm
# use mp4 for anything else
if video_encoder: # not re-encoding video
if "theora" in video_encoder:
return "ogv"
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if not video_encoder: # not re-encoding video
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogv"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
async def _get_scan_data(self, validate, file_path):
arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
result, _ = await self._execute_ffprobe(arguments)
result, _ = await self._execute("ffprobe",
f'-v quiet -print_format json -show_format -show_streams "{file_path}"')
try:
scan_data = json.loads(result)
except Exception as e:
@@ -342,7 +293,7 @@ class VideoFileAnalyzer:
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content: {file_path}')
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
@@ -350,46 +301,20 @@ class VideoFileAnalyzer:
return scan_data
@staticmethod
def _build_spec(scan_data):
assert scan_data
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
width = -1
height = -1
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
width = max(width, int(stream["width"]))
height = max(height, int(stream["height"]))
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
spec = {"duration": duration}
if height >= 0:
spec["height"] = height
if width >= 0:
spec["width"] = width
return spec
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair:
return file_path, {}
if ignore_non_video and not file_path:
return file_path, {}
return file_path
await self._verify_ffmpeg_installed()
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path, {}
return file_path
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
spec = self._build_spec(scan_data)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
@@ -399,7 +324,7 @@ class VideoFileAnalyzer:
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
if not any(messages):
return file_path, spec
return file_path
if not repair:
errors = ["Streamability verification failed:"]
@@ -409,7 +334,7 @@ class VideoFileAnalyzer:
# the plan for transcoding:
# we have to re-encode the video if it is in a nonstandard format
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
# we also re-encode if our bitrate or sample rate is too high
# we also re-encode if our bitrate is too high
try:
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
@@ -424,26 +349,25 @@ class VideoFileAnalyzer:
transcode_command.append("copy")
transcode_command.append("-movflags +faststart -c:a")
path = pathlib.Path(file_path)
extension = self._get_best_container_extension(scan_data, video_encoder)
if audio_msg or volume_msg:
audio_encoder = await self._get_audio_encoder(extension)
transcode_command.append(audio_encoder)
if volume_msg and self._conf.volume_filter:
transcode_command.append(self._conf.volume_filter)
if audio_msg == "Sample rate out of range":
transcode_command.append(" -ar 48000 ")
if volume_msg:
volume_filter = await self._get_volume_filter()
transcode_command.append(volume_filter)
else:
transcode_command.append("copy")
# TODO: put it in a temp folder and delete it after we upload?
path = pathlib.Path(file_path)
output = path.parent / f"{path.stem}_fixed.{extension}"
transcode_command.append(f'"{output}"')
ffmpeg_command = " ".join(transcode_command)
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
result, code = await self._execute_ffmpeg(ffmpeg_command)
result, code = await self._execute("ffmpeg", ffmpeg_command)
if code != 0:
raise Exception(f"Failure to complete the transcode command. Output: {result}")
except Exception as e:
@@ -451,6 +375,6 @@ class VideoFileAnalyzer:
raise
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
# TODO: delete partial output file here if it exists?
return file_path, spec
return file_path
return str(output), spec
return str(output)
-32
View File
@@ -1,32 +0,0 @@
import logging
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
async def start(self, interface: str, port: int):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
await self.runner.cleanup()
+6 -2
View File
@@ -30,10 +30,14 @@ class Claim(Signable):
COLLECTION = 'collection'
REPOST = 'repost'
__slots__ = ()
__slots__ = 'version',
message_class = ClaimMessage
def __init__(self, message=None):
super().__init__(message)
self.version = 2
@property
def claim_type(self) -> str:
return self.message.WhichOneof('type')
@@ -249,7 +253,7 @@ class Stream(BaseClaim):
if stream_type in ('image', 'video', 'audio'):
media = getattr(self, stream_type)
media_args = {'file_metadata': None}
if file_path is not None and not all((duration, width, height)):
if file_path is not None:
try:
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
except:
+1 -3
View File
@@ -47,7 +47,7 @@ types_map = {
'.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'video'),
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
'.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'),
@@ -61,7 +61,6 @@ types_map = {
'.mp3': ('audio/mpeg', 'audio'),
'.mp4': ('video/mp4', 'video'),
'.mpa': ('video/mpeg', 'video'),
'.mpd': ('application/dash+xml', 'video'),
'.mpe': ('video/mpeg', 'video'),
'.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'),
@@ -142,7 +141,6 @@ types_map = {
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.flac': ('audio/flac', 'audio'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4a': ('audio/mp4', 'audio'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
+1 -10
View File
@@ -1,15 +1,6 @@
from lbry.schema.base import Signable
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
class Support(Signable):
__slots__ = ()
message_class = SupportMessage
@property
def emoji(self) -> str:
return self.message.emoji
@emoji.setter
def emoji(self, emoji: str):
self.message.emoji = emoji
message_class = None # TODO: add support protobufs
-69
View File
@@ -1,69 +0,0 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: support.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='support.proto',
package='pb',
syntax='proto3',
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\"\x18\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\tb\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_SUPPORT = _descriptor.Descriptor(
name='Support',
full_name='pb.Support',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='emoji', full_name='pb.Support.emoji', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=21,
serialized_end=45,
)
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
DESCRIPTOR = _SUPPORT,
__module__ = 'support_pb2'
# @@protoc_insertion_point(class_scope:pb.Support)
))
_sym_db.RegisterMessage(Support)
# @@protoc_insertion_point(module_scope)
+10 -17
View File
@@ -44,25 +44,18 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
yield os.urandom(AES.block_size // 8)
def read_bytes(file_path: str, offset: int, to_read: int):
with open(file_path, 'rb') as f:
f.seek(offset)
return f.read(to_read)
async def file_reader(file_path: str):
def file_reader(file_path: str):
length = int(os.stat(file_path).st_size)
offset = 0
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = await asyncio.get_event_loop().run_in_executor(
None, read_bytes, file_path, offset, bytes_to_read
)
yield blob_bytes
offset += bytes_to_read
with open(file_path, 'rb') as stream_file:
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = stream_file.read(bytes_to_read)
yield blob_bytes
offset += bytes_to_read
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
@@ -252,7 +245,7 @@ class StreamDescriptor:
iv_generator = iv_generator or random_iv_generator()
key = key or os.urandom(AES.block_size // 8)
blob_num = -1
async for blob_bytes in file_reader(file_path):
for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
+4 -4
View File
@@ -51,15 +51,15 @@ class StreamDownloader:
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
for address, port in addresses
])
if not self.config.fixed_peers:
if not self.config.reflector_servers:
return
addresses = [
(await resolve_host(url, port, proto='tcp'), port)
for url, port in self.config.fixed_peers
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()) > 0:
+14 -28
View File
@@ -56,11 +56,8 @@ class ManagedStream(ManagedDownloadSource):
self.purchase_receipt = None
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.file_output_task: Optional[asyncio.Task] = None
self.delayed_stop_task: Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
self.fully_reflected = asyncio.Event(loop=self.loop)
self.streaming = asyncio.Event(loop=self.loop)
@@ -120,10 +117,6 @@ class ManagedStream(ManagedDownloadSource):
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
@property
def download_path(self):
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
# @classmethod
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
# file_path: str, key: Optional[bytes] = None,
@@ -197,13 +190,13 @@ class ManagedStream(ManagedDownloadSource):
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request) -> StreamResponse:
async def stream_file(self, request: Request, node: Optional['Node'] = None) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
await self.start()
await self.start(node)
response = StreamResponse(
status=206,
headers=headers
@@ -241,10 +234,9 @@ class ManagedStream(ManagedDownloadSource):
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(output_path: str, data: bytes):
with open(output_path, 'ab') as handle:
handle.write(data)
handle.flush()
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
@@ -254,12 +246,12 @@ class ManagedStream(ManagedDownloadSource):
self.finished_writing.clear()
self.started_writing.clear()
try:
open(output_path, 'wb').close()
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
@@ -332,7 +324,6 @@ class ManagedStream(ManagedDownloadSource):
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
@@ -348,21 +339,16 @@ class ManagedStream(ManagedDownloadSource):
]
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
self.claim_name, self.claim_id)
for i, blob_hash in enumerate(we_have):
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
return sent
except OSError:
# raised if a blob is deleted while it's being sent
return sent
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+5 -13
View File
@@ -35,8 +35,6 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None
self.connected.clear()
if self.pending_request:
self.pending_request.cancel()
if self.reflected_blobs:
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
@@ -58,18 +56,12 @@ class StreamReflectorClient(asyncio.Protocol):
self.response_buff = b''
return
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
async def send_request(self, request_dict: typing.Dict):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
self.pending_request = self.loop.create_task(self.response_queue.get())
return await self.pending_request
except (AttributeError, asyncio.CancelledError):
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise asyncio.TimeoutError()
finally:
self.pending_request = None
@@ -95,7 +87,7 @@ class StreamReflectorClient(asyncio.Protocol):
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
received = await self.response_queue.get()
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -119,7 +111,7 @@ class StreamReflectorClient(asyncio.Protocol):
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
received = await self.response_queue.get()
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+6 -40
View File
@@ -15,9 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -29,25 +27,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.descriptor: typing.Optional['StreamDescriptor'] = None
self.sd_blob: typing.Optional['BlobFile'] = None
self.received = []
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
self.incoming = asyncio.Event(loop=self.loop)
self.chunk_size = response_chunk_size
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
async def wait_for_stop(self):
await self.stop_event.wait()
if self.transport:
self.transport.close()
def connection_made(self, transport):
self.transport = transport
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
def connection_lost(self, exc):
if self.wait_for_stop_task:
self.wait_for_stop_task.cancel()
self.wait_for_stop_task = None
def data_received(self, data: bytes):
if self.incoming.is_set():
@@ -89,7 +73,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set()
self.send_response({"send_sd_blob": True})
try:
@@ -103,7 +86,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.transport.close()
finally:
self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle()
self.writer = None
else:
@@ -111,7 +93,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.not_incoming.set()
if self.writer:
self.writer.close_handle()
self.writer = None
@@ -131,7 +112,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified():
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set()
self.send_response({"send_blob": True})
try:
@@ -140,7 +120,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle()
self.writer = None
else:
@@ -151,18 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.stopped_listening = asyncio.Event(loop=self.loop)
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
self.response_chunk_size = response_chunk_size
self.stop_event = stop_event
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -170,20 +143,13 @@ class ReflectorServer:
async def _start_server():
server = await self.loop.create_server(
lambda: ReflectorServerProtocol(
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
self.not_incoming_event
),
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
interface, port
)
self.started_listening.set()
self.stopped_listening.clear()
log.info("Reflector server listening on TCP %s:%i", interface, port)
try:
async with server:
await server.serve_forever()
finally:
self.stopped_listening.set()
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
+2 -21
View File
@@ -38,9 +38,7 @@ class StreamManager(SourceManager):
'stream_hash',
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream',
'uploading_to_reflector',
'is_fully_reflected'
'blobs_in_stream'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
@@ -159,18 +157,8 @@ class StreamManager(SourceManager):
))
async def reflect_streams(self):
try:
return await self._reflact_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
async def _reflact_streams(self):
# todo: those debug statements are temporary for #2987 - remove them if its closed
while True:
if self.config.reflect_streams and self.config.reflector_servers:
log.debug("collecting streams to reflect")
sd_hashes = await self.storage.get_streams_to_re_reflect()
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
batch = []
@@ -181,14 +169,10 @@ class StreamManager(SourceManager):
stream.fully_reflected.is_set():
batch.append(self.reflect_stream(stream))
if len(batch) >= self.config.concurrent_reflector_uploads:
log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.gather(*batch, loop=self.loop)
log.debug("done processing %s streams", len(batch))
batch = []
if batch:
log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.gather(*batch, loop=self.loop)
log.debug("done processing %s streams", len(batch))
await asyncio.sleep(300, loop=self.loop)
async def start(self):
@@ -261,7 +245,4 @@ class StreamManager(SourceManager):
os.remove(source.full_path)
async def stream_partial_content(self, request: Request, sd_hash: str):
stream = self._sources[sd_hash]
if not stream.downloader.node:
stream.downloader.node = self.node
return await stream.stream_file(request)
return await self._sources[sd_hash].stream_file(request, self.node)
+4 -29
View File
@@ -386,7 +386,6 @@ class CommandTestCase(IntegrationTestCase):
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.fixed_peers = [('127.0.0.1', 5567)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.components_to_skip = [
@@ -491,8 +490,8 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
)
async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
return await self.confirm_and_render(
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
return self.confirm_and_render(
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
)
@@ -503,11 +502,6 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
)
async def purchase_create(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
)
async def publish(self, name, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
@@ -561,21 +555,8 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
)
async def wallet_send(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
)
async def txo_spend(self, *args, confirm=True, **kwargs):
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
if confirm:
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
await self.generate(1)
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
return self.sout(txs)
async def resolve(self, uri, **kwargs):
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
async def resolve(self, uri):
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
@@ -586,12 +567,6 @@ class CommandTestCase(IntegrationTestCase):
async def txo_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
async def txo_sum(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
async def txo_plot(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
-23
View File
@@ -3,7 +3,6 @@ import codecs
import datetime
import random
import socket
import time
import string
import sys
import json
@@ -283,25 +282,3 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
def is_running_from_bundle():
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
class LockWithMetrics(asyncio.Lock):
def __init__(self, acquire_metric, held_time_metric, loop=None):
super().__init__(loop=loop)
self._acquire_metric = acquire_metric
self._lock_held_time_metric = held_time_metric
self._lock_acquired_time = None
async def acquire(self):
start = time.perf_counter()
try:
return await super().acquire()
finally:
self._lock_acquired_time = time.perf_counter()
self._acquire_metric.observe(self._lock_acquired_time - start)
def release(self):
try:
return super().release()
finally:
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+26 -35
View File
@@ -71,7 +71,6 @@ class AddressManager:
def _query_addresses(self, **constraints):
return self.account.ledger.db.get_addresses(
read_only=constraints.pop("read_only", False),
accounts=[self.account],
chain=self.chain_number,
**constraints
@@ -311,7 +310,7 @@ class Account:
private_key=private_key,
public_key=public_key,
address_generator=d.get('address_generator', {}),
modified_on=d.get('modified_on', int(time.time())),
modified_on=d.get('modified_on', time.time()),
channel_keys=d.get('certificates', {})
)
@@ -343,7 +342,7 @@ class Account:
def merge(self, d: dict):
if d.get('modified_on', 0) > self.modified_on:
self.name = d['name']
self.modified_on = d.get('modified_on', int(time.time()))
self.modified_on = d.get('modified_on', time.time())
assert self.address_generator.name == d['address_generator']['name']
for chain_name in ('change', 'receiving'):
if chain_name in d['address_generator']:
@@ -435,9 +434,9 @@ class Account:
addresses.extend(new_addresses)
return addresses
async def get_addresses(self, read_only=False, **constraints) -> List[str]:
rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
return [r['address'] for r in rows]
async def get_addresses(self, **constraints) -> List[str]:
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
return [r[0] for r in rows]
def get_address_records(self, **constraints):
return self.ledger.db.get_addresses(accounts=[self], **constraints)
@@ -452,13 +451,13 @@ class Account:
def get_public_key(self, chain: int, index: int) -> PubKey:
return self.address_managers[chain].get_public_key(index)
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
if not include_claims:
constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
if confirmations > 0:
height = self.ledger.headers.height - (confirmations-1)
constraints.update({'height__lte': height, 'height__gt': 0})
return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
return self.ledger.db.get_balance(accounts=[self], **constraints)
async def get_max_gap(self):
change_gap = await self.change.get_max_gap()
@@ -525,18 +524,13 @@ class Account:
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
async def get_channel_private_key(self, public_key_bytes):
def get_channel_private_key(self, public_key_bytes):
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
if private_key_pem:
return await asyncio.get_event_loop().run_in_executor(
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
)
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
async def maybe_migrate_certificates(self):
def to_der(private_key_pem):
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256).get_verifying_key().to_der()
if not self.channel_keys:
return
channel_keys = {}
@@ -545,7 +539,8 @@ class Account:
continue
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
continue
public_key_der = await asyncio.get_event_loop().run_in_executor(None, to_der, private_key_pem)
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
public_key_der = private_key.get_verifying_key().to_der()
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
if self.channel_keys != channel_keys:
self.channel_keys = channel_keys
@@ -566,20 +561,20 @@ class Account:
if gap_changed:
self.wallet.save()
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False, read_only=False):
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0
get_total_balance = partial(self.get_balance, read_only=read_only, confirmations=confirmations,
include_claims=True)
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
total = await get_total_balance()
if reserved_subtotals:
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
for txo in await self.get_support_summary():
if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1):
for amount, spent, from_me, to_me, height in await self.get_support_summary():
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
continue
if txo.is_my_input:
supports_balance += txo.amount
else:
tips_balance += txo.amount
if not spent and to_me:
if from_me:
supports_balance += amount
else:
tips_balance += amount
reserved = claims_balance + supports_balance + tips_balance
else:
reserved = await self.get_balance(
@@ -596,15 +591,11 @@ class Account:
} if reserved_subtotals else None
}
def get_transaction_history(self, read_only=False, **constraints):
return self.ledger.get_transaction_history(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_transaction_history(self, **constraints):
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
def get_transaction_history_count(self, read_only=False, **constraints):
return self.ledger.get_transaction_history_count(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_transaction_history_count(self, **constraints):
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
def get_claims(self, **constraints):
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
@@ -637,7 +628,7 @@ class Account:
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
def get_support_summary(self):
return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
return self.ledger.db.get_supports_summary(account_id=self.id)
async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self)
-743
View File
@@ -1,743 +0,0 @@
HASHES = {
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
}
+1 -1
View File
@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
MAXIMUM_TRIES = 100000
STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
STRATEGIES = []
def strategy(method):
+158 -566
View File
File diff suppressed because it is too large. Load diff
+68 -138
View File
@@ -1,18 +1,16 @@
import base64
import os
import struct
import asyncio
import hashlib
import logging
import zlib
from datetime import date
from io import BytesIO
from typing import Optional, Iterator, Tuple, Callable
from contextlib import asynccontextmanager
from typing import Optional, Iterator, Tuple
from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES
from lbry.wallet.util import ArithUint256
log = logging.getLogger(__name__)
@@ -34,50 +32,25 @@ class Headers:
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
target_timespan = 150
checkpoints = HASHES
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
timestamp_average_offset = 160.6855883050695 # calculated at 733447
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
validate_difficulty: bool = True
def __init__(self, path) -> None:
self.io = None
if path == ':memory:':
self.io = BytesIO()
self.path = path
self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self):
self.io = BytesIO()
if self.path != ':memory:':
def _readit():
if os.path.exists(self.path):
with open(self.path, 'r+b') as header_file:
self.io.seek(0)
self.io.write(header_file.read())
await asyncio.get_event_loop().run_in_executor(None, _readit)
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
if bytes_size % self.header_size:
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
await self.repair()
else:
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
await self.repair(start_height=max_checkpointed_height)
await self.ensure_checkpointed_size()
await self.get_all_missing_headers()
if not os.path.exists(self.path):
self.io = open(self.path, 'w+b')
else:
self.io = open(self.path, 'r+b')
async def close(self):
if self.io is not None:
def _close():
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
with open(self.path, flags) as header_file:
header_file.write(self.io.getbuffer())
await asyncio.get_event_loop().run_in_executor(None, _close)
self.io.close()
self.io = None
self.io.close()
@staticmethod
def serialize(header):
@@ -124,92 +97,23 @@ class Headers:
return new_target
def __len__(self) -> int:
if self._size is None:
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return self._size
def __bool__(self):
return True
async def get(self, height) -> dict:
def __getitem__(self, height) -> dict:
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
try:
return self.deserialize(height, await self.get_raw_header(height))
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height, try_real_headers=True):
if height <= 0:
return
if try_real_headers and self.has_header(height):
offset = height * self.header_size
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self._read(height)
return self.deserialize(height, self.get_raw_header(height))
def _read(self, height, count=1):
offset = height * self.header_size
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
def chunk_hash(self, start, count):
return self.hash_header(self._read(start, count)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
if self.height < max_checkpointed_height:
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
async def fetch_chunk(self, height):
log.info("on-demand fetching height %s", height)
start = (height // 1000) * 1000
headers = await self.chunk_getter(start) # pylint: disable=not-callable
chunk = (
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
self._write(start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
elif start not in self.checkpoints:
return # todo: fixme
raise Exception(
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
def get_raw_header(self, height) -> bytes:
self.io.seek(height * self.header_size, os.SEEK_SET)
return self.io.read(self.header_size)
@property
def height(self) -> int:
@@ -219,9 +123,9 @@ class Headers:
def bytes_size(self):
return len(self) * self.header_size
async def hash(self, height=None) -> bytes:
def hash(self, height=None) -> bytes:
return self.hash_header(
await self.get_raw_header(height if height is not None else self.height)
self.get_raw_header(height if height is not None else self.height)
)
@staticmethod
@@ -230,18 +134,44 @@ class Headers:
return b'0' * 64
return hexlify(double_sha256(header)[::-1])
@asynccontextmanager
async def checkpointed_connector(self):
buf = BytesIO()
try:
yield buf
finally:
await asyncio.sleep(0)
final_height = len(self) + buf.tell() // self.header_size
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
buf.seek(0)
self.io.seek(0)
h = hashlib.sha256()
h.update(self.io.read())
h.update(buf.read(verifiable_bytes))
if h.hexdigest().encode() == self.checkpoint[1]:
buf.seek(0)
self._write(len(self), buf.read(verifiable_bytes))
remaining = buf.read()
buf.seek(0)
buf.write(remaining)
buf.truncate()
else:
log.warning("Checkpoint mismatch, connecting headers through slow method.")
if buf.tell() > 0:
await self.connect(len(self), buf.getvalue())
async def connect(self, start: int, headers: bytes) -> int:
added = 0
bail = False
for height, chunk in self._iterate_chunks(start, headers):
try:
# validate_chunk() is CPU bound and reads previous chunks from file system
await self.validate_chunk(height, chunk)
self.validate_chunk(height, chunk)
except InvalidHeader as e:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
if chunk:
added += self._write(height, chunk)
added += self._write(height, chunk) if chunk else 0
if bail:
break
return added
@@ -249,21 +179,20 @@ class Headers:
def _write(self, height, verified_chunk):
self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(verified_chunk) // self.header_size
# self.io.truncate()
self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = max(self._size or 0, self.io.tell() // self.header_size)
self._size = self.io.tell() // self.header_size
return written
async def validate_chunk(self, height, chunk):
def validate_chunk(self, height, chunk):
previous_hash, previous_header, previous_previous_header = None, None, None
if height > 0:
raw = await self.get_raw_header(height-1)
previous_header = self.deserialize(height-1, raw)
previous_hash = self.hash_header(raw)
previous_header = self[height-1]
previous_hash = self.hash(height-1)
if height > 1:
previous_previous_header = await self.get(height-2)
previous_previous_header = self[height-2]
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
for current_hash, current_header in self._iterate_headers(height, chunk):
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
@@ -302,30 +231,28 @@ class Headers:
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
)
async def repair(self, start_height=0):
async def repair(self):
previous_header_hash = fail = None
batch_size = 36
for height in range(start_height, self.height, batch_size):
headers = self._read(height, batch_size)
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(height, headers):
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if previous_header_hash:
if height:
if header['prev_block_hash'] != previous_header_hash:
fail = True
elif height == 0:
else:
if header_hash != self.genesis_hash:
fail = True
else:
# for sanity and clarity, since it is the only way we can end up here
assert start_height > 0 and height == start_height
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
self._size = None
return
previous_header_hash = header_hash
@@ -349,6 +276,10 @@ class Headers:
header = headers[start:end]
yield self.hash_header(header), self.deserialize(height+idx, header)
@property
def claim_trie_root(self):
return self[self.height]['claim_trie_root']
@staticmethod
def header_hash_to_pow_hash(header_hash: bytes):
header_hash_bytes = unhexlify(header_hash)[::-1]
@@ -364,4 +295,3 @@ class UnvalidatedHeaders(Headers):
validate_difficulty = False
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
checkpoints = {}
+139 -411
View File
@@ -1,14 +1,15 @@
import os
import copy
import time
import zlib
import base64
import asyncio
import logging
from io import StringIO
from datetime import datetime
from functools import partial
from operator import itemgetter
from collections import defaultdict
from collections import namedtuple, defaultdict
from binascii import hexlify, unhexlify
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict
import pylru
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
@@ -24,8 +25,7 @@ from .account import Account, AddressManager, SingleKey
from .network import Network
from .transaction import Transaction, Output
from .header import Headers, UnvalidatedHeaders
from .checkpoints import HASHES
from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
from .constants import TXO_TYPES, COIN, NULL_HASH32
from .bip32 import PubKey, PrivateKey
from .coinselection import CoinSelector
@@ -53,29 +53,25 @@ class LedgerRegistry(type):
return mcs.ledgers[ledger_id]
class TransactionEvent(NamedTuple):
address: str
tx: Transaction
class TransactionEvent(namedtuple('TransactionEvent', ('address', 'tx'))):
pass
class AddressesGeneratedEvent(NamedTuple):
address_manager: AddressManager
addresses: List[str]
class AddressesGeneratedEvent(namedtuple('AddressesGeneratedEvent', ('address_manager', 'addresses'))):
pass
class BlockHeightEvent(NamedTuple):
height: int
change: int
class BlockHeightEvent(namedtuple('BlockHeightEvent', ('height', 'change'))):
pass
class TransactionCacheItem:
__slots__ = '_tx', 'lock', 'has_tx', 'pending_verifications'
__slots__ = '_tx', 'lock', 'has_tx'
def __init__(self, tx: Optional[Transaction] = None, lock: Optional[asyncio.Lock] = None):
self.has_tx = asyncio.Event()
self.lock = lock or asyncio.Lock()
self._tx = self.tx = tx
self.pending_verifications = 0
@property
def tx(self) -> Optional[Transaction]:
@@ -109,8 +105,6 @@ class Ledger(metaclass=LedgerRegistry):
default_fee_per_byte = 50
default_fee_per_name_char = 200000
checkpoints = HASHES
def __init__(self, config=None):
self.config = config or {}
self.db: Database = self.config.get('db') or Database(
@@ -120,7 +114,6 @@ class Ledger(metaclass=LedgerRegistry):
self.headers: Headers = self.config.get('headers') or self.headers_class(
os.path.join(self.path, "headers")
)
self.headers.checkpoints = self.checkpoints
self.network: Network = self.config.get('network') or Network(self)
self.network.on_header.listen(self.receive_header)
self.network.on_status.listen(self.process_status_update)
@@ -157,13 +150,11 @@ class Ledger(metaclass=LedgerRegistry):
self._on_ready_controller = StreamController()
self.on_ready = self._on_ready_controller.stream
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 100_000))
self._tx_cache = pylru.lrucache(100000)
self._update_tasks = TaskGroup()
self._other_tasks = TaskGroup() # that we dont need to start
self._utxo_reservation_lock = asyncio.Lock()
self._header_processing_lock = asyncio.Lock()
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
self._history_lock = asyncio.Lock()
self.coin_selection_strategy = None
self._known_addresses_out_of_sync = set()
@@ -244,15 +235,11 @@ class Ledger(metaclass=LedgerRegistry):
def get_address_count(self, **constraints):
return self.db.get_address_count(**constraints)
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1):
min_amount = min(amount // 10, min_amount)
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
selector = CoinSelector(amount, fee)
async def get_spendable_utxos(self, amount: int, funding_accounts):
async with self._utxo_reservation_lock:
if self.coin_selection_strategy == 'sqlite':
return await self.db.get_spendable_utxos(self, amount + fee, funding_accounts, min_amount=min_amount,
fee_per_byte=self.fee_per_byte)
txos = await self.get_effective_amount_estimators(funding_accounts)
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
selector = CoinSelector(amount, fee)
spendables = selector.select(txos, self.coin_selection_strategy)
if spendables:
await self.reserve_outputs(s.txo for s in spendables)
@@ -275,7 +262,7 @@ class Ledger(metaclass=LedgerRegistry):
self.constraint_spending_utxos(constraints)
return self.db.get_utxo_count(**constraints)
async def get_txos(self, resolve=False, **constraints) -> List[Output]:
async def get_txos(self, resolve=False, **constraints):
txos = await self.db.get_txos(**constraints)
if resolve:
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
@@ -284,12 +271,6 @@ class Ledger(metaclass=LedgerRegistry):
def get_txo_count(self, **constraints):
return self.db.get_txo_count(**constraints)
def get_txo_sum(self, **constraints):
return self.db.get_txo_sum(**constraints)
def get_txo_plot(self, **constraints):
return self.db.get_txo_plot(**constraints)
def get_transactions(self, **constraints):
return self.db.get_transactions(**constraints)
@@ -325,12 +306,12 @@ class Ledger(metaclass=LedgerRegistry):
self.db.open(),
self.headers.open()
])
fully_synced = self.on_ready.first
asyncio.create_task(self.network.start())
await self.network.on_connected.first
first_connection = self.network.on_connected.first
asyncio.ensure_future(self.network.start())
await first_connection
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
await fully_synced
await self._on_ready_controller.stream.first
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -341,15 +322,15 @@ class Ledger(metaclass=LedgerRegistry):
async def join_network(self, *_):
log.info("Subscribing and updating accounts.")
await self._update_tasks.add(self.subscribe_accounts())
async with self._header_processing_lock:
await self.update_headers()
await self.subscribe_accounts()
await self._update_tasks.done.wait()
self._on_ready_controller.add(True)
async def stop(self):
self._update_tasks.cancel()
self._other_tasks.cancel()
await self._update_tasks.done.wait()
await self._other_tasks.done.wait()
await self.network.stop()
await self.db.close()
await self.headers.close()
@@ -359,15 +340,19 @@ class Ledger(metaclass=LedgerRegistry):
return max(self.headers.height, self._download_height)
async def initial_headers_sync(self):
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True)
self.headers.chunk_getter = get_chunk
async def doit():
for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)):
async with self._header_processing_lock:
await self.headers.ensure_chunk_at(height)
self._other_tasks.add(doit())
await self.update_headers()
target = self.network.remote_height + 1
current = len(self.headers)
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=4096, b64=True)
chunks = [asyncio.create_task(get_chunk(height)) for height in range(current, target, 4096)]
total = 0
async with self.headers.checkpointed_connector() as buffer:
for chunk in chunks:
headers = await chunk
total += buffer.write(
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
self._download_height = current + total // self.headers.header_size
log.info("Headers sync: %s / %s", self._download_height, target)
async def update_headers(self, height=None, headers=None, subscription_update=False):
rewound = 0
@@ -489,10 +474,10 @@ class Ledger(metaclass=LedgerRegistry):
address, remote_status = update
self._update_tasks.add(self.update_history(address, remote_status))
async def update_history(self, address, remote_status, address_manager: AddressManager = None,
reattempt_update: bool = True):
async def update_history(self, address, remote_status, address_manager: AddressManager = None):
async with self._address_update_locks[address]:
self._known_addresses_out_of_sync.discard(address)
local_status, local_history = await self.get_local_status_and_history(address)
if local_status == remote_status:
@@ -502,94 +487,57 @@ class Ledger(metaclass=LedgerRegistry):
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
we_need = set(remote_history) - set(local_history)
if not we_need:
remote_missing = set(local_history) - set(remote_history)
if remote_missing:
log.warning(
"%i transactions we have for %s are not in the remote address history",
len(remote_missing), address
)
return True
acquire_lock_tasks = []
cache_tasks: List[asyncio.Future[Transaction]] = []
synced_history = StringIO()
for i, (txid, remote_height) in enumerate(remote_history):
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
synced_history.write(f'{txid}:{remote_height}:')
else:
check_local = (txid, remote_height) not in we_need
cache_tasks.append(asyncio.ensure_future(
self.cache_transaction(txid, remote_height, check_local=check_local)
))
synced_txs = []
to_request = {}
pending_synced_history = {}
updated_cached_items = {}
already_synced = set()
for task in cache_tasks:
tx = await task
already_synced_offset = 0
for i, (txid, remote_height) in enumerate(remote_history):
if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height):
pending_synced_history[i] = f'{txid}:{remote_height}:'
already_synced.add((txid, remote_height))
already_synced_offset += 1
continue
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
check_db_for_txos = []
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
if cache_item is not None:
if cache_item.tx is None:
await cache_item.has_tx.wait()
assert cache_item.tx is not None
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos.append(txi.txo_ref.id)
for txid, remote_height in remote_history[already_synced_offset:]:
cache_item = self._tx_cache[txid]
acquire_lock_tasks.append(asyncio.create_task(cache_item.lock.acquire()))
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(txoid__in=check_db_for_txos, no_tx=True)
}
if acquire_lock_tasks:
await asyncio.wait(acquire_lock_tasks)
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
referenced_txo = referenced_txos.get(txi.txo_ref.id)
if referenced_txo is not None:
txi.txo_ref = referenced_txo.ref
tx_indexes = {}
for i, (txid, remote_height) in enumerate(remote_history):
tx_indexes[txid] = i
if (txid, remote_height) in already_synced:
continue
cache_item = self._tx_cache.get(txid)
cache_item.pending_verifications += 1
updated_cached_items[txid] = cache_item
assert cache_item is not None, 'cache item is none'
assert cache_item.lock.locked(), 'cache lock is not held?'
# tx = cache_item.tx
# if cache_item.tx is not None and \
# cache_item.tx.height >= remote_height and \
# (cache_item.tx.is_verified or remote_height < 1):
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
# continue
to_request[i] = (txid, remote_height)
log.debug(
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
len(remote_history), address
)
requested_txes = await self._request_transaction_batch(to_request, len(remote_history), address)
for tx in requested_txes:
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
synced_history.write(f'{tx.id}:{tx.height}:')
synced_txs.append(tx)
assert len(pending_synced_history) == len(remote_history), \
f"{len(pending_synced_history)} vs {len(remote_history)}"
synced_history = ""
for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())):
assert i == remote_i, f"{i} vs {remote_i}"
txid, height = remote_history[remote_i]
if f"{txid}:{height}:" != pending_synced_history[i]:
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
synced_history += pending_synced_history[i]
cache_size = self.config.get("tx_cache_size", 100_000)
for txid, cache_item in updated_cached_items.items():
cache_item.pending_verifications -= 1
if cache_item.pending_verifications < 0:
log.warning("config value tx cache size %i needs to be increased", cache_size)
cache_item.pending_verifications = 0
try:
cache_item.lock.release()
except RuntimeError:
log.warning("lock was already released?")
await self.db.save_transaction_io_batch(
[], address, self.address_to_hash160(address), synced_history
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
)
await asyncio.wait([
self._on_transaction_controller.add(TransactionEvent(address, tx))
for tx in synced_txs
])
if address_manager is None:
address_manager = await self.get_address_manager_for_address(address)
@@ -597,42 +545,20 @@ class Ledger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
for txid, cache_item in updated_cached_items.items():
if self._tx_cache.get(txid) is not cache_item:
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
if reattempt_update:
return await self.update_history(address, remote_status, address_manager, False)
return False
local_status, local_history = \
await self.get_local_status_and_history(address, synced_history)
await self.get_local_status_and_history(address, synced_history.getvalue())
if local_status != remote_status:
if local_history == remote_history:
log.warning(
"%s has a synced history but a mismatched status", address
)
return True
remote_set = set(remote_history)
local_set = set(local_history)
log.warning(
"%s is out of sync after syncing.\n"
"Remote: %s with %d items (%i unique), local: %s with %d items (%i unique).\n"
"Histories are mismatched on %i items.\n"
"Local is missing\n"
"%s\n"
"Remote is missing\n"
"%s\n"
"******",
address, remote_status, len(remote_history), len(remote_set),
local_status, len(local_history), len(local_set), len(remote_set.symmetric_difference(local_set)),
"\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)]),
"\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)])
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
remote_status, len(remote_history), local_status, len(local_history)
)
log.warning("local: %s", local_history)
log.warning("remote: %s", remote_history)
self._known_addresses_out_of_sync.add(address)
return False
else:
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
return True
async def cache_transaction(self, txid, remote_height, check_local=True):
@@ -644,142 +570,32 @@ class Ledger(metaclass=LedgerRegistry):
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
cache_item.pending_verifications += 1
try:
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
cache_item.tx = tx # make sure it's saved before caching it
tx.height = remote_height
if merkle and 0 < remote_height < len(self.headers):
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
finally:
cache_item.pending_verifications -= 1
async with cache_item.lock:
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
if tx is None:
# fetch from network
_raw = await self.network.retriable_call(self.network.get_transaction, txid, remote_height)
tx = Transaction(unhexlify(_raw))
cache_item.tx = tx # make sure it's saved before caching it
await self.maybe_verify_transaction(tx, remote_height)
return tx
async def maybe_verify_transaction(self, tx, remote_height):
tx.height = remote_height
cached = self._tx_cache.get(tx.id)
if not cached:
# cache txs looked up by transaction_show too
cached = TransactionCacheItem()
self._tx_cache[tx.id] = cached
cached.tx = tx
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
if not merkle:
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
header = self.headers[remote_height]
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
async def _request_transaction_batch(self, to_request, remote_history_size, address):
header_cache = {}
batches = [[]]
remote_heights = {}
synced_txs = []
heights_in_batch = 0
last_height = 0
for idx in sorted(to_request):
txid = to_request[idx][0]
height = to_request[idx][1]
remote_heights[txid] = height
if height != last_height:
heights_in_batch += 1
last_height = height
if len(batches[-1]) == 100 or heights_in_batch == 20:
batches.append([])
heights_in_batch = 1
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
last_showed_synced_count = 0
async def _single_batch(batch):
this_batch_synced = []
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
merkle_height = merkle['block_height']
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx = cache_item.tx or Transaction(unhexlify(raw), height=remote_height)
tx.height = remote_height
cache_item.tx = tx
if 'merkle' in merkle and remote_heights[txid] > 0:
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
try:
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
except IndexError:
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
else:
header_cache[remote_heights[txid]] = header
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
check_db_for_txos = []
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
if cache_item is not None:
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos.append(txi.txo_ref.id)
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
)
}
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
referenced_txo = referenced_txos.get(txi.txo_ref.id)
if referenced_txo is not None:
txi.txo_ref = referenced_txo.ref
continue
cache_item = self._tx_cache.get(txi.txo_ref.id)
if cache_item is None:
cache_item = self._tx_cache[txi.txo_ref.id] = TransactionCacheItem()
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.ref
synced_txs.append(tx)
this_batch_synced.append(tx)
await self.db.save_transaction_io_batch(
this_batch_synced, address, self.address_to_hash160(address), ""
)
await asyncio.wait([
self._on_transaction_controller.add(TransactionEvent(address, tx))
for tx in this_batch_synced
])
nonlocal last_showed_synced_count
if last_showed_synced_count + 100 < len(synced_txs):
log.info("synced %i/%i transactions for %s", len(synced_txs), remote_history_size, address)
last_showed_synced_count = len(synced_txs)
for batch in batches:
await _single_batch(batch)
return synced_txs
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
details = await self.db.get_address(address=address)
for account in self.accounts:
@@ -792,7 +608,6 @@ class Ledger(metaclass=LedgerRegistry):
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: Transaction, height=-1, timeout=1):
timeout = timeout or 600 # after 10 minutes there is almost 0 hope
addresses = set()
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
@@ -802,48 +617,28 @@ class Ledger(metaclass=LedgerRegistry):
for txo in tx.outputs:
if txo.has_address:
addresses.add(self.hash160_to_address(txo.pubkey_hash))
start = int(time.perf_counter())
while timeout and (int(time.perf_counter()) - start) <= timeout:
if await self._wait_round(tx, height, addresses):
return
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]):
records = await self.db.get_addresses(address__in=addresses)
_, pending = await asyncio.wait([
self.on_transaction.where(partial(
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
address_record['address']
)) for address_record in records
], timeout=1)
if not pending:
return True
records = await self.db.get_addresses(address__in=addresses)
for record in records:
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id:
if local_height >= height:
return True
log.warning(
"local history has higher height than remote for %s (%i vs %i)", txid,
local_height, height
)
return False
log.warning(
"local history does not contain %s, requested height %i", tx.id, height
)
return False
], timeout=timeout)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
found = False
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id and local_height >= height:
found = True
if not found:
print(record['history'], addresses, tx.id)
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _inflate_outputs(
self, query, accounts,
include_purchase_receipt=False,
include_is_my_output=False,
include_sent_supports=False,
include_sent_tips=False,
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
@@ -851,33 +646,12 @@ class Ledger(metaclass=LedgerRegistry):
txs: List[Transaction] = await asyncio.gather(*(
self.cache_transaction(*tx) for tx in outputs.txs
))
_txos, blocked = outputs.inflate(txs)
txos = []
for txo in _txos:
if isinstance(txo, Output):
# transactions and outputs are cached and shared between wallets
# we don't want to leak informaion between wallet so we add the
# wallet specific metadata on throw away copies of the txos
txo = copy.copy(txo)
channel = txo.channel
txo.purchase_receipt = None
txo.update_annotations(None)
txo.channel = channel
txos.append(txo)
includes = (
include_purchase_receipt, include_is_my_output,
include_sent_supports, include_sent_tips
)
if accounts and any(includes):
receipts = {}
if include_purchase_receipt:
if accounts:
priced_claims = []
for txo in txos:
if isinstance(txo, Output) and txo.has_price:
priced_claims.append(txo)
for tx in txs:
for txo in tx.outputs:
if txo.has_price:
priced_claims.append(txo)
if priced_claims:
receipts = {
txo.purchased_claim_id: txo for txo in
@@ -886,49 +660,14 @@ class Ledger(metaclass=LedgerRegistry):
purchased_claim_id__in=[c.claim_id for c in priced_claims]
)
}
for txo in txos:
if isinstance(txo, Output) and txo.can_decode_claim:
if include_purchase_receipt:
for txo in priced_claims:
txo.purchase_receipt = receipts.get(txo.claim_id)
if include_is_my_output:
mine = await self.db.get_txo_count(
claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True,
is_spent=False, accounts=accounts
)
if mine:
txo.is_my_output = True
else:
txo.is_my_output = False
if include_sent_supports:
supports = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=True, is_my_output=True,
is_spent=False, accounts=accounts
)
txo.sent_supports = supports
if include_sent_tips:
tips = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=True, is_my_output=False,
accounts=accounts
)
txo.sent_tips = tips
if include_received_tips:
tips = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=False, is_my_output=True,
accounts=accounts
)
txo.received_tips = tips
txos, blocked = outputs.inflate(txs)
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls, **kwargs):
async def resolve(self, accounts, urls):
resolve = partial(self.network.retriable_call, self.network.resolve)
urls_copy = list(urls)
txos = []
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
txos = (await self._inflate_outputs(resolve(urls), accounts))[0]
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -941,17 +680,11 @@ class Ledger(metaclass=LedgerRegistry):
result[url] = txo
return result
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
**kwargs) -> Tuple[List[Output], dict, int, int]:
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
)
async def claim_search(self, accounts, **kwargs) -> Tuple[List[Output], dict, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs), accounts)
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
async def get_claim_by_claim_id(self, accounts, claim_id) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id))[0]:
return claim
async def _report_state(self):
@@ -1089,15 +822,12 @@ class Ledger(metaclass=LedgerRegistry):
def get_support_count(self, **constraints):
return self.db.get_support_count(**constraints)
async def get_transaction_history(self, read_only=False, **constraints):
txs: List[Transaction] = await self.db.get_transactions(
include_is_my_output=True, include_is_spent=True,
read_only=read_only, **constraints
)
async def get_transaction_history(self, **constraints):
txs: List[Transaction] = await self.db.get_transactions(**constraints)
headers = self.headers
history = []
for tx in txs: # pylint: disable=too-many-nested-blocks
ts = headers.estimated_timestamp(tx.height)
ts = headers[tx.height]['timestamp'] if tx.height > 0 else None
item = {
'txid': tx.id,
'timestamp': ts,
@@ -1109,7 +839,7 @@ class Ledger(metaclass=LedgerRegistry):
'abandon_info': [],
'purchase_info': []
}
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
is_my_inputs = all([txi.is_my_account for txi in tx.inputs])
if is_my_inputs:
# fees only matter if we are the ones paying them
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
@@ -1202,8 +932,8 @@ class Ledger(metaclass=LedgerRegistry):
history.append(item)
return history
def get_transaction_history_count(self, read_only=False, **constraints):
return self.db.get_transaction_count(read_only=read_only, **constraints)
def get_transaction_history_count(self, **constraints):
return self.db.get_transaction_count(**constraints)
async def get_detailed_balance(self, accounts, confirmations=0):
result = {
@@ -1236,7 +966,6 @@ class TestNetLedger(Ledger):
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
checkpoints = {}
class RegTestLedger(Ledger):
@@ -1251,4 +980,3 @@ class RegTestLedger(Ledger):
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
genesis_bits = 0x207fffff
target_timespan = 1
checkpoints = {}
+15 -18
View File
@@ -184,7 +184,6 @@ class WalletManager:
'auto_connect': True,
'default_servers': config.lbryum_servers,
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
@@ -249,28 +248,26 @@ class WalletManager:
log.warning("Failed to migrate %s receiving addresses!",
len(set(receiving_addresses).difference(set(migrated_receiving))))
async def get_best_blockhash(self):
def get_best_blockhash(self):
if len(self.ledger.headers) <= 0:
return self.ledger.genesis_hash
return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
return self.ledger.headers.hash(self.ledger.headers.height).decode()
def get_unused_address(self):
return self.default_account.receiving.get_or_create_usable_address()
async def get_transaction(self, txid: str):
async def get_transaction(self, txid):
tx = await self.db.get_transaction(txid=txid)
if tx:
return tx
try:
raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
except CodeMessageError as e:
if 'No such mempool or blockchain transaction.' in e.message:
return {'success': False, 'code': 404, 'message': 'transaction not found'}
return {'success': False, 'code': e.code, 'message': e.message}
height = merkle.get('block_height')
tx = Transaction(unhexlify(raw), height=height)
if height and height > 0:
await self.ledger.maybe_verify_transaction(tx, height, merkle)
if not tx:
try:
raw = await self.ledger.network.get_transaction(txid)
height = await self.ledger.network.get_transaction_height(txid)
except CodeMessageError as e:
if 'No such mempool or blockchain transaction.' in e.message:
return {'success': False, 'code': 404, 'message': 'transaction not found'}
return {'success': False, 'code': e.code, 'message': e.message}
tx = Transaction(unhexlify(raw))
await self.ledger.maybe_verify_transaction(tx, height)
return tx
async def create_purchase_transaction(
@@ -300,8 +297,8 @@ class WalletManager:
async def broadcast_or_release(self, tx, blocking=False):
try:
await self.ledger.broadcast(tx)
if blocking:
await self.ledger.wait(tx, timeout=None)
except:
await self.ledger.release_tx(tx)
raise
if blocking:
await self.ledger.wait(tx, timeout=None)
+5 -14
View File
@@ -55,7 +55,7 @@ class ClientSession(BaseClientSession):
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
log.debug("send %s to %s:%i", method, *self.server)
try:
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
@@ -87,7 +87,7 @@ class ClientSession(BaseClientSession):
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
# self.synchronous_close()
self.synchronous_close()
raise
finally:
self.pending_amount -= 1
@@ -149,13 +149,14 @@ class ClientSession(BaseClientSession):
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self.pending_amount = 0
self._on_disconnect_controller.add(True)
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 65, 0)
MINIMUM_REQUIRED = (0, 59, 0)
def __init__(self, ledger):
self.ledger = ledger
@@ -255,15 +256,6 @@ class Network:
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_batch(self, txids):
# use any server if its old, otherwise restrict to who gave us the history
return self.rpc('blockchain.transaction.get_batch', txids, True)
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.info', [tx_hash], restricted)
def get_transaction_height(self, tx_hash, known_height=None):
restricted = not known_height or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
@@ -288,13 +280,12 @@ class Network:
async def subscribe_address(self, address, *addresses):
addresses = list((address, ) + addresses)
server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None
try:
return await self.rpc('blockchain.address.subscribe', addresses, True)
except asyncio.TimeoutError:
log.warning(
"timed out subscribing to addresses from %s:%i",
*server_addr_and_port
*self.client.server_address_and_port
)
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
if self.client:
+6 -41
View File
@@ -55,8 +55,7 @@ class Conductor:
async def start_blockchain(self):
if not self.blockchain_started:
asyncio.create_task(self.blockchain_node.start())
await self.blockchain_node.running.wait()
await self.blockchain_node.start()
await self.blockchain_node.generate(200)
self.blockchain_started = True
@@ -256,10 +255,6 @@ class BlockchainNode:
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
self.rpcuser = 'rpcuser'
self.rpcpassword = 'rpcpassword'
self.stopped = False
self.restart_ready = asyncio.Event()
self.restart_ready.set()
self.running = asyncio.Event()
@property
def rpc_url(self):
@@ -320,27 +315,13 @@ class BlockchainNode:
f'-port={self.peerport}'
]
self.log.info(' '.join(command))
while not self.stopped:
if self.running.is_set():
await asyncio.sleep(1)
continue
await self.restart_ready.wait()
try:
self.transport, self.protocol = await loop.subprocess_exec(
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
self.running.set()
except asyncio.CancelledError:
self.running.clear()
raise
except Exception as e:
self.running.clear()
log.exception('failed to start lbrycrdd', exc_info=e)
self.transport, self.protocol = await loop.subprocess_exec(
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
async def stop(self, cleanup=True):
self.stopped = True
try:
self.transport.terminate()
await self.protocol.stopped.wait()
@@ -349,16 +330,6 @@ class BlockchainNode:
if cleanup:
self.cleanup()
async def clear_mempool(self):
self.restart_ready.clear()
self.transport.terminate()
await self.protocol.stopped.wait()
self.transport.close()
self.running.clear()
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
self.restart_ready.set()
await self.running.wait()
def cleanup(self):
shutil.rmtree(self.data_path, ignore_errors=True)
@@ -390,12 +361,6 @@ class BlockchainNode:
def get_block_hash(self, block):
return self._cli_cmnd('getblockhash', str(block))
def sendrawtransaction(self, tx):
return self._cli_cmnd('sendrawtransaction', tx)
async def get_block(self, block_hash):
return json.loads(await self._cli_cmnd('getblock', block_hash, '1'))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress')
+9 -34
View File
@@ -33,16 +33,14 @@ from asyncio import Event, CancelledError
import logging
import time
from contextlib import suppress
from prometheus_client import Counter, Histogram
from lbry.wallet.tasks import TaskGroup
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
from .jsonrpc import RPCError, ProtocolError
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
from .util import Concurrency
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT
class Connector:
@@ -206,7 +204,7 @@ class SessionBase(asyncio.Protocol):
"""
return self._address
def peer_address_str(self, for_log=True):
def peer_address_str(self):
"""Returns the peer's IP address and port as a human-readable
string."""
if not self._address:
@@ -375,26 +373,10 @@ class BatchRequest:
raise BatchError(self)
NAMESPACE = "wallet_server"
class RPCSession(SessionBase):
"""Base class for protocols where a message can lead to a response,
for example JSON RPC."""
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS)
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
namespace=NAMESPACE, labelnames=("method", "version"))
REQUEST_ERRORS_COUNT = Counter(
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
labelnames=("method", "version")
)
RESET_CONNECTIONS = Counter(
"reset_clients", "Number of reset connections by client version",
namespace=NAMESPACE, labelnames=("version",)
)
def __init__(self, *, framer=None, loop=None, connection=None):
super().__init__(framer=framer, loop=loop)
self.connection = connection or self.default_connection()
@@ -407,7 +389,6 @@ class RPCSession(SessionBase):
except MemoryError:
self.logger.warning('received oversized message from %s:%s, dropping connection',
self._address[0], self._address[1])
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
self._close()
return
@@ -441,7 +422,7 @@ class RPCSession(SessionBase):
'internal server error')
if isinstance(request, Request):
message = request.send_result(result)
self.RESPONSE_TIMES.labels(
RESPONSE_TIMES.labels(
method=request.method,
version=self.client_version
).observe(time.perf_counter() - start)
@@ -449,7 +430,7 @@ class RPCSession(SessionBase):
await self._send_message(message)
if isinstance(result, Exception):
self._bump_errors()
self.REQUEST_ERRORS_COUNT.labels(
REQUEST_ERRORS_COUNT.labels(
method=request.method,
version=self.client_version
).inc()
@@ -483,17 +464,11 @@ class RPCSession(SessionBase):
raise result
return result
async def send_notification(self, method, args=()) -> bool:
async def send_notification(self, method, args=()):
"""Send an RPC notification over the network."""
message = self.connection.send_notification(Notification(method, args))
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
try:
await self._send_message(message)
return True
except asyncio.TimeoutError:
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
self.abort()
return False
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
await self._send_message(message)
def send_batch(self, raise_errors=False):
"""Return a BatchRequest. Intended to be used like so:
-27
View File
@@ -438,17 +438,6 @@ class OutputScript(Script):
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
))
SUPPORT_CLAIM_DATA_OPCODES = (
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
OP_2DROP, OP_2DROP
)
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
))
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
))
UPDATE_CLAIM_OPCODES = (
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
OP_2DROP, OP_2DROP
@@ -485,8 +474,6 @@ class OutputScript(Script):
CLAIM_NAME_SCRIPT,
SUPPORT_CLAIM_PUBKEY,
SUPPORT_CLAIM_SCRIPT,
SUPPORT_CLAIM_DATA_PUBKEY,
SUPPORT_CLAIM_DATA_SCRIPT,
UPDATE_CLAIM_PUBKEY,
UPDATE_CLAIM_SCRIPT,
SELL_CLAIM, SELL_SCRIPT,
@@ -540,16 +527,6 @@ class OutputScript(Script):
'pubkey_hash': pubkey_hash
})
@classmethod
def pay_support_data_pubkey_hash(
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
'claim_name': claim_name,
'claim_id': claim_id,
'support': support,
'pubkey_hash': pubkey_hash
})
@classmethod
def sell_script(cls, price):
return cls(template=cls.SELL_SCRIPT, values={
@@ -598,10 +575,6 @@ class OutputScript(Script):
def is_support_claim(self):
return self.template.name.startswith('support_claim+')
@property
def is_support_claim_data(self):
return self.template.name.startswith('support_claim+data+')
@property
def is_sell_claim(self):
return self.template.name.startswith('sell_claim+')
+12 -30
View File
@@ -2,8 +2,7 @@ import time
import asyncio
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Optional
from prometheus_client import Gauge, Histogram
import lbry
from lbry.schema.claim import Claim
from lbry.wallet.server.db.writer import SQLDB
@@ -11,6 +10,7 @@ from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
class Prefetcher:
@@ -129,12 +129,6 @@ class ChainError(Exception):
"""Raised on error processing blocks."""
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class BlockProcessor:
"""Process blocks and update the DB state to match.
@@ -142,16 +136,6 @@ class BlockProcessor:
Coordinate backing up in case of chain reorganisations.
"""
block_count_metric = Gauge(
"block_count", "Number of processed blocks", namespace=NAMESPACE
)
block_update_time_metric = Histogram(
"block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
reorg_count_metric = Gauge(
"reorg_count", "Number of reorgs", namespace=NAMESPACE
)
def __init__(self, env, db, daemon, notifications):
self.env = env
self.db = db
@@ -183,7 +167,6 @@ class BlockProcessor:
self.state_lock = asyncio.Lock()
self.search_cache = {}
self.history_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
@@ -214,11 +197,10 @@ class BlockProcessor:
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
for cache in self.search_cache.values():
cache.clear()
self.history_cache.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
self.block_count_metric.set(self.height)
self.block_update_time_metric.observe(processed_time)
BLOCK_COUNT.set(self.height)
BLOCK_UPDATE_TIMES.observe(processed_time)
if not self.db.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
@@ -237,7 +219,7 @@ class BlockProcessor:
'resetting the prefetcher')
await self.prefetcher.reset_height(self.height)
async def reorg_chain(self, count: Optional[int] = None):
async def reorg_chain(self, count=None):
"""Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
@@ -251,7 +233,7 @@ class BlockProcessor:
async def get_raw_blocks(last_height, hex_hashes):
heights = range(last_height, last_height - len(hex_hashes), -1)
try:
blocks = [await self.db.read_raw_block(height) for height in heights]
blocks = [self.db.read_raw_block(height) for height in heights]
self.logger.info(f'read {len(blocks)} blocks from disk')
return blocks
except FileNotFoundError:
@@ -271,9 +253,7 @@ class BlockProcessor:
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
async def reorg_hashes(self, count):
"""Return a pair (start, last, hashes) of blocks to back up during a
@@ -290,7 +270,7 @@ class BlockProcessor:
return start, last, await self.db.fs_block_hashes(start, count)
async def calc_reorg_range(self, count: Optional[int]):
async def calc_reorg_range(self, count):
"""Calculate the reorg range"""
def diff_pos(hashes1, hashes2):
@@ -351,9 +331,11 @@ class BlockProcessor:
# performed on the DB.
if self._caught_up_event.is_set():
await self.flush(True)
elif time.perf_counter() > self.next_cache_check:
await self.flush(True)
self.next_cache_check = time.perf_counter() + 30
elif time.time() > self.next_cache_check:
flush_arg = self.check_cache_size()
if flush_arg is not None:
await self.flush(flush_arg)
self.next_cache_check = time.time() + 30
def check_cache_size(self):
"""Flush a cache if it gets too big."""
+7 -19
View File
@@ -6,12 +6,11 @@ from functools import wraps
from pylru import lrucache
import aiohttp
from prometheus_client import Gauge, Histogram
from lbry.wallet.rpc.jsonrpc import RPCError
from lbry.wallet.server.util import hex_to_bytes, class_logger
from lbry.wallet.rpc import JSONRPC
from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT
class DaemonError(Exception):
"""Raised when the daemon returns an error in its results."""
@@ -25,23 +24,12 @@ class WorkQueueFullError(Exception):
"""Internal - when the daemon's work queue is full."""
NAMESPACE = "wallet_server"
class Daemon:
"""Handles connections to a daemon at the given URL."""
WARMING_UP = -28
id_counter = itertools.count()
lbrycrd_request_time_metric = Histogram(
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
)
lbrycrd_pending_count_metric = Gauge(
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
labelnames=("method",)
)
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
max_retry=4.0):
self.coin = coin
@@ -55,8 +43,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(100000)
self._block_cache = lrucache(10000)
self._block_hash_cache = lrucache(1000000)
self._block_cache = lrucache(100000)
async def close(self):
if self.connector:
@@ -141,7 +129,7 @@ class Daemon:
while True:
try:
for method in methods:
self.lbrycrd_pending_count_metric.labels(method=method).inc()
LBRYCRD_PENDING_COUNT.labels(method=method).inc()
result = await self._send_data(data)
result = processor(result)
if on_good_message:
@@ -166,7 +154,7 @@ class Daemon:
on_good_message = 'running normally'
finally:
for method in methods:
self.lbrycrd_pending_count_metric.labels(method=method).dec()
LBRYCRD_PENDING_COUNT.labels(method=method).dec()
await asyncio.sleep(retry)
retry = max(min(self.max_retry, retry * 2), self.init_retry)
@@ -187,7 +175,7 @@ class Daemon:
if params:
payload['params'] = params
result = await self._send(payload, processor)
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start)
return result
async def _send_vector(self, method, params_iterable, replace_errs=False):
@@ -212,7 +200,7 @@ class Daemon:
result = []
if payload:
result = await self._send(payload, processor)
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start)
return result
async def _is_rpc_available(self, method):
+10 -26
View File
@@ -545,19 +545,11 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
@@ -604,19 +596,11 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
+2 -3
View File
@@ -1,9 +1,8 @@
from . import zscore
from . import ar
from . import variable_decay
TRENDING_ALGORITHMS = {
'zscore': zscore,
'ar': ar,
'variable_decay': variable_decay
'ar': ar
}
@@ -1,431 +0,0 @@
"""
Delayed AR with variable decay rate.
The spike height function is also simpler.
"""
import copy
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
# How frequently to write trending values to the db
SAVE_INTERVAL = 10
# Renormalisation interval
RENORM_INTERVAL = 1000
# Assertion
assert RENORM_INTERVAL % SAVE_INTERVAL == 0
# Decay coefficient per renormalisation interval
DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
# Log trending calculations?
TRENDING_LOG = True
def install(connection):
"""
Install the trending algorithm.
"""
check_trending_values(connection)
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
f.close()
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
reset them. This will allow resyncing from a standard snapshot.
"""
c = connection.cursor()
needs_reset = False
for row in c.execute("SELECT COUNT(*) num FROM claim WHERE trending_global <> 0;"):
if row[0] != 0:
needs_reset = True
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True, end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def spike_height(x, x_old):
"""
Compute the size of a trending spike (normed - constant units).
"""
# Sign of trending spike
sign = 1.0
if x < x_old:
sign = -1.0
# Magnitude
mag = abs(x**0.25 - x_old**0.25)
# Minnow boost
mag *= 1.0 + 2E4/(x + 100.0)**2
return sign*mag
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
def test_trending():
"""
Quick trending test for claims with different support patterns.
Actually use the run() function.
"""
# Create a fake "claims.db" for testing
# pylint: disable=I1101
dbc = apsw.Connection(":memory:")
db = dbc.cursor()
# Create table
db.execute("""
BEGIN;
CREATE TABLE claim (claim_hash TEXT PRIMARY KEY,
amount REAL NOT NULL DEFAULT 0.0,
support_amount REAL NOT NULL DEFAULT 0.0,
trending_mixed REAL NOT NULL DEFAULT 0.0);
COMMIT;
""")
# Insert initial states of claims
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
for key in stuff:
l.append((key, stuff[key]))
return l
db.executemany("""
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
# Main loop
for height in range(1, 1000):
# One-off supports
if height == 1:
everything["huge_whale"] += 5E5
everything["medium_whale"] += 5E4
everything["small_whale"] += 5E3
# Every block
if height < 500:
everything["huge_whale_botted"] += 5E5/500
everything["minnow"] += 1
# Remove supports
if height == 500:
for key in everything:
everything[key] = 0.01
# Whack into the db
db.executemany("""
UPDATE claim SET amount = 1E8*? WHERE claim_hash = ?;
""", [(y, x) for (x, y) in to_list_of_tuples(everything)])
# Call run()
run(db, height, height, everything.keys())
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trending_data.claims:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
test_trending()
-9
View File
@@ -433,15 +433,6 @@ class SQLDB:
return {r.channel_hash for r in affected_channels}
return set()
def delete_claims_above_height(self, height: int):
claim_hashes = [x[0] for x in self.execute(
"SELECT claim_hash FROM claim WHERE height>?", (height, )
).fetchall()]
while claim_hashes:
batch = set(claim_hashes[:500])
claim_hashes = claim_hashes[500:]
self.delete_claims(batch)
def _clear_claim_metadata(self, claim_hashes: Set[bytes]):
if claim_hashes:
for table in ('tag',): # 'language', 'location', etc
+89 -116
View File
@@ -32,9 +32,6 @@ from lbry.wallet.server.history import History
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
HEADER_PREFIX = b'H'
TX_COUNT_PREFIX = b'T'
TX_HASH_PREFIX = b'X'
@attr.s(slots=True)
@@ -66,7 +63,15 @@ class LevelDB:
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.executor = None
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset
self.header_len = self.coin.static_header_len
else:
self.header_offset = self.dynamic_header_offset
self.header_len = self.dynamic_header_len
self.logger.info(f'switching current directory to {env.db_dir}')
@@ -82,71 +87,51 @@ class LevelDB:
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.hashes_db = None
self.headers_db = None
self.tx_count_db = None
path = partial(os.path.join, self.env.db_dir)
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
path('meta/headers_offsets'), 2, 16000000)
async def _read_tx_counts(self):
if self.tx_counts is not None:
return
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
def get_counts():
return tuple(
util.unpack_be_uint64(tx_count)
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
)
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
size = (self.db_height + 1) * 4
tx_counts = self.tx_counts_file.read(0, size)
assert len(tx_counts) == size
self.tx_counts = array.array('I', tx_counts)
if self.tx_counts:
assert self.db_tx_count == self.tx_counts[-1], \
f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)"
assert self.db_tx_count == self.tx_counts[-1]
else:
assert self.db_tx_count == 0
async def _open_dbs(self, for_sync, compacting):
if self.executor is None:
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
coin_path = os.path.join(self.env.db_dir, 'COIN')
if not os.path.isfile(coin_path):
with util.open_file(coin_path, create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
assert self.headers_db is None
self.headers_db = self.db_class('headers', for_sync)
if self.headers_db.is_new:
self.logger.info('created new headers db')
self.logger.info(f'opened headers DB (for sync: {for_sync})')
assert self.tx_count_db is None
self.tx_count_db = self.db_class('tx_count', for_sync)
if self.tx_count_db.is_new:
self.logger.info('created new tx count db')
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
assert self.hashes_db is None
self.hashes_db = self.db_class('hashes', for_sync)
if self.hashes_db.is_new:
self.logger.info('created new tx hashes db')
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
assert self.utxo_db is None
# First UTXO DB
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new utxo db')
self.logger.info(f'opened utxo db (for sync: {for_sync})')
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
with util.open_file(coin_path, create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file.write(0, bytes(8))
else:
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
self.read_utxo_state()
# Then history DB
self.utxo_flush_count = self.history.open_db(
self.db_class, for_sync, self.utxo_flush_count, compacting
)
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
self.utxo_flush_count,
compacting)
self.clear_excess_undo_info()
# Read TX counts (requires meta directory)
@@ -155,11 +140,7 @@ class LevelDB:
def close(self):
self.utxo_db.close()
self.history.close_db()
self.headers_db.close()
self.tx_count_db.close()
self.hashes_db.close()
self.executor.shutdown(wait=True)
self.executor = None
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -171,31 +152,18 @@ class LevelDB:
synchronization. When serving clients we want the open files for
serving network connections.
"""
self.logger.info("opened for sync")
await self._open_dbs(True, False)
async def open_for_serving(self):
"""Open the databases for serving. If they are already open they are
closed first.
"""
self.logger.info('closing DBs to re-open for serving')
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
if self.headers_db:
self.headers_db.close()
self.headers_db = None
if self.tx_count_db:
self.tx_count_db.close()
self.tx_count_db = None
if self.hashes_db:
self.hashes_db.close()
self.hashes_db = None
await self._open_dbs(False, False)
self.logger.info("opened for serving")
# Header merkle cache
@@ -280,30 +248,31 @@ class LevelDB:
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
hashes = b''.join(flush_data.block_tx_hashes)
flush_data.block_tx_hashes.clear()
assert len(hashes) % 32 == 0
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
start_time = time.perf_counter()
start_time = time.time()
height_start = self.fs_height + 1
tx_num = prior_tx_count
offset = self.header_offset(height_start)
self.headers_file.write(offset, b''.join(flush_data.headers))
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
flush_data.headers.clear()
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
tx_count = self.tx_counts[height_start]
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
tx_num += 1
offset += 32
offset = height_start * self.tx_counts.itemsize
self.tx_counts_file.write(offset,
self.tx_counts[height_start:].tobytes())
offset = prior_tx_count * 32
self.hashes_file.write(offset, hashes)
flush_data.block_tx_hashes.clear()
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
flush_data.headers.clear()
elapsed = time.perf_counter() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
if self.utxo_db.for_sync:
elapsed = time.time() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
def flush_history(self):
self.history.flush()
@@ -381,6 +350,28 @@ class LevelDB:
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
def fs_update_header_offsets(self, offset_start, height_start, headers):
if self.coin.STATIC_BLOCK_HEADERS:
return
offset = offset_start
offsets = []
for h in headers:
offset += len(h)
offsets.append(pack("<Q", offset))
# For each header we get the offset of the next header, hence we
# start writing from the next height
pos = (height_start + 1) * 8
self.headers_offsets_file.write(pos, b''.join(offsets))
def dynamic_header_offset(self, height):
assert not self.coin.STATIC_BLOCK_HEADERS
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
return offset
def dynamic_header_len(self, height):
return self.dynamic_header_offset(height + 1)\
- self.dynamic_header_offset(height)
def backup_fs(self, height, tx_count):
"""Back up during a reorg. This just updates our pointers."""
self.fs_height = height
@@ -412,13 +403,9 @@ class LevelDB:
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(
self.headers_db.iterator(
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
include_key=False
)
), disk_count
offset = self.header_offset(start_height)
size = self.header_offset(start_height + disk_count) - offset
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
@@ -431,7 +418,7 @@ class LevelDB:
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
tx_hash = self.hashes_file.read(tx_num * 32, 32)
return tx_hash, tx_height
async def fs_block_hashes(self, height, count):
@@ -441,8 +428,9 @@ class LevelDB:
offset = 0
headers = []
for n in range(count):
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
offset += self.coin.BASIC_HEADER_SIZE
hlen = self.header_len(height + n)
headers.append(headers_concat[offset:offset + hlen])
offset += hlen
return [self.coin.header_hash(header) for header in headers]
@@ -454,20 +442,9 @@ class LevelDB:
limit to None to get them all.
"""
def read_history():
hashx_history = []
for key, hist in self.history.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
hashx_history.append((tx_hash, tx_height))
if limit and len(hashx_history) >= limit:
return hashx_history
return hashx_history
tx_nums = list(self.history.get_txnums(hashX, limit))
fs_tx_hash = self.fs_tx_hash
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
@@ -497,20 +474,16 @@ class LevelDB:
batch_put(self.undo_key(height), b''.join(undo_info))
def raw_block_prefix(self):
return 'block'
return 'meta/block'
def raw_block_path(self, height):
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
async def read_raw_block(self, height):
def read_raw_block(self, height):
"""Returns a raw block read from disk. Raises FileNotFoundError
if the block isn't on-disk."""
def read():
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
def write_raw_block(self, block, height):
"""Write a raw block to disk."""
+1 -30
View File
@@ -13,7 +13,6 @@ import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
from prometheus_client import Histogram
import attr
@@ -80,16 +79,6 @@ class MemPoolAPI(ABC):
daemon's height at the time the mempool was obtained."""
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
mempool_process_time_metric = Histogram(
"processed_mempool", "Time to process mempool and notify touched addresses",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
class MemPool:
"""Representation of the daemon's mempool.
@@ -104,7 +93,7 @@ class MemPool:
hashXs: hashX -> set of all hashes of txs touching the hashX
"""
def __init__(self, coin, api, refresh_secs=1.0, log_status_secs=120.0):
def __init__(self, coin, api, refresh_secs=5.0, log_status_secs=120.0):
assert isinstance(api, MemPoolAPI)
self.coin = coin
self.api = api
@@ -118,7 +107,6 @@ class MemPool:
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
self.mempool_process_time_metric = mempool_process_time_metric
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
@@ -219,7 +207,6 @@ class MemPool:
async def _refresh_hashes(self, synchronized_event):
"""Refresh our view of the daemon's mempool."""
while True:
start = time.perf_counter()
height = self.api.cached_height()
hex_hashes = await self.api.mempool_hashes()
if height != await self.api.height():
@@ -230,8 +217,6 @@ class MemPool:
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(touched, height)
duration = time.perf_counter() - start
self.mempool_process_time_metric.observe(duration)
try:
# we wait up to `refresh_secs` but go early if a broadcast happens (which triggers wakeup event)
await asyncio.wait_for(self.wakeup.wait(), timeout=self.refresh_secs)
@@ -389,17 +374,3 @@ class MemPool:
if hX == hashX:
utxos.append(UTXO(-1, pos, tx_hash, 0, value))
return utxos
def get_mempool_height(self, tx_hash):
# Height Progression
# -2: not broadcast
# -1: in mempool but has unconfirmed inputs
# 0: in mempool and all inputs confirmed
# +num: confirmed in a specific block (height)
if tx_hash not in self.txs:
return -2
tx = self.txs[tx_hash]
unspent_inputs = sum(1 if hash in self.txs else 0 for hash, idx in tx.prevouts)
if unspent_inputs:
return -1
return 0
+82
View File
@@ -0,0 +1,82 @@
import os
from aiohttp import web
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
from lbry import __version__ as version
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server import util
import lbry.wallet.server.version as wallet_server_version
NAMESPACE = "wallet_server"
CPU_COUNT = f"{os.cpu_count()}"
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
VERSION_INFO.info({
'build': BUILD,
"commit": COMMIT_HASH,
"docker_tag": DOCKER_TAG,
'version': version,
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
"cpu_count": CPU_COUNT
})
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version", ))
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
namespace=NAMESPACE, labelnames=("method", "version"))
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
labelnames=("method", "version"))
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
)
SQLITE_INTERNAL_ERROR_COUNT = Counter(
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
)
SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE)
SQLITE_PENDING_COUNT = Gauge(
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
)
LBRYCRD_REQUEST_TIMES = Histogram(
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
)
LBRYCRD_PENDING_COUNT = Gauge(
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, labelnames=("method",)
)
CLIENT_VERSIONS = Counter(
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
BLOCK_COUNT = Gauge(
"block_count", "Number of processed blocks", namespace=NAMESPACE
)
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
class PrometheusServer:
def __init__(self):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.runner = None
async def start(self, port: int):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
await self.runner.cleanup()
+2 -2
View File
@@ -6,7 +6,7 @@ import typing
import lbry
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
from lbry.prometheus import PrometheusServer
from lbry.wallet.server.prometheus import PrometheusServer
class Notifications:
@@ -143,4 +143,4 @@ class Server:
async def start_prometheus(self):
if not self.prometheus_server and self.env.prometheus_port:
self.prometheus_server = PrometheusServer()
await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port)
await self.prometheus_server.start(self.env.prometheus_port)
+66 -147
View File
@@ -20,15 +20,16 @@ from functools import partial
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
from lbry.wallet.rpc.framing import NewlineFramer
import lbry.wallet.server.version as VERSION
@@ -118,62 +119,9 @@ class SessionGroup:
self.semaphore = asyncio.Semaphore(20)
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class SessionManager:
"""Holds global state about all sessions."""
version_info_metric = Info(
'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE
)
version_info_metric.info({
'build': BUILD,
"commit": COMMIT_HASH,
"docker_tag": DOCKER_TAG,
'version': lbry.__version__,
"min_version": util.version_string(VERSION.PROTOCOL_MIN),
"cpu_count": str(os.cpu_count())
})
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
)
db_error_metric = Counter(
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
)
executor_time_metric = Histogram(
"executor_time", "SQLite executor times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
pending_query_metric = Gauge(
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
)
client_version_metric = Counter(
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
address_history_metric = Histogram(
"address_history", "Time to fetch an address history",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
notifications_in_flight_metric = Gauge(
"notifications_in_flight", "Count of notifications in flight",
namespace=NAMESPACE
)
notifications_sent_metric = Histogram(
"notifications_sent", "Time to send an address notification",
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
shutdown_event: asyncio.Event):
env.max_send = max(350000, env.max_send)
@@ -190,7 +138,7 @@ class SessionManager:
self.cur_group = SessionGroup(0)
self.txs_sent = 0
self.start_time = time.time()
self.history_cache = self.bp.history_cache
self.history_cache = pylru.lrucache(256)
self.notified_height: typing.Optional[int] = None
# Cache some idea of room to avoid recounting on each subscription
self.subs_room = 0
@@ -615,25 +563,32 @@ class SessionManager:
async def broadcast_transaction(self, raw_tx):
hex_hash = await self.daemon.broadcast_transaction(raw_tx)
self.mempool.wakeup.set()
self.txs_sent += 1
return hex_hash
async def limited_history(self, hashX):
"""A caching layer."""
if hashX not in self.history_cache:
hc = self.history_cache
if hashX not in hc:
# History DoS limit. Each element of history is about 99
# bytes when encoded as JSON. This limits resource usage
# on bloated history requests, and uses a smaller divisor
# so large requests are logged before refusing them.
limit = self.env.max_send // 97
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
return self.history_cache[hashX]
hc[hashX] = await self.db.limited_history(hashX, limit=limit)
return hc[hashX]
async def _notify_sessions(self, height, touched):
"""Notify sessions about height changes and touched addresses."""
height_changed = height != self.notified_height
if height_changed:
await self._refresh_hsub_results(height)
# Invalidate our history cache for touched hashXs
hc = self.history_cache
for hashX in set(hc).intersection(touched):
del hc[hashX]
if self.sessions:
await asyncio.wait([
session.notify(touched, height_changed) for session in self.sessions
@@ -722,7 +677,7 @@ class SessionBase(RPCSession):
context = {'conn_id': f'{self.session_id}'}
self.logger = util.ConnectionLogger(self.logger, context)
self.group = self.session_mgr.add_session(self)
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
SESSIONS_COUNT.labels(version=self.client_version).inc()
peer_addr_str = self.peer_address_str()
self.logger.info(f'{self.kind} {peer_addr_str}, '
f'{self.session_mgr.session_count():,d} total')
@@ -731,7 +686,7 @@ class SessionBase(RPCSession):
"""Handle client disconnection."""
super().connection_lost(exc)
self.session_mgr.remove_session(self)
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
SESSIONS_COUNT.labels(version=self.client_version).dec()
msg = ''
if not self._can_send.is_set():
msg += ' whilst paused'
@@ -755,7 +710,7 @@ class SessionBase(RPCSession):
"""Handle an incoming request. ElectrumX doesn't receive
notifications from client sessions.
"""
self.session_mgr.request_count_metric.labels(method=request.method, version=self.client_version).inc()
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
if isinstance(request, Request):
handler = self.request_handlers.get(request.method)
handler = partial(handler, self)
@@ -846,8 +801,6 @@ class LBRYElectrumX(SessionBase):
'blockchain.scripthash.subscribe': cls.scripthash_subscribe,
'blockchain.transaction.broadcast': cls.transaction_broadcast,
'blockchain.transaction.get': cls.transaction_get,
'blockchain.transaction.get_batch': cls.transaction_get_batch,
'blockchain.transaction.info': cls.transaction_info,
'blockchain.transaction.get_merkle': cls.transaction_merkle,
'server.add_peer': cls.add_peer,
'server.banner': cls.banner,
@@ -933,42 +886,50 @@ class LBRYElectrumX(SessionBase):
"""
if height_changed and self.subscribe_headers:
args = (await self.subscribe_headers_result(), )
if not (await self.send_notification('blockchain.headers.subscribe', args)):
return
async def send_history_notification(alias, hashX):
start = time.perf_counter()
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
start = time.perf_counter()
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
await self.send_notification('blockchain.headers.subscribe', args)
except asyncio.TimeoutError:
self.logger.info("timeout sending headers notification to %s", self.peer_address_str(for_log=True))
self.abort()
return
touched = touched.intersection(self.hashX_subs)
if touched or (height_changed and self.mempool_statuses):
notified = set()
mempool_addrs = tuple(self.mempool_statuses.keys())
changed = {}
for hashX in touched:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
for hashX in mempool_addrs:
if hashX not in notified:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
status = await self.address_status(hashX)
changed[alias] = status
if touched:
es = '' if len(touched) == 1 else 'es'
self.logger.info(f'notified {len(notified)} mempool/{len(touched):,d} touched address{es}')
# Check mempool hashXs - the status is a function of the
# confirmed state of other transactions. Note: we cannot
# iterate over mempool_statuses as it changes size.
for hashX in tuple(self.mempool_statuses):
# Items can be evicted whilst await-ing status; False
# ensures such hashXs are notified
old_status = self.mempool_statuses.get(hashX, False)
status = await self.address_status(hashX)
if status != old_status:
alias = self.hashX_subs[hashX]
changed[alias] = status
for alias, status in changed.items():
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
await self.send_notification(method, (alias, status))
except asyncio.TimeoutError:
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
self.abort()
return
if changed:
es = '' if len(changed) == 1 else 'es'
self.logger.info(f'notified of {len(changed):,d} address{es}')
def get_metrics_or_placeholder_for_api(self, query_name):
""" Do not hold on to a reference to the metrics
@@ -983,7 +944,7 @@ class LBRYElectrumX(SessionBase):
async def run_in_executor(self, query_name, func, kwargs):
start = time.perf_counter()
try:
self.session_mgr.pending_query_metric.inc()
SQLITE_PENDING_COUNT.inc()
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
@@ -992,18 +953,18 @@ class LBRYElectrumX(SessionBase):
except reader.SQLiteInterruptedError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_interrupt(start, error.metrics)
self.session_mgr.interrupt_count_metric.inc()
SQLITE_INTERRUPT_COUNT.inc()
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, error.metrics)
self.session_mgr.db_operational_error_metric.inc()
SQLITE_OPERATIONAL_ERROR_COUNT.inc()
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except Exception:
log.exception("dear devs, please handle this exception better")
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, {})
self.session_mgr.db_error_metric.inc()
SQLITE_INTERNAL_ERROR_COUNT.inc()
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
else:
if self.env.track_metrics:
@@ -1012,8 +973,8 @@ class LBRYElectrumX(SessionBase):
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
finally:
self.session_mgr.pending_query_metric.dec()
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
SQLITE_PENDING_COUNT.dec()
SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start)
async def run_and_cache_query(self, query_name, function, kwargs):
metrics = self.get_metrics_or_placeholder_for_api(query_name)
@@ -1178,7 +1139,6 @@ class LBRYElectrumX(SessionBase):
"""
# Note history is ordered and mempool unordered in electrum-server
# For mempool, height is -1 if it has unconfirmed inputs, otherwise 0
db_history = await self.session_mgr.limited_history(hashX)
mempool = await self.mempool.transaction_summaries(hashX)
@@ -1197,6 +1157,7 @@ class LBRYElectrumX(SessionBase):
self.mempool_statuses[hashX] = status
else:
self.mempool_statuses.pop(hashX, None)
return status
async def hashX_listunspent(self, hashX):
@@ -1218,10 +1179,7 @@ class LBRYElectrumX(SessionBase):
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
try:
del self.hashX_subs[hashX]
except ValueError:
pass
del self.hashX_subs[hashX]
def address_to_hashX(self, address):
try:
@@ -1483,10 +1441,10 @@ class LBRYElectrumX(SessionBase):
raise RPCError(BAD_REQUEST,
f'unsupported client: {client_name}')
if self.client_version != client_name[:17]:
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
SESSIONS_COUNT.labels(version=self.client_version).dec()
self.client_version = client_name[:17]
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
self.session_mgr.client_version_metric.labels(version=self.client_version).inc()
SESSIONS_COUNT.labels(version=self.client_version).inc()
CLIENT_VERSIONS.labels(version=self.client_version).inc()
# Find the highest common protocol version. Disconnect if
# that protocol version in unsupported.
@@ -1512,7 +1470,6 @@ class LBRYElectrumX(SessionBase):
try:
hex_hash = await self.session_mgr.broadcast_transaction(raw_tx)
self.txs_sent += 1
self.mempool.wakeup.set()
self.logger.info(f'sent tx: {hex_hash}')
return hex_hash
except DaemonError as e:
@@ -1522,44 +1479,6 @@ class LBRYElectrumX(SessionBase):
raise RPCError(BAD_REQUEST, 'the transaction was rejected by '
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
async def transaction_get_batch(self, *tx_hashes):
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = {}
for tx_hash in tx_hashes:
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
merkle = {}
if block_hash:
block = await self.daemon.deserialised_block(block_hash)
height = block['height']
try:
pos = block['tx'].index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
merkle["pos"] = pos
else:
height = -1
merkle['block_height'] = height
batch_result[tx_hash] = [raw_tx, merkle]
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
"""Return the serialized raw transaction given its hash
@@ -1582,7 +1501,7 @@ class LBRYElectrumX(SessionBase):
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon.deserialised_block(block_hash)
block = await self.daemon_request('deserialised_block', block_hash)
return block_hash, block['tx']
def _get_merkle_branch(self, tx_hashes, tx_pos):
+1 -3
View File
@@ -333,13 +333,11 @@ unpack_le_uint64_from = struct_le_Q.unpack_from
unpack_be_uint16_from = struct_be_H.unpack_from
unpack_be_uint32_from = struct_be_I.unpack_from
unpack_be_uint64 = lambda x: int.from_bytes(x, byteorder='big')
pack_le_int32 = struct_le_i.pack
pack_le_int64 = struct_le_q.pack
pack_le_uint16 = struct_le_H.pack
pack_le_uint32 = struct_le_I.pack
pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big')
pack_le_uint64 = struct_le_Q.pack
pack_be_uint16 = struct_be_H.pack
pack_be_uint32 = struct_be_I.pack
pack_byte = structB.pack
-7
View File
@@ -7,15 +7,10 @@ class TaskGroup:
self._loop = loop or get_event_loop()
self._tasks = set()
self.done = Event()
self.started = Event()
def __len__(self):
return len(self._tasks)
def add(self, coro):
task = self._loop.create_task(coro)
self._tasks.add(task)
self.started.set()
self.done.clear()
task.add_done_callback(self._remove)
return task
@@ -24,10 +19,8 @@ class TaskGroup:
self._tasks.remove(task)
if len(self._tasks) < 1:
self.done.set()
self.started.clear()
def cancel(self):
for task in self._tasks:
task.cancel()
self.done.set()
self.started.clear()
+51 -120
View File
@@ -2,7 +2,6 @@ import struct
import hashlib
import logging
import typing
import asyncio
from binascii import hexlify, unhexlify
from typing import List, Iterable, Optional, Tuple
@@ -19,9 +18,7 @@ from lbry.crypto.hash import hash160, sha256
from lbry.crypto.base58 import Base58
from lbry.schema.url import normalize_name
from lbry.schema.claim import Claim
from lbry.schema.base import Signable
from lbry.schema.purchase import Purchase
from lbry.schema.support import Support
from .script import InputScript, OutputScript
from .constants import COIN, NULL_HASH32
@@ -162,11 +159,11 @@ class Input(InputOutput):
return self.txo_ref.txo.amount
@property
def is_my_input(self) -> Optional[bool]:
def is_my_account(self) -> Optional[bool]:
""" True if the output this input spends is yours. """
if self.txo_ref.txo is None:
return False
return self.txo_ref.txo.is_my_output
return self.txo_ref.txo.is_my_account
@classmethod
def deserialize_from(cls, stream):
@@ -210,30 +207,25 @@ class OutputEffectiveAmountEstimator:
class Output(InputOutput):
__slots__ = (
'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input',
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account',
'channel', 'private_key', 'meta',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims', '_signable'
'reposted_claim', 'claims',
)
def __init__(self, amount: int, script: OutputScript,
tx_ref: TXRef = None, position: int = None,
is_internal_transfer: Optional[bool] = None, is_spent: Optional[bool] = None,
is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None,
sent_supports: Optional[int] = None, sent_tips: Optional[int] = None,
received_tips: Optional[int] = None,
is_change: Optional[bool] = None, is_spent: Optional[bool] = None,
is_received: Optional[bool] = None, is_my_account: Optional[bool] = None,
channel: Optional['Output'] = None, private_key: Optional[str] = None
) -> None:
super().__init__(tx_ref, position)
self.amount = amount
self.script = script
self.is_internal_transfer = is_internal_transfer
self.is_change = is_change
self.is_spent = is_spent
self.is_my_output = is_my_output
self.is_my_input = is_my_input
self.sent_supports = sent_supports
self.sent_tips = sent_tips
self.received_tips = received_tips
self.is_received = is_received
self.is_my_account = is_my_account
self.channel = channel
self.private_key = private_key
self.purchase: 'Output' = None # txo containing purchase metadata
@@ -241,26 +233,17 @@ class Output(InputOutput):
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
self.reposted_claim: 'Output' = None # txo representing claim being reposted
self.claims: List['Output'] = None # resolved claims for collection
self._signable: Optional[Signable] = None
self.meta = {}
def update_annotations(self, annotated: 'Output'):
def update_annotations(self, annotated):
if annotated is None:
self.is_internal_transfer = None
self.is_change = None
self.is_spent = None
self.is_my_output = None
self.is_my_input = None
self.sent_supports = None
self.sent_tips = None
self.received_tips = None
self.is_my_account = None
else:
self.is_internal_transfer = annotated.is_internal_transfer
self.is_change = annotated.is_change
self.is_spent = annotated.is_spent
self.is_my_output = annotated.is_my_output
self.is_my_input = annotated.is_my_input
self.sent_supports = annotated.sent_supports
self.sent_tips = annotated.sent_tips
self.received_tips = annotated.received_tips
self.is_my_account = annotated.is_my_account
self.channel = annotated.channel if annotated else None
self.private_key = annotated.private_key if annotated else None
@@ -315,10 +298,6 @@ class Output(InputOutput):
def is_support(self) -> bool:
return self.script.is_support_claim
@property
def is_support_data(self) -> bool:
return self.script.is_support_claim_data
@property
def claim_hash(self) -> bytes:
if self.script.is_claim_name:
@@ -354,33 +333,9 @@ class Output(InputOutput):
def can_decode_claim(self):
try:
return self.claim
except Exception:
except: # pylint: disable=bare-except
return False
@property
def support(self) -> Support:
if self.is_support_data:
if not isinstance(self.script.values['support'], Support):
self.script.values['support'] = Support.from_bytes(self.script.values['support'])
return self.script.values['support']
raise ValueError('Only supports with data can be represented as Supports.')
@property
def can_decode_support(self):
try:
return self.support
except Exception:
return False
@property
def signable(self) -> Signable:
if self._signable is None:
if self.is_claim:
self._signable = self.claim
elif self.is_support_data:
self._signable = self.support
return self._signable
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
@@ -392,22 +347,22 @@ class Output(InputOutput):
return self.private_key is not None
def get_signature_digest(self, ledger):
if self.signable.unsigned_payload:
if self.claim.unsigned_payload:
pieces = [
Base58.decode(self.get_address(ledger)),
self.signable.unsigned_payload,
self.signable.signing_channel_hash[::-1]
self.claim.unsigned_payload,
self.claim.signing_channel_hash[::-1]
]
else:
pieces = [
self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]
return sha256(b''.join(pieces))
def get_encoded_signature(self):
signature = hexlify(self.signable.signature)
signature = hexlify(self.claim.signature)
r = int(signature[:int(len(signature)/2)], 16)
s = int(signature[int(len(signature)/2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
@@ -431,23 +386,21 @@ class Output(InputOutput):
def sign(self, channel: 'Output', first_input_id=None):
self.channel = channel
self.signable.signing_channel_hash = channel.claim_hash
self.claim.signing_channel_hash = channel.claim_hash
digest = sha256(b''.join([
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]))
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.claim.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.script.generate()
def clear_signature(self):
self.channel = None
self.signable.clear_signature()
self.claim.clear_signature()
async def generate_channel_private_key(self):
self.private_key = await asyncio.get_event_loop().run_in_executor(
None, ecdsa.SigningKey.generate, ecdsa.SECP256k1, None, hashlib.sha256
)
def generate_channel_private_key(self):
self.private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der()
self.script.generate()
return self.private_key
@@ -477,14 +430,6 @@ class Output(InputOutput):
)
return cls(amount, script)
@classmethod
def pay_support_data_pubkey_hash(
cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_support_data_pubkey_hash(
claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash
)
return cls(amount, script)
@classmethod
def add_purchase_data(cls, purchase: Purchase) -> 'Output':
script = OutputScript.return_data(purchase)
@@ -536,7 +481,7 @@ class Output(InputOutput):
class Transaction:
def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
height: int = -2, position: int = -1, julian_day: int = None) -> None:
height: int = -2, position: int = -1) -> None:
self._raw = raw
self._raw_sans_segwit = None
self.is_segwit_flag = 0
@@ -554,7 +499,6 @@ class Transaction:
# +num: confirmed in a specific block (height)
self.height = height
self.position = position
self._day = julian_day
if raw is not None:
self._deserialize()
@@ -578,11 +522,6 @@ class Transaction:
def hash(self):
return self.ref.hash
def get_julian_day(self, ledger):
if self._day is None and self.height > 0:
self._day = ledger.headers.estimated_julian_day(self.height)
return self._day
@property
def raw(self):
if self._raw is None:
@@ -653,21 +592,21 @@ class Transaction:
for txi in self.inputs:
if txi.txo_ref.txo is None:
continue
if txi.is_my_input is True:
if txi.is_my_account is None:
raise ValueError(
"Cannot access net_account_balance if inputs/outputs do not "
"have is_my_account set properly."
)
if txi.is_my_account:
balance -= txi.amount
elif txi.is_my_input is None:
raise ValueError(
"Cannot access net_account_balance if inputs do not "
"have is_my_input set properly."
)
for txo in self.outputs:
if txo.is_my_output is True:
balance += txo.amount
elif txo.is_my_output is None:
if txo.is_my_account is None:
raise ValueError(
"Cannot access net_account_balance if outputs do not "
"have is_my_output set properly."
"Cannot access net_account_balance if inputs/outputs do not "
"have is_my_account set properly."
)
if txo.is_my_account:
balance += txo.amount
return balance
@property
@@ -812,7 +751,7 @@ class Transaction:
change_hash160 = change_account.ledger.address_to_hash160(change_address)
change_amount = change - cost_of_change
change_output = Output.pay_pubkey_hash(change_amount, change_hash160)
change_output.is_internal_transfer = True
change_output.is_change = True
tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)])
if tx._outputs:
@@ -899,20 +838,12 @@ class Transaction:
@classmethod
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None):
funding_accounts: List['Account'], change_account: 'Account'):
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
if signing_channel is not None:
support = Support()
support.emoji = '👍'
support_output = Output.pay_support_data_pubkey_hash(
amount, claim_name, claim_id, support, ledger.address_to_hash160(holding_address)
)
support_output.sign(signing_channel, b'placeholder txid:nout')
else:
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
return cls.create([], [support_output], funding_accounts, change_account, sign=False)
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
return cls.create([], [support_output], funding_accounts, change_account)
@classmethod
def purchase(cls, claim_id: str, amount: int, merchant_address: bytes,
@@ -925,17 +856,17 @@ class Transaction:
@property
def my_inputs(self):
for txi in self.inputs:
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output:
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_account:
yield txi
def _filter_my_outputs(self, f):
for txo in self.outputs:
if txo.is_my_output and f(txo.script):
if txo.is_my_account and f(txo.script):
yield txo
def _filter_other_outputs(self, f):
for txo in self.outputs:
if not txo.is_my_output and f(txo.script):
if not txo.is_my_account and f(txo.script):
yield txo
def _filter_any_outputs(self, f):
@@ -967,7 +898,7 @@ class Transaction:
def my_abandon_outputs(self):
for txi in self.inputs:
abandon = txi.txo_ref.txo
if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved:
if abandon is not None and abandon.is_my_account and abandon.script.is_claim_involved:
is_update = False
if abandon.script.is_claim_name or abandon.script.is_update_claim:
for update in self.my_update_outputs:
-4
View File
@@ -3,10 +3,6 @@ from typing import TypeVar, Sequence, Optional
from .constants import COIN
def date_to_julian_day(d):
return d.toordinal() + 1721424.5
def coins_to_satoshis(coins):
if not isinstance(coins, str):
raise ValueError("{coins} must be a string")
+5 -2
View File
@@ -17,7 +17,7 @@ def enable_logging():
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
handler.setFormatter(formatter)
root.addHandler(handler)
@@ -33,7 +33,7 @@ async def process_video(analyzer, video_file):
transcode = input("Would you like to make a repaired clone now? [y/N] ")
if transcode == "y":
try:
new_video_file, _ = await analyzer.verify_or_repair(True, True, video_file)
new_video_file = await analyzer.verify_or_repair(True, True, video_file)
print("Successfully created ", new_video_file)
except Exception as e:
print("Unable to complete the transcode. Message: ", str(e))
@@ -49,6 +49,9 @@ def main():
video_file = sys.argv[1]
conf = TranscodeConfig()
analyzer = VideoFileAnalyzer(conf)
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
try:
asyncio.run(process_video(analyzer, video_file))
except KeyboardInterrupt:
-35
View File
@@ -1,35 +0,0 @@
import asyncio
import os
from lbry.extras.cli import ensure_directory_exists
from lbry.conf import Config
from lbry.wallet.header import Headers
import lbry.wallet.checkpoints
async def main():
outpath = lbry.wallet.checkpoints.__file__
ledger_path = os.path.join(Config().wallet_dir, 'lbc_mainnet')
ensure_directory_exists(ledger_path)
headers_path = os.path.join(ledger_path, 'headers')
headers = Headers(headers_path)
await headers.open()
print(f"Working on headers at {outpath}")
print("Verifying integrity, might take a while.")
await headers.repair()
target = ((headers.height - 100) // 1000) * 1000
current_checkpoint_tip = max(lbry.wallet.checkpoints.HASHES.keys())
if target <= current_checkpoint_tip:
print(f"We have nothing to add: Local: {target}, checkpoint: {current_checkpoint_tip}")
return
print(f"Headers file at {headers.height}, checkpointing up to {target}."
f"Current checkpoint at {current_checkpoint_tip}.")
with open(outpath, 'w') as outfile:
print('HASHES = {', file=outfile)
for height in range(0, target, 1000):
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
print('}', file=outfile)
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(main())
-1
View File
@@ -3,7 +3,6 @@ import os
import asyncio
import socket
import ipaddress
import lbry.wallet
from lbry.conf import Config
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.blob.blob_manager import BlobManager
+5 -11
View File
@@ -59,8 +59,7 @@ def get_label(pr, prefix):
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
RELEASE_TEXT = 'release-text:'
RELEASE_TEXT_LINES = 'release-text-lines:'
RELEASE_TEXT = "release-text:"
def get_backwards_incompatible(desc: str):
@@ -70,15 +69,9 @@ def get_backwards_incompatible(desc: str):
def get_release_text(desc: str):
in_release_lines = False
for line in desc.splitlines():
if in_release_lines:
yield line.rstrip()
elif line.startswith(RELEASE_TEXT_LINES):
in_release_lines = True
elif line.startswith(RELEASE_TEXT):
yield line[len(RELEASE_TEXT):].strip()
yield ''
if line.startswith(RELEASE_TEXT):
yield line[len(RELEASE_TEXT):]
class Version:
@@ -156,7 +149,7 @@ def release(args):
for incompat in get_backwards_incompatible(pr.body or ""):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body or ""):
release_texts.append(release_text)
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
if type_label == 'fixup':
fixups.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
else:
@@ -176,6 +169,7 @@ def release(args):
w('')
for release_text in release_texts:
w(release_text)
w('')
if incompats:
w('')
w(f'### Backwards Incompatible Changes')
-1
View File
@@ -35,7 +35,6 @@ disable=
too-many-statements,
too-many-nested-blocks,
too-many-public-methods,
too-many-return-statements,
too-many-instance-attributes,
protected-access,
unused-argument
+1 -1
View File
@@ -46,7 +46,7 @@ setup(
'msgpack==0.6.1',
'prometheus_client==0.7.1',
'ecdsa==0.13.3',
'pyyaml==5.3.1',
'pyyaml==4.2b1',
'docopt==0.6.2',
'hachoir',
'multidict==4.6.1',
@@ -1,120 +1,33 @@
import logging
import asyncio
from binascii import hexlify
from lbry.testcase import CommandTestCase
from lbry.testcase import IntegrationTestCase
class BlockchainReorganizationTests(CommandTestCase):
class BlockchainReorganizationTests(IntegrationTestCase):
VERBOSITY = logging.WARN
async def assertBlockHash(self, height):
self.assertEqual(
(await self.ledger.headers.hash(height)).decode(),
self.ledger.headers.hash(height).decode(),
await self.blockchain.get_block_hash(height)
)
async def test_reorg(self):
bp = self.conductor.spv_node.server.bp
bp.reorg_count_metric.set(0)
# invalidate current block, move forward 2
self.assertEqual(self.ledger.headers.height, 206)
await self.assertBlockHash(206)
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
self.assertEqual(self.ledger.headers.height, 200)
await self.assertBlockHash(200)
await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode())
await self.blockchain.generate(2)
await self.ledger.on_header.where(lambda e: e.height == 207)
self.assertEqual(self.ledger.headers.height, 207)
await self.assertBlockHash(206)
await self.assertBlockHash(207)
self.assertEqual(1, bp.reorg_count_metric._samples()[0][2])
await self.ledger.on_header.where(lambda e: e.height == 201)
self.assertEqual(self.ledger.headers.height, 201)
await self.assertBlockHash(200)
await self.assertBlockHash(201)
# invalidate current block, move forward 3
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode())
await self.blockchain.generate(3)
await self.ledger.on_header.where(lambda e: e.height == 208)
self.assertEqual(self.ledger.headers.height, 208)
await self.assertBlockHash(206)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
async def test_reorg_change_claim_height(self):
# sanity check
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertListEqual(txos, [])
still_valid = await self.daemon.jsonrpc_stream_create(
'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
await self.ledger.wait(still_valid)
await self.generate(1)
# create a claim and verify it's returned by claim_search
self.assertEqual(self.ledger.headers.height, 207)
broadcast_tx = await self.daemon.jsonrpc_stream_create(
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
await self.ledger.wait(broadcast_tx)
await self.generate(1)
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
self.assertEqual(self.ledger.headers.height, 208)
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
txo = txos[0]
self.assertEqual(txo.tx_ref.id, broadcast_tx.id)
self.assertEqual(txo.tx_ref.height, 208)
# check that our tx is in block 208 as returned by lbrycrdd
invalidated_block_hash = (await self.ledger.headers.hash(208)).decode()
block_207 = await self.blockchain.get_block(invalidated_block_hash)
self.assertIn(txo.tx_ref.id, block_207['tx'])
self.assertEqual(208, txos[0].tx_ref.height)
# reorg the last block dropping our claim tx
await self.blockchain.invalidate_block(invalidated_block_hash)
await self.blockchain.clear_mempool()
await self.blockchain.generate(2)
# verify the claim was dropped from block 208 as returned by lbrycrdd
reorg_block_hash = await self.blockchain.get_block_hash(208)
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
block_207 = await self.blockchain.get_block(reorg_block_hash)
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
# verify the dropped claim is no longer returned by claim search
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertListEqual(txos, [])
# verify the claim published a block earlier wasn't also reverted
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
self.assertEqual(1, len(txos))
self.assertEqual(207, txos[0].tx_ref.height)
# broadcast the claim in a different block
new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode())
self.assertEqual(broadcast_tx.id, new_txid)
await self.blockchain.generate(1)
# wait for the client to catch up
await asyncio.wait_for(self.on_header(210), 1.0)
# verify the claim is in the new block and that it is returned by claim_search
block_210 = await self.blockchain.get_block((await self.ledger.headers.hash(210)).decode())
self.assertIn(txo.tx_ref.id, block_210['tx'])
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
self.assertEqual(txos[0].tx_ref.id, new_txid)
self.assertEqual(210, txos[0].tx_ref.height)
# this should still be unchanged
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
self.assertEqual(1, len(txos))
self.assertEqual(207, txos[0].tx_ref.height)
await self.ledger.on_header.where(lambda e: e.height == 202)
self.assertEqual(self.ledger.headers.height, 202)
await self.assertBlockHash(200)
await self.assertBlockHash(201)
await self.assertBlockHash(202)
@@ -1,7 +1,6 @@
import os.path
import tempfile
import logging
import asyncio
from binascii import unhexlify
from urllib.request import urlopen
@@ -10,7 +9,6 @@ from lbry.error import InsufficientFundsError
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
from lbry.testcase import CommandTestCase
from lbry.wallet.transaction import Transaction
from lbry.wallet.util import satoshis_to_coins as lbc
log = logging.getLogger(__name__)
@@ -80,12 +78,6 @@ class ClaimSearchCommand(ClaimTestCase):
] * 23828
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
# this should do nothing... if the resolve (which is retried) results in the server disconnecting,
# it kerplodes
await asyncio.wait_for(self.daemon.jsonrpc_resolve([
f'0000000000000000000000000000000000000000{i}' for i in range(30000)
]), 30)
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
@@ -428,21 +420,11 @@ class TransactionCommands(ClaimTestCase):
class TransactionOutputCommands(ClaimTestCase):
async def test_txo_list_and_sum_filtering(self):
async def test_txo_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
await self.channel_update(channel_id, bid='0.5')
self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
self.assertEqual('1.5', lbc(await self.txo_sum(type='channel')))
stream_id = self.get_claim_id(await self.stream_create(bid='1.3'))
self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
await self.stream_update(stream_id, bid='0.7')
self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
self.assertEqual('2.0', lbc(await self.txo_sum(type='stream')))
self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], is_not_spent=True)))
self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel'])))
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
# type filtering
r = await self.txo_list(type='channel')
@@ -498,161 +480,25 @@ class TransactionOutputCommands(ClaimTestCase):
self.assertTrue(r[0]['is_spent'])
self.assertTrue(r[1]['is_spent'])
async def test_txo_list_my_input_output_filtering(self):
async def test_txo_list_received_filtering(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create('@kept-channel')
await self.channel_create('@sent-channel', claim_address=address2)
await self.wallet_send('2.9', address2)
await self.channel_create(claim_address=address2)
# all txos on second wallet
received_payment, received_channel = await self.txo_list(
wallet_id=wallet2.id, is_my_input_or_output=True)
self.assertEqual('1.0', received_channel['amount'])
self.assertFalse(received_channel['is_my_input'])
self.assertTrue(received_channel['is_my_output'])
self.assertFalse(received_channel['is_internal_transfer'])
self.assertEqual('2.9', received_payment['amount'])
self.assertFalse(received_payment['is_my_input'])
self.assertTrue(received_payment['is_my_output'])
self.assertFalse(received_payment['is_internal_transfer'])
r = await self.txo_list(include_is_received=True)
self.assertEqual(2, len(r))
self.assertFalse(r[0]['is_received'])
self.assertTrue(r[1]['is_received'])
rt = await self.txo_list(is_not_received=True)
self.assertEqual(1, len(rt))
self.assertEqual(rt[0], r[0])
rf = await self.txo_list(is_received=True)
self.assertEqual(1, len(rf))
self.assertEqual(rf[0], r[1])
# all txos on default wallet
r = await self.txo_list(is_my_input_or_output=True)
self.assertEqual(
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
[t['amount'] for t in r]
)
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_payment['is_my_input'])
self.assertFalse(sent_payment['is_my_output'])
self.assertFalse(sent_payment['is_internal_transfer'])
self.assertTrue(change3['is_my_input'])
self.assertTrue(change3['is_my_output'])
self.assertTrue(change3['is_internal_transfer'])
self.assertTrue(sent_channel['is_my_input'])
self.assertFalse(sent_channel['is_my_output'])
self.assertFalse(sent_channel['is_internal_transfer'])
self.assertTrue(change2['is_my_input'])
self.assertTrue(change2['is_my_output'])
self.assertTrue(change2['is_internal_transfer'])
self.assertTrue(kept_channel['is_my_input'])
self.assertTrue(kept_channel['is_my_output'])
self.assertFalse(kept_channel['is_internal_transfer'])
self.assertTrue(change1['is_my_input'])
self.assertTrue(change1['is_my_output'])
self.assertTrue(change1['is_internal_transfer'])
self.assertFalse(initial_funds['is_my_input'])
self.assertTrue(initial_funds['is_my_output'])
self.assertFalse(initial_funds['is_internal_transfer'])
# my stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
# my unspent stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
# only "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
self.assertEqual([change3, change2, change1], r)
# only unspent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
self.assertEqual([change3], r)
# only spent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
self.assertEqual([change2, change1], r)
# all my unspent stuff
r = await self.txo_list(is_my_output=True, is_not_spent=True)
self.assertEqual([change3, kept_channel], r)
# stuff i sent
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_payment, sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
stream_id = self.get_claim_id(await self.stream_create())
await self.support_create(stream_id, '0.3')
await self.support_create(stream_id, '0.2')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.4')
await self.support_create(stream_id, '0.5')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.6')
plot = await self.txo_plot(type='support')
self.assertEqual([
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=1)
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
async def test_txo_spend(self):
stream_id = self.get_claim_id(await self.stream_create())
for _ in range(10):
await self.support_create(stream_id, '0.1')
await self.assertBalance(self.account, '7.978478')
self.assertEqual('1.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
txs = await self.txo_spend(type='support', batch_size=3, include_full_tx=True)
self.assertEqual(4, len(txs))
self.assertEqual(3, len(txs[0]['inputs']))
self.assertEqual(3, len(txs[1]['inputs']))
self.assertEqual(3, len(txs[2]['inputs']))
self.assertEqual(1, len(txs[3]['inputs']))
self.assertEqual('0.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
await self.assertBalance(self.account, '8.977606')
await self.support_create(stream_id, '0.1')
txs = await self.daemon.jsonrpc_txo_spend(type='support', batch_size=3)
self.assertEqual(1, len(txs))
self.assertEqual({'txid'}, set(txs[0]))
r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id)
self.assertEqual(1, len(r))
self.assertTrue(r[0]['is_received'])
class ClaimCommands(ClaimTestCase):
@@ -661,15 +507,12 @@ class ClaimCommands(ClaimTestCase):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, title='foo')
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
# catch a bug where cli sends is_spent=False by default
r = await self.claim_list(claim_type='stream', is_spent=False)
r = await self.claim_list(claim_type='stream')
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
@@ -774,61 +617,7 @@ class ClaimCommands(ClaimTestCase):
self.assertTrue(r[1]['meta']['is_controlling'])
# check that metadata is transfered
self.assertTrue(r[0]['is_my_output'])
async def assertClaimList(self, claim_ids, **kwargs):
self.assertEqual(claim_ids, [c['claim_id'] for c in await self.claim_list(**kwargs)])
async def test_list_streams_in_channel_and_order_by(self):
channel1_id = self.get_claim_id(await self.channel_create('@chan-one'))
channel2_id = self.get_claim_id(await self.channel_create('@chan-two'))
stream1_id = self.get_claim_id(await self.stream_create('stream-a', bid='0.3', channel_id=channel1_id))
stream2_id = self.get_claim_id(await self.stream_create('stream-b', bid='0.9', channel_id=channel1_id))
stream3_id = self.get_claim_id(await self.stream_create('stream-c', bid='0.6', channel_id=channel2_id))
await self.assertClaimList([stream2_id, stream1_id], channel_id=channel1_id)
await self.assertClaimList([stream3_id], channel_id=channel2_id)
await self.assertClaimList([stream3_id, stream2_id, stream1_id], channel_id=[channel1_id, channel2_id])
await self.assertClaimList([stream1_id, stream2_id, stream3_id], claim_type='stream', order_by='name')
await self.assertClaimList([stream1_id, stream3_id, stream2_id], claim_type='stream', order_by='amount')
await self.assertClaimList([stream3_id, stream2_id, stream1_id], claim_type='stream', order_by='height')
async def test_claim_list_with_tips(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.wallet_send('5.0', address2)
stream1_id = self.get_claim_id(await self.stream_create('one'))
stream2_id = self.get_claim_id(await self.stream_create('two'))
claims = await self.claim_list()
self.assertNotIn('received_tips', claims[0])
self.assertNotIn('received_tips', claims[1])
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('0.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
await self.support_create(stream1_id, '0.7', tip=True)
await self.support_create(stream1_id, '0.3', tip=True, wallet_id=wallet2.id)
await self.support_create(stream1_id, '0.2', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.4', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.5', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.1', tip=True, wallet_id=wallet2.id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('1.0', claims[0]['received_tips'])
self.assertEqual('1.2', claims[1]['received_tips'])
await self.support_abandon(stream1_id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('1.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
await self.support_abandon(stream2_id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('0.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
self.assertTrue(r[0]['is_mine'])
class ChannelCommands(CommandTestCase):
@@ -1179,16 +968,12 @@ class StreamCommands(ClaimTestCase):
claim_id = self.get_claim_id(tx)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
self.assertItemCount(await self.daemon.jsonrpc_txo_list(reposted_claim_id=claim_id), 0)
self.assertItemCount(await self.daemon.jsonrpc_txo_list(type='repost'), 0)
tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1')
repost_id = self.get_claim_id(tx)
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
self.assertEqual((await self.txo_list(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
self.assertEqual((await self.txo_list(type='repost'))[0]['claim_id'], repost_id)
# tags are inherited (non-common / indexed tags)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2)
@@ -1828,8 +1613,7 @@ class SupportCommands(CommandTestCase):
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2',
funding_account_ids=[account2.id])
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(tip['txid'])
@@ -1860,8 +1644,7 @@ class SupportCommands(CommandTestCase):
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2',
funding_account_ids=[account2.id])
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(support['txid'])
@@ -1888,12 +1671,6 @@ class SupportCommands(CommandTestCase):
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
async def test_signed_supports(self):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
tx = await self.support_create(stream_id, '0.3', channel_id=channel_id)
self.assertTrue(tx['outputs'][0]['is_channel_signature_valid'])
class CollectionCommands(CommandTestCase):
@@ -31,7 +31,7 @@ class BasicTransactionTest(IntegrationTestCase):
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
await channel_txo.generate_channel_private_key()
channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
+8 -5
View File
@@ -1,3 +1,4 @@
import logging
import asyncio
import lbry
@@ -89,11 +90,13 @@ class ReconnectTests(IntegrationTestCase):
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
await asyncio.sleep(0.1)
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
await self.ledger.headers.open()
await self.ledger.network.start()
await self.ledger.network.on_connected.first
await self.ledger.initial_headers_sync()
self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height)
# locks header processing so we make sure we are the only ones modifying it
async with self.ledger._header_processing_lock:
await self.ledger.headers.open()
await self.ledger.network.start()
await self.ledger.network.on_connected.first
await self.ledger.initial_headers_sync()
self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height)
async def test_connection_drop_still_receives_events_after_reconnected(self):
address1 = await self.account.receiving.get_or_create_usable_address()
@@ -142,12 +142,12 @@ class PurchaseCommandTests(CommandTestCase):
self.assertEqual(result['items'][0]['purchase_info'][0]['claim_id'], claim_id2)
self.assertEqual(result['items'][2]['purchase_info'][0]['claim_id'], claim_id1)
result = await self.claim_search(include_purchase_receipt=True)
result = await self.claim_search()
self.assertEqual(result[0]['claim_id'], result[0]['purchase_receipt']['claim_id'])
self.assertEqual(result[1]['claim_id'], result[1]['purchase_receipt']['claim_id'])
url = result[0]['canonical_url']
resolve = await self.resolve(url, include_purchase_receipt=True)
resolve = await self.resolve(url)
self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id'])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
@@ -49,11 +49,11 @@ class ResolveCommand(BaseResolveTestCase):
self.assertTrue(claim['is_channel_signature_valid'])
self.assertEqual(
claim['timestamp'],
self.ledger.headers.estimated_timestamp(claim['height'])
self.ledger.headers[claim['height']]['timestamp']
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers.estimated_timestamp(claim['signing_channel']['height'])
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
)
# resolving claim foo by itself
@@ -87,6 +87,7 @@ class ResolveCommand(BaseResolveTestCase):
await self.conductor.spv_node.stop()
resolve_task = asyncio.create_task(self.resolve('foo'))
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.on_ready.first
self.assertIsNotNone((await resolve_task)['claim_id'])
async def test_winning_by_effective_amount(self):
@@ -262,73 +263,6 @@ class ResolveCommand(BaseResolveTestCase):
await self.resolve('@olds/bad_example')
)
async def test_resolve_with_includes(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.wallet_send('1.0', address2)
stream = await self.stream_create(
'priced', '0.1', wallet_id=wallet2.id,
fee_amount='0.5', fee_currency='LBC', fee_address=address2
)
stream_id = self.get_claim_id(stream)
resolve = await self.resolve('priced')
self.assertNotIn('is_my_output', resolve)
self.assertNotIn('purchase_receipt', resolve)
self.assertNotIn('sent_supports', resolve)
self.assertNotIn('sent_tips', resolve)
self.assertNotIn('received_tips', resolve)
# is_my_output
resolve = await self.resolve('priced', include_is_my_output=True)
self.assertFalse(resolve['is_my_output'])
resolve = await self.resolve('priced', wallet_id=wallet2.id, include_is_my_output=True)
self.assertTrue(resolve['is_my_output'])
# purchase receipt
resolve = await self.resolve('priced', include_purchase_receipt=True)
self.assertNotIn('purchase_receipt', resolve)
await self.purchase_create(stream_id)
resolve = await self.resolve('priced', include_purchase_receipt=True)
self.assertEqual('0.5', resolve['purchase_receipt']['amount'])
# my supports and my tips
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True
)
self.assertEqual('0.0', resolve['sent_supports'])
self.assertEqual('0.0', resolve['sent_tips'])
self.assertEqual('0.0', resolve['received_tips'])
await self.support_create(stream_id, '0.3')
await self.support_create(stream_id, '0.2')
await self.support_create(stream_id, '0.4', tip=True)
await self.support_create(stream_id, '0.5', tip=True)
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True
)
self.assertEqual('0.5', resolve['sent_supports'])
self.assertEqual('0.9', resolve['sent_tips'])
self.assertEqual('0.0', resolve['received_tips'])
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True,
wallet_id=wallet2.id
)
self.assertEqual('0.0', resolve['sent_supports'])
self.assertEqual('0.0', resolve['sent_tips'])
self.assertEqual('0.9', resolve['received_tips'])
self.assertEqual('1.4', resolve['meta']['support_amount'])
# make sure nothing is leaked between wallets through cached tx/txos
resolve = await self.resolve('priced')
self.assertNotIn('is_my_output', resolve)
self.assertNotIn('purchase_receipt', resolve)
self.assertNotIn('sent_supports', resolve)
self.assertNotIn('sent_tips', resolve)
self.assertNotIn('received_tips', resolve)
class ResolveAfterReorg(BaseResolveTestCase):
@@ -336,7 +270,7 @@ class ResolveAfterReorg(BaseResolveTestCase):
blocks = self.ledger.headers.height - start
self.blockchain.block_expected = start - 1
# go back to start
await self.blockchain.invalidate_block((await self.ledger.headers.hash(start)).decode())
await self.blockchain.invalidate_block(self.ledger.headers.hash(start).decode())
# go to previous + 1
await self.generate(blocks + 2)
@@ -5,7 +5,6 @@ from itertools import chain
from lbry.wallet.transaction import Transaction, Output, Input
from lbry.testcase import IntegrationTestCase
from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis
from lbry.wallet.manager import WalletManager
class BasicTransactionTests(IntegrationTestCase):
@@ -123,13 +122,13 @@ class BasicTransactionTests(IntegrationTestCase):
await self.blockchain.generate(1)
await self.ledger.wait(tx) # confirmed
tx = (await account1.get_transactions(include_is_my_input=True, include_is_my_output=True))[1]
tx = (await account1.get_transactions())[1]
self.assertEqual(satoshis_to_coins(tx.inputs[0].amount), '1.1')
self.assertEqual(satoshis_to_coins(tx.inputs[1].amount), '1.1')
self.assertEqual(satoshis_to_coins(tx.outputs[0].amount), '2.0')
self.assertEqual(tx.outputs[0].get_address(self.ledger), address2)
self.assertTrue(tx.outputs[0].is_internal_transfer)
self.assertTrue(tx.outputs[1].is_internal_transfer)
self.assertFalse(tx.outputs[0].is_change)
self.assertTrue(tx.outputs[1].is_change)
async def test_history_edge_cases(self):
await self.blockchain.generate(300)
@@ -174,130 +173,3 @@ class BasicTransactionTests(IntegrationTestCase):
self.assertTrue(await self.ledger.update_history(address, remote_status))
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
def wait_for_txid(self, txid, address):
return self.ledger.on_transaction.where(
lambda e: e.tx.id == txid and e.address == address
)
async def _test_transaction(self, send_amount, address, inputs, change):
tx = await Transaction.create(
[], [Output.pay_pubkey_hash(send_amount, self.ledger.address_to_hash160(address))], [self.account],
self.account
)
await self.ledger.broadcast(tx)
input_amounts = [txi.amount for txi in tx.inputs]
self.assertListEqual(inputs, input_amounts)
self.assertEqual(len(inputs), len(tx.inputs))
self.assertEqual(2, len(tx.outputs))
self.assertEqual(send_amount, tx.outputs[0].amount)
self.assertEqual(change, tx.outputs[1].amount)
return tx
async def assertSpendable(self, amounts):
spendable = await self.ledger.db.get_spendable_utxos(
self.ledger, 2000000000000, [self.account], set_reserved=False, return_insufficient_funds=True
)
got_amounts = [estimator.effective_amount for estimator in spendable]
self.assertListEqual(amounts, got_amounts)
async def test_sqlite_coin_chooser(self):
wallet_manager = WalletManager([self.wallet], {self.ledger.get_id(): self.ledger})
await self.blockchain.generate(300)
await self.assertBalance(self.account, '0.0')
address = await self.account.receiving.get_or_create_usable_address()
other_account = self.wallet.generate_account(self.ledger)
other_address = await other_account.receiving.get_or_create_usable_address()
self.ledger.coin_selection_strategy = 'sqlite'
await self.ledger.subscribe_account(self.account)
txids = []
txids.append(await self.blockchain.send_to_address(address, 1.0))
txids.append(await self.blockchain.send_to_address(address, 1.0))
txids.append(await self.blockchain.send_to_address(address, 3.0))
txids.append(await self.blockchain.send_to_address(address, 5.0))
txids.append(await self.blockchain.send_to_address(address, 10.0))
await asyncio.wait([self.wait_for_txid(txid, address) for txid in txids], timeout=1)
await self.assertBalance(self.account, '20.0')
await self.assertSpendable([99992600, 99992600, 299992600, 499992600, 999992600])
# send 1.5 lbc
first_tx = await Transaction.create(
[], [Output.pay_pubkey_hash(150000000, self.ledger.address_to_hash160(other_address))], [self.account],
self.account
)
self.assertEqual(2, len(first_tx.inputs))
self.assertEqual(2, len(first_tx.outputs))
self.assertEqual(100000000, first_tx.inputs[0].amount)
self.assertEqual(100000000, first_tx.inputs[1].amount)
self.assertEqual(150000000, first_tx.outputs[0].amount)
self.assertEqual(49980200, first_tx.outputs[1].amount)
await self.assertBalance(self.account, '18.0')
await self.assertSpendable([299992600, 499992600, 999992600])
await wallet_manager.broadcast_or_release(first_tx, blocking=True)
await self.assertSpendable([49972800, 299992600, 499992600, 999992600])
# 0.499, 3.0, 5.0, 10.0
await self.assertBalance(self.account, '18.499802')
# send 1.5lbc again
second_tx = await self._test_transaction(150000000, other_address, [49980200, 300000000], 199960400)
await self.assertSpendable([499992600, 999992600])
# replicate cancelling the api call after the tx broadcast while ledger.wait'ing it
e = asyncio.Event()
real_broadcast = self.ledger.broadcast
async def broadcast(tx):
try:
return await real_broadcast(tx)
finally:
e.set()
self.ledger.broadcast = broadcast
broadcast_task = asyncio.create_task(wallet_manager.broadcast_or_release(second_tx, blocking=True))
# wait for the broadcast to finish
await e.wait()
# cancel the api call
broadcast_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await broadcast_task
# test if sending another 1.5 lbc will try to double spend the inputs from the cancelled tx
tx1 = await self._test_transaction(150000000, other_address, [500000000], 349987600)
await self.ledger.wait(tx1, timeout=1)
# wait for the cancelled transaction too, so that it's in the database
# needed to keep everything deterministic
await self.ledger.wait(second_tx, timeout=1)
await self.assertSpendable([199953000, 349980200, 999992600])
# spend deep into the mempool and see what else breaks
tx2 = await self._test_transaction(150000000, other_address, [199960400], 49948000)
await self.assertSpendable([349980200, 999992600])
await self.ledger.wait(tx2, timeout=1)
await self.assertSpendable([49940600, 349980200, 999992600])
tx3 = await self._test_transaction(150000000, other_address, [49948000, 349987600], 249915800)
await self.assertSpendable([999992600])
await self.ledger.wait(tx3, timeout=1)
await self.assertSpendable([249908400, 999992600])
tx4 = await self._test_transaction(150000000, other_address, [249915800], 99903400)
await self.assertSpendable([999992600])
await self.ledger.wait(tx4, timeout=1)
await self.assertBalance(self.account, '10.999034')
await self.assertSpendable([99896000, 999992600])
# spend more
tx5 = await self._test_transaction(100000000, other_address, [99903400, 1000000000], 999883600)
await self.assertSpendable([])
await self.ledger.wait(tx5, timeout=1)
await self.assertSpendable([999876200])
await self.assertBalance(self.account, '9.998836')
@@ -19,29 +19,6 @@ class WalletCommands(CommandTestCase):
await self.daemon.jsonrpc_wallet_add(wallet.id)
self.assertEqual(len(session.hashX_subs), 28)
async def test_wallet_syncing_status(self):
address = await self.daemon.jsonrpc_address_unused()
self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing'])
await self.blockchain.send_to_address(address, 1)
await self.ledger._update_tasks.started.wait()
self.assertTrue(self.daemon.jsonrpc_wallet_status()['is_syncing'])
await self.ledger._update_tasks.done.wait()
self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing'])
wallet = self.daemon.component_manager.get_actual_component('wallet')
wallet_manager = wallet.wallet_manager
# when component manager hasn't started yet
wallet.wallet_manager = None
self.assertEqual(
{'is_encrypted': None, 'is_syncing': None, 'is_locked': None},
self.daemon.jsonrpc_wallet_status()
)
wallet.wallet_manager = wallet_manager
self.assertEqual(
{'is_encrypted': False, 'is_syncing': False, 'is_locked': False},
self.daemon.jsonrpc_wallet_status()
)
async def test_wallet_reconnect(self):
await self.conductor.spv_node.stop(True)
self.conductor.spv_node.port = 54320
@@ -105,11 +82,6 @@ class WalletCommands(CommandTestCase):
async def test_granular_balances(self):
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True)
account3 = wallet2.default_account
address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id)
await self.confirm_tx(await self.blockchain.send_to_address(address3, 1))
await self.generate(1)
account_balance = self.daemon.jsonrpc_account_balance
wallet_balance = self.daemon.jsonrpc_wallet_balance
@@ -156,7 +128,7 @@ class WalletCommands(CommandTestCase):
# tip received
support1 = await self.support_create(
self.get_claim_id(stream1), '0.3', tip=True, wallet_id=wallet2.id
self.get_claim_id(stream1), '0.3', tip=True, funding_account_ids=[account2.id]
)
self.assertEqual(await account_balance(), {
'total': '9.27741',
@@ -165,8 +137,8 @@ class WalletCommands(CommandTestCase):
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
self.assertEqual(await wallet_balance(), {
'total': '10.27741',
'available': '6.97741',
'total': '9.977268',
'available': '6.677268',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
@@ -181,8 +153,8 @@ class WalletCommands(CommandTestCase):
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(), {
'total': '10.277303',
'available': '7.277303',
'total': '9.977161',
'available': '6.977161',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
@@ -193,17 +165,17 @@ class WalletCommands(CommandTestCase):
# tip another claim
await self.support_create(
self.get_claim_id(stream2), '0.2', tip=True, wallet_id=wallet2.id
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id]
)
self.assertEqual(await account_balance(), {
'total': '9.277303',
'available': '6.277303',
'total': '9.077157',
'available': '6.077157',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(), {
'total': '10.439196',
'available': '7.139196',
'total': '9.938908',
'available': '6.638908',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'}
})
@@ -291,9 +263,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
wallet = daemon.wallet_manager.default_wallet
wallet.save()
self.assertEqual(daemon.jsonrpc_wallet_status(), {
'is_locked': False, 'is_encrypted': False, 'is_syncing': False
})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertIsNone(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK))
self.assertWalletEncrypted(wallet.storage.path, False)
@@ -305,26 +275,22 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
daemon.jsonrpc_wallet_decrypt() # already not encrypted
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
daemon.jsonrpc_wallet_lock()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True})
# can't sign transactions with locked wallet
with self.assertRaises(AssertionError):
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_unlock('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_decrypt()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': False})
self.assertWalletEncrypted(wallet.storage.path, False)
@@ -336,8 +302,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
self.assertTrue(daemon2.jsonrpc_wallet_encrypt('password'))
self.assertTrue(daemon2.jsonrpc_wallet_lock())
self.assertTrue(daemon2.jsonrpc_wallet_unlock("password"))
self.assertEqual(daemon2.jsonrpc_wallet_status(),
{'is_locked': False, 'is_encrypted': True, 'is_syncing': False})
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
async def test_sync_with_encryption_and_password_change(self):
daemon, daemon2 = self.daemon, self.daemon2
@@ -358,8 +323,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True)
# sync_apply with new password2 also sets it as new local password
self.assertEqual(wallet.encryption_password, 'password2')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': True})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
@@ -376,8 +340,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
await daemon2.jsonrpc_sync_apply('password3', data=data['data'], blocking=True)
# the other device got new password and on disk encryption
self.assertEqual(wallet2.encryption_password, 'password3')
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': True})
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon2.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet2.storage.path, True)
@@ -45,6 +45,14 @@ class TestSessions(IntegrationTestCase):
await self.ledger.network.broadcast('13370042004200')
class TestSegwitServer(IntegrationTestCase):
LEDGER = lbry.wallet
ENABLE_SEGWIT = True
async def test_at_least_it_starts(self):
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
@@ -73,7 +81,7 @@ class TestUsagePayment(CommandTestCase):
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
@@ -82,7 +90,7 @@ class TestUsagePayment(CommandTestCase):
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
@@ -71,11 +71,6 @@ class FileCommands(CommandTestCase):
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
@@ -180,14 +175,14 @@ class FileCommands(CommandTestCase):
file_list = await self.file_list()
self.assertEqual(
file_list[0]['timestamp'],
self.ledger.headers.estimated_timestamp(file_list[0]['height'])
None
)
self.assertEqual(file_list[0]['confirmations'], -1)
await self.daemon.jsonrpc_resolve('foo')
file_list = await self.file_list()
self.assertEqual(
file_list[0]['timestamp'],
self.ledger.headers.estimated_timestamp(file_list[0]['height'])
self.ledger.headers[file_list[0]['height']]['timestamp']
)
self.assertEqual(file_list[0]['confirmations'], 1)
-7
View File
@@ -4,7 +4,6 @@ from lbry.testcase import CommandTestCase
class EpicAdventuresOfChris45(CommandTestCase):
async def test_no_this_is_not_a_test_its_an_adventure(self):
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
# and everything in between; it's also the setting of some record setting
# integration tests.
@@ -195,9 +194,3 @@ class EpicAdventuresOfChris45(CommandTestCase):
}},
await self.resolve(uri)
)
# He closes and opens the wallet server databases to see how horribly they break
db = self.conductor.spv_node.server.db
db.close()
await db.open_for_serving()
# They didn't! (error would be AssertionError: 276 vs 266 (264 counts) on startup)
@@ -106,16 +106,11 @@ class MockedCommentServer:
return False
def hide_comments(self, pieces: list):
hidden = []
comments_hidden = []
for p in pieces:
if self.hide_comment(**p):
hidden.append(p['comment_id'])
comment_ids = {c['comment_id'] for c in pieces}
return {
'hidden': hidden,
'visible': list(comment_ids - set(hidden))
}
comments_hidden.append(p['comment_id'])
return {'hidden': comments_hidden}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
@@ -143,19 +138,12 @@ class MockedCommentServer:
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[self.get_comment_id(comment_id)]
return {
'channel_id': comment['channel_id'],
'channel_name': comment['channel_name'],
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
}
def get_comments_by_id(self, comment_ids: list):
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
return {
'page': 1,
'page_size': len(comment_ids),
'total_pages': 1,
'items': comments,
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
methods = {
'get_claim_comments': get_claim_comments,
+6 -17
View File
@@ -1,3 +1,4 @@
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.testcase import CommandTestCase
@@ -28,24 +29,12 @@ class SettingsManagement(CommandTestCase):
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
# tests that changing share_usage_data propagates to the relevant properties
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
loggly = get_loggly_handler(self.daemon.conf)
self.addCleanup(loggly.close)
self.assertFalse(self.daemon.analytics_manager.enabled)
self.assertFalse(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', True)
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
self.assertTrue(self.daemon.analytics_manager.enabled)
self.assertTrue(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', False)
class TroubleshootingCommands(CommandTestCase):
async def test_tracemalloc_commands(self):
self.addCleanup(self.daemon.jsonrpc_tracemalloc_disable)
self.assertFalse(self.daemon.jsonrpc_tracemalloc_disable())
self.assertTrue(self.daemon.jsonrpc_tracemalloc_enable())
class WeirdObject():
pass
hold_em = [WeirdObject() for _ in range(500)]
top = self.daemon.jsonrpc_tracemalloc_top(1)
self.assertEqual(1, len(top))
self.assertEqual('hold_em = [WeirdObject() for _ in range(500)]', top[0]['code'])
self.assertTrue(top[0]['line'].startswith('other/test_other_commands.py:'))
self.assertGreaterEqual(top[0]['count'], 500)
self.assertGreater(top[0]['size'], 0) # just matters that its a positive integer
+14 -43
View File
@@ -32,14 +32,13 @@ class TranscodeValidation(ClaimTestCase):
self.conf = TranscodeConfig()
self.conf.volume_analysis_time = 0 # disable it as the test file isn't very good here
self.analyzer = VideoFileAnalyzer(self.conf)
self.assertTrue((await self.analyzer.status())["available"]) # ensure ffmpeg path detected
file_ogg = self.make_name("ogg", ".ogg")
self.video_file_ogg = str(file_ogg)
if not file_ogg.exists():
command = f'-i "{self.video_file_name}" -c:v libtheora -q:v 4 -c:a libvorbis -q:a 4 ' \
f'-c:s copy -c:d copy "{file_ogg}"'
with MeasureTime(f"Creating {file_ogg.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
file_webm = self.make_name("webm", ".webm")
@@ -48,19 +47,16 @@ class TranscodeValidation(ClaimTestCase):
command = f'-i "{self.video_file_name}" -c:v libvpx-vp9 -crf 36 -b:v 0 -cpu-used 2 ' \
f'-c:a libopus -b:a 128k -c:s copy -c:d copy "{file_webm}"'
with MeasureTime(f"Creating {file_webm.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
async def test_should_work(self):
new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_name)
self.assertEqual(self.video_file_name, new_file_name)
new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg)
self.assertEqual(self.video_file_ogg, new_file_name)
new_file_name, spec = await self.analyzer.verify_or_repair(True, False, self.video_file_webm)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_webm)
self.assertEqual(self.video_file_webm, new_file_name)
self.assertEqual(spec["width"], 1280)
self.assertEqual(spec["height"], 720)
self.assertEqual(spec["duration"], 16)
async def test_volume(self):
self.conf.volume_analysis_time = 200
@@ -72,13 +68,13 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Container format is not in the approved list"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_video_codec(self):
@@ -86,7 +82,7 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx265 -preset superfast "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Video codec is not in the approved list"):
@@ -94,7 +90,7 @@ class TranscodeValidation(ClaimTestCase):
with self.assertRaisesRegex(Exception, "faststart flag was not used"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_max_bit_rate(self):
@@ -108,13 +104,13 @@ class TranscodeValidation(ClaimTestCase):
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx264 ' \
f'-vf format=yuv444p "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "pixel format does not match the approved"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_audio_codec(self):
@@ -122,13 +118,13 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:a pcm_s16le "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute_ffmpeg(command)
output, code = await self.analyzer._execute("ffmpeg", command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Audio codec is not in the approved list"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_extension_choice(self):
@@ -155,31 +151,6 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_path = "I don't really exist/"
self.analyzer._env_copy.pop("PATH", None)
await self.analyzer.status(reset=True)
self.conf.ffmpeg_folder = "I don't really exist/"
with self.assertRaisesRegex(Exception, "Unable to locate"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
async def test_dont_recheck_ffmpeg_installation(self):
call_count = 0
original = self.daemon._video_file_analyzer._verify_ffmpeg_installed
def _verify_ffmpeg_installed():
nonlocal call_count
call_count += 1
return original()
self.daemon._video_file_analyzer._verify_ffmpeg_installed = _verify_ffmpeg_installed
self.assertEqual(0, call_count)
await self.daemon.jsonrpc_status()
self.assertEqual(1, call_count)
# counter should not go up again
await self.daemon.jsonrpc_status()
self.assertEqual(1, call_count)
# this should force rechecking the installation
await self.daemon.jsonrpc_ffmpeg_find()
self.assertEqual(2, call_count)
+1 -2
View File
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
@@ -24,7 +24,6 @@ class TestBlobManager(AsyncioTestCase):
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
+6 -12
View File
@@ -28,20 +28,19 @@ def mock_config():
class BlobExchangeTestBase(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.client_wallet_dir = tempfile.mkdtemp()
self.client_dir = tempfile.mkdtemp()
self.server_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.client_wallet_dir)
self.addCleanup(shutil.rmtree, self.client_dir)
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
fixed_peers=[])
reflector_servers=[])
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir,
wallet=self.client_wallet_dir, fixed_peers=[])
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
reflector_servers=[])
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
@@ -131,14 +130,10 @@ class TestBlobExchange(BlobExchangeTestBase):
write_blob = blob._write_blob
write_called_count = 0
async def _wrap_write_blob(blob_bytes):
def wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
await write_blob(blob_bytes)
def wrap_write_blob(blob_bytes):
return asyncio.create_task(_wrap_write_blob(blob_bytes))
write_blob(blob_bytes)
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
@@ -171,7 +166,6 @@ class TestBlobExchange(BlobExchangeTestBase):
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
+10 -10
View File
@@ -18,31 +18,31 @@ class TestSigningComments(AsyncioTestCase):
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
}
async def test01_successful_create_sign_and_validate_comment(self):
channel = await get_channel('@BusterBluth')
def test01_successful_create_sign_and_validate_comment(self):
channel = get_channel('@BusterBluth')
stream = get_stream('pop secret')
comment = self.create_claim_comment_body('Cool stream', stream, channel)
sign_comment(comment, channel)
self.assertTrue(is_comment_signed_by_channel(comment, channel))
async def test02_fail_to_validate_spoofed_channel(self):
pdiddy = await get_channel('@PDitty')
channel2 = await get_channel('@TomHaverford')
def test02_fail_to_validate_spoofed_channel(self):
pdiddy = get_channel('@PDitty')
channel2 = get_channel('@TomHaverford')
stream = get_stream()
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
async def test03_successful_sign_abandon_comment(self):
rswanson = await get_channel('@RonSwanson')
def test03_successful_sign_abandon_comment(self):
rswanson = get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
jeanralphio = await get_channel('@JeanRalphio')
def test04_invalid_signature(self):
rswanson = get_channel('@RonSwanson')
jeanralphio = get_channel('@JeanRalphio')
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
chair_comment = self.create_claim_comment_body(
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
@@ -120,8 +120,6 @@ class FakeComponent:
class FakeDelayedWallet(FakeComponent):
component_name = "wallet"
depends_on = []
ledger = None
default_wallet = None
async def stop(self):
await asyncio.sleep(1)
+1 -26
View File
@@ -1,7 +1,7 @@
import asyncio
import unittest
from lbry.utils import generate_id
from lbry.dht.peer import PeerManager, make_kademlia_peer, is_valid_public_ipv4
from lbry.dht.peer import PeerManager, make_kademlia_peer
from lbry.testcase import AsyncioTestCase
@@ -50,31 +50,6 @@ class PeerTest(AsyncioTestCase):
for i in range(32):
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], f"{224 + i}.0.0.0", 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '255.255.255.255', 1000)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], 'beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', 1000
)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], '2001:db8::ff00:42:8329', 1000
)
def test_is_valid_ipv4(self):
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef'))
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', True))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329'))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329', True))
self.assertFalse(is_valid_public_ipv4('127.0.0.1'))
self.assertTrue(is_valid_public_ipv4('127.0.0.1', True))
self.assertFalse(is_valid_public_ipv4('172.16.0.1'))
self.assertFalse(is_valid_public_ipv4('172.16.0.1', True))
self.assertTrue(is_valid_public_ipv4('1.2.3.4'))
self.assertTrue(is_valid_public_ipv4('1.2.3.4', True))
self.assertFalse(is_valid_public_ipv4('derp'))
self.assertFalse(is_valid_public_ipv4('derp', True))
def test_boolean(self):
self.assertNotEqual(self.first_contact, self.second_contact)
+1 -1
View File
@@ -92,7 +92,7 @@ class TestCostEst(unittest.TestCase):
@unittest.SkipTest
class TestJsonRpc(unittest.TestCase):
def setUp(self):
async def noop():
def noop():
return None
test_utils.reset_time(self)
@@ -1,53 +0,0 @@
import unittest
from aiohttp.test_utils import make_mocked_request as request
from aiohttp.web import HTTPForbidden
from lbry.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.extras.daemon.security import is_request_allowed as allowed, ensure_request_allowed as ensure
class TestAllowedOrigin(unittest.TestCase):
def test_allowed_origin_default(self):
conf = Config()
# lack of Origin is always allowed
self.assertTrue(allowed(request('GET', '/'), conf))
# deny all other Origins
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_star(self):
conf = Config(allowed_origin='*')
# everything is allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_specified(self):
conf = Config(allowed_origin='localhost')
# no origin and only localhost are allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_ensure_default(self):
conf = Config()
ensure(request('GET', '/'), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
self.assertIn("'localhost' are not allowed", log.output[0])
def test_ensure_specific(self):
conf = Config(allowed_origin='localhost')
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)
self.assertIn("'hackers.com' are not allowed", log.output[0])
self.assertIn("'allowed_origin' limits requests to: 'localhost'", log.output[0])
+1 -115
View File
@@ -10,7 +10,7 @@ from lbry.stream.stream_manager import StreamManager
from lbry.stream.reflector.server import ReflectorServer
class TestReflector(AsyncioTestCase):
class TestStreamAssembler(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
@@ -22,7 +22,6 @@ class TestReflector(AsyncioTestCase):
self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf)
self.addCleanup(self.blob_manager.stop)
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
server_tmp_dir = tempfile.mkdtemp()
@@ -31,7 +30,6 @@ class TestReflector(AsyncioTestCase):
self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf)
self.addCleanup(self.server_blob_manager.stop)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
@@ -48,15 +46,12 @@ class TestReflector(AsyncioTestCase):
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(100, self.stream.reflector_progress)
self.assertSetEqual(
set(sent),
set(map(lambda b: b.blob_hash,
self.stream.descriptor.blobs[:-1] + [self.blob_manager.get_blob(self.stream.sd_hash)]))
)
self.assertTrue(self.stream.is_fully_reflected)
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
self.assertTrue(server_sd_blob.get_is_verified())
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
@@ -78,112 +73,3 @@ class TestReflector(AsyncioTestCase):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
async def test_result_from_disconnect_mid_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
stop.set()
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_after_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
async def test_result_from_disconnect_mid_data_transfer(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
)
self.assertFalse(self.stream.is_fully_reflected)
async def test_delete_file_during_reflector_upload(self):
stop = asyncio.Event()
incoming = asyncio.Event()
not_incoming = asyncio.Event()
reflector = ReflectorServer(
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
not_incoming_event=not_incoming
)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
await incoming.wait()
await not_incoming.wait()
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
)
self.assertFalse(self.stream.is_fully_reflected)
+6 -10
View File
@@ -65,7 +65,7 @@ def get_claim_transaction(claim_name, claim=b''):
)
async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
claim = Claim()
if fee:
if fee['currency'] == 'LBC':
@@ -86,9 +86,6 @@ async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
})
class FakeHeaders:
def estimated_timestamp(self, height):
return 1984
def __init__(self, height):
self.height = height
@@ -97,7 +94,7 @@ async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
wallet = Wallet()
ledger = Ledger({
'db': Database(os.path.join(wallet_dir, 'blockchain.db')),
'db': Database(':memory:'),
'headers': FakeHeaders(514082)
})
await ledger.db.open()
@@ -110,7 +107,7 @@ async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
network=manager.ledger.network, server=('fakespv.lbry.com', 50001)
)
async def mock_resolve(*args, **kwargs):
async def mock_resolve(*args):
result = {txo.meta['permanent_url']: txo}
await storage.save_claim_from_output(ledger, txo)
return result
@@ -136,8 +133,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.loop, self.server_blob_manager.blob_dir, file_path, old_sort=old_sort
)
self.sd_hash = descriptor.sd_hash
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, self.client_wallet_dir,
balance, fee)
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
analytics_manager = AnalyticsManager(
self.client_config,
binascii.hexlify(generate_id()).decode(),
@@ -188,7 +184,7 @@ class TestStreamManager(BlobExchangeTestBase):
await self._test_time_to_first_bytes(check_post)
async def test_fixed_peer_delay_dht_peers_found(self):
self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)]
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
server_from_client = None
self.server_from_client, server_from_client = server_from_client, self.server_from_client
@@ -232,7 +228,7 @@ class TestStreamManager(BlobExchangeTestBase):
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup)
async def test_override_fixed_peer_delay_dht_disabled(self):
self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)]
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
self.client_config.components_to_skip = ['dht', 'hash_announcer']
self.client_config.fixed_peer_delay = 9001.0
self.server_from_client = None
+25 -4
View File
@@ -15,6 +15,7 @@ from lbry.testcase import AsyncioTestCase
from lbry.extras.cli import normalize_value, main, setup_logging
from lbry.extras.system_info import get_platform
from lbry.extras.daemon.daemon import Daemon
from lbry.extras.daemon.loggly_handler import HTTPSLogglyHandler
from lbry.conf import Config
from lbry.extras import cli
@@ -62,8 +63,10 @@ class CLILoggingTest(AsyncioTestCase):
log = log.getChild("lbry")
self.assertTrue(log.isEnabledFor(logging.INFO))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 1)
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 2)
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
self.assertFalse(log.handlers[1].enabled)
async with get_logger(["start", "--verbose"]) as log:
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
@@ -77,14 +80,32 @@ class CLILoggingTest(AsyncioTestCase):
self.assertFalse(log.getChild("lbry").isEnabledFor(logging.DEBUG))
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
async def test_loggly(self):
async with get_logger(["start"]) as log: # default share_usage_data=False
log = log.getChild("lbry")
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=True) as log:
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertTrue(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async def test_quiet(self):
async with get_logger(["start"]) as log: # default is loud
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 2)
self.assertEqual(len(log.handlers), 3)
self.assertIs(type(log.handlers[1]), logging.StreamHandler)
async with get_logger(["start", "--quiet"]) as log:
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 1)
self.assertEqual(len(log.handlers), 2)
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
@@ -197,7 +218,7 @@ class DaemonDocsTests(TestCase):
try:
docopt.docopt(fn.__doc__, ())
except docopt.DocoptLanguageError as err:
failures.append(f"invalid docstring for {name}, {err.args[0]}")
failures.append(f"invalid docstring for {name}, {err.message}")
except docopt.DocoptExit:
pass
if failures:
-5
View File
@@ -90,15 +90,10 @@ class ConfigurationTests(unittest.TestCase):
def test_environment(self):
c = TestConfig()
self.assertEqual(c.test_str, 'the default')
c.set_environment({'LBRY_TEST_STR': 'from environ'})
self.assertEqual(c.test_str, 'from environ')
self.assertEqual(c.test_int, 9)
c.set_environment({'LBRY_TEST_INT': '1'})
self.assertEqual(c.test_int, 1)
def test_persisted(self):
with tempfile.TemporaryDirectory() as temp_dir:
+22 -24
View File
@@ -211,7 +211,6 @@ class TestQueries(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
await self.ledger.headers.open()
self.wallet = Wallet()
await self.ledger.db.open()
@@ -267,10 +266,9 @@ class TestQueries(AsyncioTestCase):
# SQLite is usually compiled with 999 variables limit: https://www.sqlite.org/limits.html
# This can be removed when there is a better way. See: https://github.com/lbryio/lbry-sdk/issues/2281
fetchall = self.ledger.db.db.execute_fetchall
def check_parameters_length(sql, parameters, read_only=False):
def check_parameters_length(sql, parameters):
self.assertLess(len(parameters or []), 999)
return fetchall(sql, parameters, read_only)
return fetchall(sql, parameters)
self.ledger.db.db.execute_fetchall = check_parameters_length
account = await self.create_account()
@@ -354,37 +352,37 @@ class TestQueries(AsyncioTestCase):
self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertListEqual([3, 2, 1], [tx.height for tx in txs])
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts, include_is_my_output=True)
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_input, True)
self.assertEqual(txs[0].outputs[0].is_my_output, False)
self.assertEqual(txs[1].inputs[0].is_my_input, False)
self.assertEqual(txs[1].outputs[0].is_my_output, True)
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2], include_is_my_output=True)
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_input, True)
self.assertEqual(txs[0].outputs[0].is_my_output, False)
self.assertEqual(txs[1].inputs[0].is_my_input, False)
self.assertEqual(txs[1].outputs[0].is_my_output, True)
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2]))
tx = await self.ledger.db.get_transaction(txid=tx2.id)
self.assertEqual(tx.id, tx2.id)
self.assertIsNone(tx.inputs[0].is_my_input)
self.assertIsNone(tx.outputs[0].is_my_output)
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id, include_is_my_output=True)
self.assertTrue(tx.inputs[0].is_my_input)
self.assertFalse(tx.outputs[0].is_my_output)
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id, include_is_my_output=True)
self.assertFalse(tx.inputs[0].is_my_input)
self.assertTrue(tx.outputs[0].is_my_output)
self.assertFalse(tx.inputs[0].is_my_account)
self.assertFalse(tx.outputs[0].is_my_account)
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
self.assertTrue(tx.inputs[0].is_my_account)
self.assertFalse(tx.outputs[0].is_my_account)
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
self.assertFalse(tx.inputs[0].is_my_account)
self.assertTrue(tx.outputs[0].is_my_account)
# height 0 sorted to the top with the rest in descending order
tx4 = await self.create_tx_from_nothing(account1, 0)
txos = await self.ledger.db.get_txos()
self.assertListEqual([0, 3, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
+27 -70
View File
@@ -1,15 +1,12 @@
import os
import asyncio
import tempfile
from binascii import unhexlify
from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha256
from lbry.wallet.util import ArithUint256
from lbry.testcase import AsyncioTestCase
from lbry.wallet.ledger import Headers as _Headers
class Headers(_Headers):
checkpoints = {}
from lbry.wallet.ledger import Headers
def block_bytes(blocks):
@@ -18,12 +15,11 @@ def block_bytes(blocks):
class TestHeaders(AsyncioTestCase):
async def test_deserialize(self):
def test_deserialize(self):
self.maxDiff = None
h = Headers(':memory:')
await h.open()
await h.connect(0, HEADERS)
self.assertEqual(await h.get(0), {
h.io.write(HEADERS)
self.assertEqual(h[0], {
'bits': 520159231,
'block_height': 0,
'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001',
@@ -33,7 +29,7 @@ class TestHeaders(AsyncioTestCase):
'timestamp': 1446058291,
'version': 1
})
self.assertEqual(await h.get(10), {
self.assertEqual(h[10], {
'bits': 509349720,
'block_height': 10,
'merkle_root': b'f4d8fded6a181d4a8a2817a0eb423cc0f414af29490004a620e66c35c498a554',
@@ -46,18 +42,13 @@ class TestHeaders(AsyncioTestCase):
async def test_connect_from_genesis(self):
headers = Headers(':memory:')
await headers.open()
self.assertEqual(headers.height, -1)
await headers.connect(0, HEADERS)
self.assertEqual(headers.height, 19)
async def test_connect_from_middle(self):
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS[:block_bytes(10)])
h = Headers(headers_temporary_file)
await h.open()
h = Headers(':memory:')
h.io.write(HEADERS[:block_bytes(10)])
self.assertEqual(h.height, 9)
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
self.assertEqual(h.height, 19)
@@ -118,19 +109,17 @@ class TestHeaders(AsyncioTestCase):
async def test_bounds(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS)
self.assertEqual(19, headers.height)
with self.assertRaises(IndexError):
_ = await headers.get(3001)
_ = headers[3001]
with self.assertRaises(IndexError):
_ = await headers.get(-1)
self.assertIsNotNone(await headers.get(19))
self.assertIsNotNone(await headers.get(0))
_ = headers[-1]
self.assertIsNotNone(headers[19])
self.assertIsNotNone(headers[0])
async def test_repair(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS[:block_bytes(11)])
self.assertEqual(10, headers.height)
await headers.repair()
@@ -148,52 +137,21 @@ class TestHeaders(AsyncioTestCase):
self.assertEqual(7, headers.height)
await headers.connect(len(headers), HEADERS[block_bytes(8):])
self.assertEqual(19, headers.height)
# verify from middle
await headers.repair(start_height=10)
self.assertEqual(19, headers.height)
async def test_do_not_estimate_unconfirmed(self):
async def test_checkpointed_writer(self):
headers = Headers(':memory:')
await headers.open()
self.assertIsNone(headers.estimated_timestamp(-1))
self.assertIsNone(headers.estimated_timestamp(0))
self.assertIsNotNone(headers.estimated_timestamp(1))
async def test_dont_estimate_whats_there(self):
getblocks = lambda start, end: HEADERS[block_bytes(start):block_bytes(end)]
headers.checkpoint = 10, hexlify(sha256(getblocks(10, 11)))
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(0, 10))
self.assertEqual(len(headers), 10)
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(10, 19))
self.assertEqual(len(headers), 19)
headers = Headers(':memory:')
await headers.open()
estimated = headers.estimated_timestamp(10)
await headers.connect(0, HEADERS)
real_time = (await headers.get(10))['timestamp']
after_downloading_header_estimated = headers.estimated_timestamp(10)
self.assertNotEqual(estimated, after_downloading_header_estimated)
self.assertEqual(after_downloading_header_estimated, real_time)
async def test_misalignment_triggers_repair_on_open(self):
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS)
headers = Headers(headers_temporary_file)
with self.assertLogs(level='WARN') as cm:
await headers.open()
await headers.close()
self.assertEqual(cm.output, [])
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.seek(0)
headers_file.truncate()
headers_file.write(HEADERS[:block_bytes(10)])
headers_file.write(b'ops')
headers_file.write(HEADERS[block_bytes(10):])
await headers.open()
self.assertEqual(
cm.output, [
'WARNING:lbry.wallet.header:Reader file size doesnt match header size. '
'Repairing, might take a while.',
'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating '
'it.'
]
)
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(0, 19))
self.assertEqual(len(headers), 19)
async def test_concurrency(self):
BLOCKS = 19
@@ -206,13 +164,12 @@ class TestHeaders(AsyncioTestCase):
await headers.connect(block_index, HEADERS[block_bytes(block_index):block_bytes(block_index + 1)])
async def reader():
for block_index in range(BLOCKS):
while len(headers) <= block_index:
while len(headers) < block_index:
await asyncio.sleep(0.000001)
assert (await headers.get(block_index))['block_height'] == block_index
assert headers[block_index]['block_height'] == block_index
reader_task = asyncio.create_task(reader())
await writer()
await reader_task
await headers.close()
HEADERS = unhexlify(
+18 -42
View File
@@ -33,19 +33,6 @@ class MockNetwork:
self.get_transaction_called.append(tx_hash)
return self.transaction[tx_hash]
async def get_transaction_and_merkle(self, tx_hash, known_height=None):
tx = await self.get_transaction(tx_hash)
merkle = {'block_height': -1}
if known_height:
merkle = await self.get_merkle(tx_hash, known_height)
return tx, merkle
async def get_transaction_batch(self, txids):
return {
txid: await self.get_transaction_and_merkle(txid)
for txid in txids
}
class LedgerTestCase(AsyncioTestCase):
@@ -54,8 +41,6 @@ class LedgerTestCase(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
self.ledger.headers.checkpoints = {}
await self.ledger.headers.open()
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
await self.ledger.db.open()
@@ -82,17 +67,12 @@ class LedgerTestCase(AsyncioTestCase):
serialized = self.make_header(**kwargs)
self.ledger.headers.io.seek(0, os.SEEK_END)
self.ledger.headers.io.write(serialized)
self.ledger.headers._size = self.ledger.headers.io.seek(0, os.SEEK_END) // self.ledger.headers.header_size
self.ledger.headers._size = None
class TestSynchronization(LedgerTestCase):
async def test_update_history(self):
txid1 = '252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792'
txid2 = 'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9'
txid3 = 'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0'
txid4 = '047cf1d53ef68f0fd586d46f90c09ff8e57a4180f67e7f4b8dd0135c3741e828'
account = Account.generate(self.ledger, Wallet(), "torba")
address = await account.receiving.get_or_create_usable_address()
address_details = await self.ledger.db.get_address(address=address)
@@ -103,50 +83,46 @@ class TestSynchronization(LedgerTestCase):
self.add_header(block_height=2, merkle_root=b'abcd04')
self.add_header(block_height=3, merkle_root=b'abcd04')
self.ledger.network = MockNetwork([
{'tx_hash': txid1, 'height': 0},
{'tx_hash': txid2, 'height': 1},
{'tx_hash': txid3, 'height': 2},
{'tx_hash': 'abcd01', 'height': 0},
{'tx_hash': 'abcd02', 'height': 1},
{'tx_hash': 'abcd03', 'height': 2},
], {
txid1: hexlify(get_transaction(get_output(1)).raw),
txid2: hexlify(get_transaction(get_output(2)).raw),
txid3: hexlify(get_transaction(get_output(3)).raw),
'abcd01': hexlify(get_transaction(get_output(1)).raw),
'abcd02': hexlify(get_transaction(get_output(2)).raw),
'abcd03': hexlify(get_transaction(get_output(3)).raw),
})
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid1, txid2, txid3])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
address_details['history'],
f'{txid1}:0:'
f'{txid2}:1:'
f'{txid3}:2:'
'252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792:0:'
'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9:1:'
'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0:2:'
)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])
self.ledger.network.history.append({'tx_hash': txid4, 'height': 3})
self.ledger.network.transaction[txid4] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 3})
self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid4])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd04'])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
address_details['history'],
f'{txid1}:0:'
f'{txid2}:1:'
f'{txid3}:2:'
f'{txid4}:3:'
'252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792:0:'
'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9:1:'
'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0:2:'
'047cf1d53ef68f0fd586d46f90c09ff8e57a4180f67e7f4b8dd0135c3741e828:3:'
)
+14 -14
View File
@@ -21,9 +21,9 @@ def get_tx():
return Transaction().add_inputs([get_input()])
async def get_channel(claim_name='@foo'):
def get_channel(claim_name='@foo'):
channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc')
await channel_txo.generate_channel_private_key()
channel_txo.generate_channel_private_key()
get_tx().add_outputs([channel_txo])
return channel_txo
@@ -36,32 +36,32 @@ def get_stream(claim_name='foo'):
class TestSigningAndValidatingClaim(AsyncioTestCase):
async def test_successful_create_sign_and_validate(self):
channel = await get_channel()
def test_successful_create_sign_and_validate(self):
channel = get_channel()
stream = get_stream()
stream.sign(channel)
self.assertTrue(stream.is_signed_by(channel))
async def test_fail_to_validate_on_wrong_channel(self):
def test_fail_to_validate_on_wrong_channel(self):
stream = get_stream()
stream.sign(await get_channel())
self.assertFalse(stream.is_signed_by(await get_channel()))
stream.sign(get_channel())
self.assertFalse(stream.is_signed_by(get_channel()))
async def test_fail_to_validate_altered_claim(self):
channel = await get_channel()
def test_fail_to_validate_altered_claim(self):
channel = get_channel()
stream = get_stream()
stream.sign(channel)
self.assertTrue(stream.is_signed_by(channel))
stream.claim.stream.title = 'hello'
self.assertFalse(stream.is_signed_by(channel))
async def test_valid_private_key_for_cert(self):
channel = await get_channel()
def test_valid_private_key_for_cert(self):
channel = get_channel()
self.assertTrue(channel.is_channel_private_key(channel.private_key))
async def test_fail_to_load_wrong_private_key_for_cert(self):
channel = await get_channel()
self.assertFalse(channel.is_channel_private_key((await get_channel()).private_key))
def test_fail_to_load_wrong_private_key_for_cert(self):
channel = get_channel()
self.assertFalse(channel.is_channel_private_key(get_channel().private_key))
class TestValidatingOldSignatures(AsyncioTestCase):
+14 -86
View File
@@ -1,7 +1,4 @@
import os
import unittest
import tempfile
import shutil
from binascii import hexlify, unhexlify
from itertools import cycle
@@ -85,14 +82,14 @@ class TestSizeAndFeeEstimation(AsyncioTestCase):
class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
def test_is_my_output_not_set(self):
def test_is_my_account_not_set(self):
tx = get_transaction()
with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"):
_ = tx.net_account_balance
tx.inputs[0].txo_ref.txo.is_my_output = True
tx.inputs[0].txo_ref.txo.is_my_account = True
with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"):
_ = tx.net_account_balance
tx.outputs[0].is_my_output = True
tx.outputs[0].is_my_account = True
# all inputs/outputs are set now so it should work
_ = tx.net_account_balance
@@ -101,9 +98,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_output = True
tx.outputs[0].is_my_output = False
tx.outputs[1].is_my_output = True
tx.inputs[0].txo_ref.txo.is_my_account = True
tx.outputs[0].is_my_account = False
tx.outputs[1].is_my_account = True
self.assertEqual(tx.net_account_balance, -200*CENT)
def test_paying_from_other_account_to_my_account(self):
@@ -111,9 +108,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_output = False
tx.outputs[0].is_my_output = True
tx.outputs[1].is_my_output = False
tx.inputs[0].txo_ref.txo.is_my_account = False
tx.outputs[0].is_my_account = True
tx.outputs[1].is_my_account = False
self.assertEqual(tx.net_account_balance, 190*CENT)
def test_paying_from_my_account_to_my_account(self):
@@ -121,9 +118,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_output = True
tx.outputs[0].is_my_output = True
tx.outputs[1].is_my_output = True
tx.inputs[0].txo_ref.txo.is_my_account = True
tx.outputs[0].is_my_account = True
tx.outputs[1].is_my_account = True
self.assertEqual(tx.net_account_balance, -10*CENT) # lost to fee
@@ -305,11 +302,9 @@ class TestTransactionSigning(AsyncioTestCase):
class TransactionIOBalancing(AsyncioTestCase):
async def asyncSetUp(self):
wallet_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, wallet_dir)
self.ledger = Ledger({
'db': Database(os.path.join(wallet_dir, 'blockchain.db')),
'headers': Headers(':memory:'),
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
await self.ledger.db.open()
self.account = Account.from_dict(
@@ -424,70 +419,3 @@ class TransactionIOBalancing(AsyncioTestCase):
self.assertListEqual([0.01, 1], self.inputs(tx))
# change is now needed to consume extra input
self.assertListEqual([0.97], self.outputs(tx))
async def test_basic_use_cases_sqlite(self):
self.ledger.coin_selection_strategy = 'sqlite'
self.ledger.fee_per_byte = int(0.01*CENT)
# available UTXOs for filling missing inputs
utxos = await self.create_utxos([
1, 1, 3, 5, 10
])
self.assertEqual(5, len(await self.ledger.get_utxos()))
# pay 3 coins (3.07 w/ fees)
tx = await self.tx(
[], # inputs
[self.txo(3)] # outputs
)
await self.ledger.db.db.run(self.ledger.db._transaction_io, tx, tx.outputs[0].get_address(self.ledger), tx.id)
self.assertListEqual(self.inputs(tx), [1.0, 1.0, 3.0])
# a change of 1.95 is added to reach balance
self.assertListEqual(self.outputs(tx), [3, 1.95])
# utxos: 1.95, 3, 5, 10
self.assertEqual(2, len(await self.ledger.get_utxos()))
# pay 4.946 coins (5.00 w/ fees)
tx = await self.tx(
[], # inputs
[self.txo(4.946)] # outputs
)
self.assertEqual(1, len(await self.ledger.get_utxos()))
self.assertListEqual(self.inputs(tx), [5.0])
self.assertEqual(2, len(tx.outputs))
self.assertEqual(494600000, tx.outputs[0].amount)
# utxos: 3, 1.95, 4.946, 10
await self.ledger.release_outputs(utxos)
# supplied input and output, but input is not enough to cover output
tx = await self.tx(
[self.txi(self.txo(10))], # inputs
[self.txo(11)] # outputs
)
# additional input is chosen (UTXO 1)
self.assertListEqual([10, 1.0, 1.0], self.inputs(tx))
# change is now needed to consume extra input
self.assertListEqual([11, 0.95], self.outputs(tx))
await self.ledger.release_outputs(utxos)
# liquidating a UTXO
tx = await self.tx(
[self.txi(self.txo(10))], # inputs
[] # outputs
)
self.assertListEqual([10], self.inputs(tx))
# missing change added to consume the amount
self.assertListEqual([9.98], self.outputs(tx))
await self.ledger.release_outputs(utxos)
# liquidating at a loss, requires adding extra inputs
tx = await self.tx(
[self.txi(self.txo(0.01))], # inputs
[] # outputs
)
# UTXO 1 is added to cover some of the fee
self.assertListEqual([0.01, 1], self.inputs(tx))
# change is now needed to consume extra input
self.assertListEqual([0.97], self.outputs(tx))