Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac06b782a8 | ||
|
|
6495694ac8 | ||
|
|
115034fccb | ||
|
|
309e957a85 | ||
|
|
d7007e402e | ||
|
|
91323a21cf | ||
|
|
fea893d76c | ||
|
|
761bc6ba4c | ||
|
|
75172feb4e | ||
|
|
3285fb1608 | ||
|
|
03a4c6910d | ||
|
|
485b958599 | ||
|
|
da47ba2f67 | ||
|
|
c39195488a | ||
|
|
227fb0ae9b | ||
|
|
b12ff5b503 | ||
|
|
0946c72b88 | ||
|
|
7d49b046d4 | ||
|
|
5f0426c840 | ||
|
|
73e239cc5f | ||
|
|
ad670f721a | ||
|
|
028a4a70cf | ||
|
|
77d7960347 | ||
|
|
39821146bd | ||
|
|
7d505a41ac | ||
|
|
e457b2f0d6 | ||
|
|
c9cf7fd4d4 | ||
|
|
b0371dd33d | ||
|
|
25e16c3565 | ||
|
|
7b39527863 | ||
|
|
d861b08866 | ||
|
|
fb438dc108 | ||
|
|
4e6b4f179b | ||
|
|
00d038c8f3 | ||
|
|
a9f6a68952 | ||
|
|
b9142bbc5a | ||
|
|
6c812f663e | ||
|
|
a93ec9783a |
No files matched your search
@@ -1,11 +1,7 @@
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
|
||||
install:
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install lbry-libtorrent
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
|
||||
@@ -4,7 +4,7 @@ LBRY is a decentralized peer-to-peer protocol for publishing and accessing digit
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Built on Python 3.7 and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.99.0"
|
||||
__version__ = "0.100.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+101
-14
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
T = TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
@@ -24,11 +24,11 @@ CURRENCIES = {
|
||||
}
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
class Setting(Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
def __init__(self, doc: str, default: Optional[T] = None,
|
||||
previous_names: Optional[List[str]] = None,
|
||||
metavar: Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
@@ -45,7 +45,7 @@ class Setting(typing.Generic[T]):
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
for location in obj.search_order:
|
||||
@@ -53,7 +53,7 @@ class Setting(typing.Generic[T]):
|
||||
return location[self.name]
|
||||
return self.default
|
||||
|
||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
||||
def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
|
||||
if val == NOT_SET:
|
||||
for location in obj.modify_order:
|
||||
if self.name in location:
|
||||
@@ -63,6 +63,18 @@ class Setting(typing.Generic[T]):
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def is_set(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return location[self.name] == self.default
|
||||
return False
|
||||
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -87,7 +99,7 @@ class String(Setting[str]):
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
@@ -200,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
|
||||
class StringChoice(String):
|
||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
||||
def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
if not valid_values:
|
||||
raise ValueError("No valid values provided")
|
||||
@@ -273,6 +285,75 @@ class Strings(ListSetting):
|
||||
f"'{self.name}' must be a string."
|
||||
|
||||
|
||||
class KnownHubsList:
|
||||
|
||||
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
|
||||
self.file_name = file_name
|
||||
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
|
||||
self.hubs: Dict[Tuple[str, int], Dict] = {}
|
||||
if self.exists:
|
||||
self.load()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.path and os.path.exists(self.path)
|
||||
|
||||
@property
|
||||
def serialized(self) -> Dict[str, Dict]:
|
||||
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
|
||||
|
||||
def filter(self, match_none=False, **kwargs):
|
||||
if not kwargs:
|
||||
return self.hubs
|
||||
result = {}
|
||||
for hub, details in self.hubs.items():
|
||||
for key, constraint in kwargs.items():
|
||||
value = details.get(key)
|
||||
if value == constraint or (match_none and value is None):
|
||||
result[hub] = details
|
||||
break
|
||||
return result
|
||||
|
||||
def load(self):
|
||||
if self.path:
|
||||
with open(self.path, 'r') as known_hubs_file:
|
||||
raw = known_hubs_file.read()
|
||||
for hub, details in yaml.safe_load(raw).items():
|
||||
self.set(hub, details)
|
||||
|
||||
def save(self):
|
||||
if self.path:
|
||||
with open(self.path, 'w') as known_hubs_file:
|
||||
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
|
||||
|
||||
def set(self, hub: str, details: Dict):
|
||||
if hub and hub.count(':') == 1:
|
||||
host, port = hub.split(':')
|
||||
hub_parts = (host, int(port))
|
||||
if hub_parts not in self.hubs:
|
||||
self.hubs[hub_parts] = details
|
||||
return hub
|
||||
|
||||
def add_hubs(self, hubs: List[str]):
|
||||
added = False
|
||||
for hub in hubs:
|
||||
if self.set(hub, {}) is not None:
|
||||
added = True
|
||||
return added
|
||||
|
||||
def items(self):
|
||||
return self.hubs.items()
|
||||
|
||||
def __bool__(self):
|
||||
return len(self) > 0
|
||||
|
||||
def __len__(self):
|
||||
return self.hubs.__len__()
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.hubs)
|
||||
|
||||
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
@@ -377,7 +458,7 @@ class ConfigFileAccess:
|
||||
del self.data[key]
|
||||
|
||||
|
||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
||||
TBC = TypeVar('TBC', bound='BaseConfig')
|
||||
|
||||
|
||||
class BaseConfig:
|
||||
@@ -508,6 +589,9 @@ class CLIConfig(TranscodeConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
@@ -544,6 +628,7 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
hub_timeout = Float("Timeout when making a hub request", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
@@ -565,6 +650,7 @@ class Config(CLIConfig):
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
@@ -655,6 +741,7 @@ class Config(CLIConfig):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.set_default_paths()
|
||||
self.known_hubs = KnownHubsList(self)
|
||||
|
||||
def set_default_paths(self):
|
||||
if 'darwin' in sys.platform.lower():
|
||||
@@ -676,7 +763,7 @@ class Config(CLIConfig):
|
||||
return os.path.join(self.data_dir, 'lbrynet.log')
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_windows_directories() -> Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
@@ -698,14 +785,14 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_darwin_directories() -> Tuple[str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
|
||||
+7
-7
@@ -4,7 +4,7 @@ import logging
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
@@ -31,12 +31,12 @@ class PeerManager:
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048)
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048)
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048)
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
|
||||
@@ -32,17 +32,15 @@ TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
def _event_properties(analytics_id: str, event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
'analytics_id': analytics_id
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
def _download_properties(conf: Config, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: typing.Optional[int],
|
||||
tried_peers_count: typing.Optional[int], connection_failures_count: typing.Optional[int],
|
||||
@@ -54,7 +52,6 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
||||
error: typing.Optional[str] = None, error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
@@ -105,17 +102,15 @@ def _make_context(platform):
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
def __init__(self, conf: Config):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.analytics_id = utils.generate_id().hex()
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
@@ -132,7 +127,6 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
@@ -194,7 +188,7 @@ class AnalyticsManager:
|
||||
error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
self.conf, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, connection_failures_count, added_fixed_peers, fixed_peers_delay,
|
||||
sd_hash, sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error, error_msg,
|
||||
wallet_server
|
||||
@@ -227,7 +221,7 @@ class AnalyticsManager:
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'properties': _event_properties(self.analytics_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
@@ -138,7 +138,7 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
|
||||
@@ -325,8 +325,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
self._video_file_analyzer = VideoFileAnalyzer(conf)
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
self.session_id = base58.b58encode(utils.generate_id()).decode()
|
||||
self.analytics_manager = analytics.AnalyticsManager(conf, self.installation_id, self.session_id)
|
||||
self.analytics_manager = analytics.AnalyticsManager(conf)
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
conf, analytics_manager=self.analytics_manager,
|
||||
skip_components=conf.components_to_skip or []
|
||||
|
||||
@@ -234,8 +234,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
elif txo.script.is_support_claim_data:
|
||||
output['value_type'] = 'emoji'
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
|
||||
+13
-5
@@ -32,6 +32,17 @@ def calculate_sha384_file_hash(file_path):
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
def country_int_to_str(country: int) -> str:
|
||||
r = LocationMessage.Country.Name(country)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
|
||||
|
||||
def country_str_to_int(country: str) -> int:
|
||||
if len(country) == 3:
|
||||
country = 'R' + country
|
||||
return LocationMessage.Country.Value(country)
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -423,14 +434,11 @@ class Language(Metadata):
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
r = LocationMessage.Country.Name(self.message.region)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
return country_int_to_str(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
if len(region) == 3:
|
||||
region = 'R'+region
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
self.message.region = country_str_to_int(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
|
||||
+21
-9
@@ -350,7 +350,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_config = Config()
|
||||
self.server_config = Config(
|
||||
data_dir=server_tmp_dir,
|
||||
wallet_dir=server_tmp_dir,
|
||||
download_dir=server_tmp_dir
|
||||
)
|
||||
self.server_config.transaction_cache_size = 10000
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
@@ -374,6 +378,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await daemon.stop()
|
||||
|
||||
async def add_daemon(self, wallet_node=None, seed=None):
|
||||
start_wallet_node = False
|
||||
if wallet_node is None:
|
||||
wallet_node = WalletNode(
|
||||
self.wallet_node.manager_class,
|
||||
@@ -381,22 +386,23 @@ class CommandTestCase(IntegrationTestCase):
|
||||
port=self.extra_wallet_node_port
|
||||
)
|
||||
self.extra_wallet_node_port += 1
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
start_wallet_node = True
|
||||
|
||||
upload_dir = os.path.join(wallet_node.data_path, 'uploads')
|
||||
os.mkdir(upload_dir)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = wallet_node.data_path
|
||||
conf.wallet_dir = wallet_node.data_path
|
||||
conf.download_dir = wallet_node.data_path
|
||||
conf = Config(
|
||||
# needed during instantiation to access known_hubs path
|
||||
data_dir=wallet_node.data_path,
|
||||
wallet_dir=wallet_node.data_path,
|
||||
download_dir=wallet_node.data_path
|
||||
)
|
||||
conf.upload_dir = upload_dir # not a real conf setting
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = True
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||
conf.known_dht_nodes = []
|
||||
@@ -408,7 +414,13 @@ class CommandTestCase(IntegrationTestCase):
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
if start_wallet_node:
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
else:
|
||||
wallet_node.manager.config = conf
|
||||
wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
wallet_component = WalletComponent(component_manager)
|
||||
|
||||
+5
-9
@@ -21,7 +21,6 @@ import pkg_resources
|
||||
import certifi
|
||||
import aiohttp
|
||||
from prometheus_client import Counter
|
||||
from prometheus_client.registry import REGISTRY
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
|
||||
@@ -276,12 +275,6 @@ class LRUCacheWithMetrics:
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
if self._track_metrics: # needed for tests
|
||||
try:
|
||||
REGISTRY.unregister(self.hits)
|
||||
REGISTRY.unregister(self.misses)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
@@ -310,11 +303,14 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def items(self):
|
||||
return self.cache.items()
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
def pop(self, key, default=None):
|
||||
return self.cache.pop(key, default)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
Binary file not shown.
+18
-2
@@ -8,7 +8,7 @@ from decimal import Decimal
|
||||
from typing import List, Type, MutableSequence, MutableMapping, Optional
|
||||
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError
|
||||
from lbry.conf import Config
|
||||
from lbry.conf import Config, NOT_SET
|
||||
|
||||
from .dewies import dewies_to_lbc
|
||||
from .account import Account
|
||||
@@ -182,7 +182,12 @@ class WalletManager:
|
||||
|
||||
ledger_config = {
|
||||
'auto_connect': True,
|
||||
'explicit_servers': [],
|
||||
'hub_timeout': config.hub_timeout,
|
||||
'default_servers': config.lbryum_servers,
|
||||
'known_hubs': config.known_hubs,
|
||||
'jurisdiction': config.jurisdiction,
|
||||
'concurrent_hub_requests': config.concurrent_hub_requests,
|
||||
'data_path': config.wallet_dir,
|
||||
'tx_cache_size': config.transaction_cache_size
|
||||
}
|
||||
@@ -195,6 +200,10 @@ class WalletManager:
|
||||
os.path.join(wallets_directory, 'default_wallet')
|
||||
)
|
||||
|
||||
if Config.lbryum_servers.is_set_to_default(config):
|
||||
with config.update_config() as c:
|
||||
c.lbryum_servers = NOT_SET
|
||||
|
||||
manager = cls.from_config({
|
||||
'ledgers': {ledger_id: ledger_config},
|
||||
'wallets': [
|
||||
@@ -225,9 +234,16 @@ class WalletManager:
|
||||
async def reset(self):
|
||||
self.ledger.config = {
|
||||
'auto_connect': True,
|
||||
'default_servers': self.config.lbryum_servers,
|
||||
'explicit_servers': [],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
'known_hubs': self.config.known_hubs,
|
||||
'jurisdiction': self.config.jurisdiction,
|
||||
'hub_timeout': self.config.hub_timeout,
|
||||
'concurrent_hub_requests': self.config.concurrent_hub_requests,
|
||||
'data_path': self.config.wallet_dir,
|
||||
}
|
||||
if Config.lbryum_servers.is_set(self.config):
|
||||
self.ledger.config['explicit_servers'] = self.config.lbryum_servers
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
|
||||
|
||||
+64
-21
@@ -14,12 +14,13 @@ from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
from lbry.conf import KnownHubsList
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -29,7 +30,11 @@ class ClientSession(BaseClientSession):
|
||||
self.response_time: Optional[float] = None
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._concurrency = asyncio.Semaphore(concurrency)
|
||||
|
||||
@property
|
||||
def concurrency(self):
|
||||
return self._concurrency._value
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -55,9 +60,9 @@ class ClientSession(BaseClientSession):
|
||||
return result
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
try:
|
||||
await self._concurrency.acquire()
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
request = asyncio.ensure_future(super().send_request(method, args))
|
||||
@@ -91,7 +96,7 @@ class ClientSession(BaseClientSession):
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
self._concurrency.release()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
@@ -154,7 +159,6 @@ class Network:
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
|
||||
self._on_connected_controller = StreamController()
|
||||
self.on_connected = self._on_connected_controller.stream
|
||||
@@ -165,9 +169,13 @@ class Network:
|
||||
self._on_status_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_status = self._on_status_controller.stream
|
||||
|
||||
self._on_hub_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_hub = self._on_hub_controller.stream
|
||||
|
||||
self.subscription_controllers = {
|
||||
'blockchain.headers.subscribe': self._on_header_controller,
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
'blockchain.peers.subscribe': self._on_hub_controller,
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
@@ -179,6 +187,16 @@ class Network:
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
@property
|
||||
def known_hubs(self):
|
||||
if 'known_hubs' not in self.config:
|
||||
return KnownHubsList()
|
||||
return self.config['known_hubs']
|
||||
|
||||
@property
|
||||
def jurisdiction(self):
|
||||
return self.config.get("jurisdiction")
|
||||
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
@@ -189,6 +207,7 @@ class Network:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self.on_hub.listen(self._update_hubs)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
@@ -216,7 +235,13 @@ class Network:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
if self.config['explicit_servers']:
|
||||
hubs = self.config['explicit_servers']
|
||||
elif self.known_hubs:
|
||||
hubs = self.known_hubs
|
||||
else:
|
||||
hubs = self.config['default_servers']
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in hubs))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]:
|
||||
@@ -226,8 +251,9 @@ class Network:
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
n = len(ip_to_hostnames)
|
||||
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['default_servers']))
|
||||
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['explicit_servers']))
|
||||
pongs = {}
|
||||
known_hubs = self.known_hubs
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
@@ -242,6 +268,9 @@ class Network:
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
known_hubs.hubs.setdefault((ip_to_hostnames[remote][0], remote[1]), {}).update(
|
||||
{"country": pong.country_name}
|
||||
)
|
||||
if pong.available:
|
||||
pongs[(ip_to_hostnames[remote][0], remote[1])] = pong
|
||||
return pongs
|
||||
@@ -256,14 +285,19 @@ class Network:
|
||||
random_server = random.choice(list(ip_to_hostnames.keys()))
|
||||
host, port = random_server
|
||||
log.warning("trying fallback to randomly selected spv: %s:%i", host, port)
|
||||
known_hubs.hubs.setdefault((host, port), {})
|
||||
return {(host, port): None}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
for (host, port), pong in fastest_spvs.items():
|
||||
if (pong is not None and self.jurisdiction is not None) and \
|
||||
(pong.country_name != self.jurisdiction):
|
||||
continue
|
||||
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
|
||||
concurrency=self.config['concurrent_hub_requests'])
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
@@ -293,6 +327,8 @@ class Network:
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.debug("discover other hubs %s:%i", *client.server)
|
||||
await self._update_hubs(await client.send_request('server.peers.subscribe', []))
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
@@ -344,24 +380,31 @@ class Network:
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
async with self._concurrency:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
async def _update_hubs(self, hubs):
|
||||
if hubs and hubs != ['']:
|
||||
try:
|
||||
if self.known_hubs.add_hubs(hubs):
|
||||
self.known_hubs.save()
|
||||
except Exception:
|
||||
log.exception("could not add hubs: %s", hubs)
|
||||
|
||||
def get_transaction(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
|
||||
@@ -17,6 +17,7 @@ import lbry
|
||||
from lbry.wallet.server.server import Server
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
|
||||
from lbry.conf import KnownHubsList, Config
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -107,7 +108,8 @@ class Conductor:
|
||||
class WalletNode:
|
||||
|
||||
def __init__(self, manager_class: Type[WalletManager], ledger_class: Type[Ledger],
|
||||
verbose: bool = False, port: int = 5280, default_seed: str = None) -> None:
|
||||
verbose: bool = False, port: int = 5280, default_seed: str = None,
|
||||
data_path: str = None) -> None:
|
||||
self.manager_class = manager_class
|
||||
self.ledger_class = ledger_class
|
||||
self.verbose = verbose
|
||||
@@ -115,12 +117,12 @@ class WalletNode:
|
||||
self.ledger: Optional[Ledger] = None
|
||||
self.wallet: Optional[Wallet] = None
|
||||
self.account: Optional[Account] = None
|
||||
self.data_path: Optional[str] = None
|
||||
self.data_path: str = data_path or tempfile.mkdtemp()
|
||||
self.port = port
|
||||
self.default_seed = default_seed
|
||||
self.known_hubs = KnownHubsList()
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True, config=None):
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
@@ -130,12 +132,17 @@ class WalletNode:
|
||||
'ledgers': {
|
||||
self.ledger_class.get_id(): {
|
||||
'api_port': self.port,
|
||||
'default_servers': [(spv_node.hostname, spv_node.port)],
|
||||
'data_path': self.data_path
|
||||
'explicit_servers': [(spv_node.hostname, spv_node.port)],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
'data_path': self.data_path,
|
||||
'known_hubs': config.known_hubs if config else KnownHubsList(),
|
||||
'hub_timeout': 30,
|
||||
'concurrent_hub_requests': 32,
|
||||
}
|
||||
},
|
||||
'wallets': [wallet_file_name]
|
||||
})
|
||||
self.manager.config = config
|
||||
self.ledger = self.manager.ledgers[self.ledger_class]
|
||||
self.wallet = self.manager.default_wallet
|
||||
if not self.wallet:
|
||||
@@ -172,6 +179,7 @@ class SPVNode:
|
||||
self.udp_port = self.port
|
||||
self.session_timeout = 600
|
||||
self.rpc_port = '0' # disabled by default
|
||||
self.stopped = False
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
@@ -201,10 +209,13 @@ class SPVNode:
|
||||
await self.server.start()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
if self.stopped:
|
||||
return
|
||||
try:
|
||||
await self.server.db.search_index.delete_index()
|
||||
await self.server.db.search_index.stop()
|
||||
await self.server.stop()
|
||||
self.stopped = True
|
||||
finally:
|
||||
cleanup and self.cleanup()
|
||||
|
||||
|
||||
@@ -798,11 +798,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
finally:
|
||||
if self.sql:
|
||||
self.sql.commit()
|
||||
if self.sql and self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
|
||||
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -17,10 +17,6 @@ class FindShortestID:
|
||||
if self.short_id:
|
||||
return '#'+self.short_id
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return cls(), cls.step, cls.finalize
|
||||
|
||||
|
||||
def register_canonical_functions(connection):
|
||||
connection.createaggregatefunction("shortest_id", FindShortestID.factory, 2)
|
||||
connection.create_aggregate("shortest_id", 2, FindShortestID)
|
||||
@@ -5,7 +5,7 @@ import os
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Process
|
||||
|
||||
import apsw
|
||||
import sqlite3
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch.helpers import async_bulk
|
||||
from lbry.wallet.server.env import Env
|
||||
@@ -15,12 +15,11 @@ from lbry.wallet.server.db.elasticsearch.search import extract_doc, SearchIndex,
|
||||
|
||||
async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
logging.info("shard %d starting", shard_num)
|
||||
def exec_factory(cursor, statement, bindings):
|
||||
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
|
||||
cursor.setrowtrace(lambda cursor, row: tpl(*row))
|
||||
return True
|
||||
|
||||
db.setexectrace(exec_factory)
|
||||
def namedtuple_factory(cursor, row):
|
||||
Row = namedtuple('Row', (d[0] for d in cursor.description))
|
||||
return Row(*row)
|
||||
db.row_factory = namedtuple_factory
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
@@ -77,15 +76,10 @@ async def make_es_index(index=None):
|
||||
|
||||
|
||||
async def run(db_path, clients, blocks, shard, index_name='claims'):
|
||||
def itsbusy(*_):
|
||||
logging.info("shard %d: db is busy, retry", shard)
|
||||
return True
|
||||
db = apsw.Connection(db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
|
||||
db.setbusyhandler(itsbusy)
|
||||
db.cursor().execute('pragma journal_mode=wal;')
|
||||
db.cursor().execute('pragma temp_store=memory;')
|
||||
|
||||
producer = get_all(db.cursor(), shard, clients, limit=blocks, index_name=index_name)
|
||||
db = sqlite3.connect(db_path, isolation_level=None, check_same_thread=False, uri=True)
|
||||
db.execute('pragma journal_mode=wal;')
|
||||
db.execute('pragma temp_store=memory;')
|
||||
producer = get_all(db, shard, clients, limit=blocks, index_name=index_name)
|
||||
await asyncio.gather(*(consume(producer, index_name=index_name) for _ in range(min(8, clients))))
|
||||
|
||||
|
||||
|
||||
@@ -231,7 +231,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
""", list(recalculate_claim_hashes)):
|
||||
trending_data.update_claim(row[0], 1E-8*row[1], time_boost)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
@@ -5,7 +5,7 @@ decay rate for high valued claims.
|
||||
|
||||
import math
|
||||
import time
|
||||
import apsw
|
||||
import sqlite3
|
||||
|
||||
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
|
||||
HALF_LIFE = 200
|
||||
@@ -96,16 +96,16 @@ class TrendingDB:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.conn = apsw.Connection(":memory:")
|
||||
self.conn = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.initialised = False
|
||||
self.write_needed = set()
|
||||
|
||||
def execute(self, query, *args, **kwargs):
|
||||
return self.cursor.execute(query, *args, **kwargs)
|
||||
return self.conn.execute(query, *args, **kwargs)
|
||||
|
||||
def executemany(self, query, *args, **kwargs):
|
||||
return self.cursor.executemany(query, *args, **kwargs)
|
||||
return self.conn.executemany(query, *args, **kwargs)
|
||||
|
||||
def begin(self):
|
||||
self.execute("BEGIN;")
|
||||
@@ -251,7 +251,7 @@ class TrendingDB:
|
||||
FROM claims
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in self.write_needed)});
|
||||
""", self.write_needed).fetchall()
|
||||
""", list(self.write_needed)).fetchall()
|
||||
|
||||
db.executemany("""UPDATE claim SET trending_mixed = ?
|
||||
WHERE claim_hash = ?;""", rows)
|
||||
@@ -278,7 +278,7 @@ class TrendingDB:
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
""", list(recalculate_claim_hashes)):
|
||||
claim_hash, lbc = row
|
||||
|
||||
# Insert into trending db if it does not exist
|
||||
|
||||
@@ -47,14 +47,10 @@ class ZScore:
|
||||
return self.last
|
||||
return (self.last - self.mean) / (self.standard_deviation or 1)
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return cls(), cls.step, cls.finalize
|
||||
|
||||
|
||||
def install(connection):
|
||||
connection.createaggregatefunction("zscore", ZScore.factory, 1)
|
||||
connection.cursor().execute(CREATE_TREND_TABLE)
|
||||
connection.create_aggregate("zscore", 1, ZScore)
|
||||
connection.executescript(CREATE_TREND_TABLE)
|
||||
|
||||
|
||||
def run(db, height, final_height, affected_claims):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
|
||||
import apsw
|
||||
import sqlite3
|
||||
from typing import Union, Tuple, Set, List
|
||||
from itertools import chain
|
||||
from decimal import Decimal
|
||||
@@ -21,6 +21,7 @@ from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
from lbry.wallet.server.db.elasticsearch import SearchIndex
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
|
||||
class SQLDB:
|
||||
@@ -233,21 +234,14 @@ class SQLDB:
|
||||
self.pending_deletes = set()
|
||||
|
||||
def open(self):
|
||||
self.db = apsw.Connection(
|
||||
self._db_path,
|
||||
flags=(
|
||||
apsw.SQLITE_OPEN_READWRITE |
|
||||
apsw.SQLITE_OPEN_CREATE |
|
||||
apsw.SQLITE_OPEN_URI
|
||||
)
|
||||
)
|
||||
def exec_factory(cursor, statement, bindings):
|
||||
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
|
||||
cursor.setrowtrace(lambda cursor, row: tpl(*row))
|
||||
return True
|
||||
self.db.setexectrace(exec_factory)
|
||||
self.execute(self.PRAGMAS)
|
||||
self.execute(self.CREATE_TABLES_QUERY)
|
||||
self.db = sqlite3.connect(self._db_path, isolation_level=None, check_same_thread=False, uri=True)
|
||||
|
||||
def namedtuple_factory(cursor, row):
|
||||
Row = namedtuple('Row', (d[0] for d in cursor.description))
|
||||
return Row(*row)
|
||||
self.db.row_factory = namedtuple_factory
|
||||
self.db.executescript(self.PRAGMAS)
|
||||
self.db.executescript(self.CREATE_TABLES_QUERY)
|
||||
register_canonical_functions(self.db)
|
||||
self.blocked_streams = {}
|
||||
self.blocked_channels = {}
|
||||
@@ -319,10 +313,10 @@ class SQLDB:
|
||||
return f"DELETE FROM {table} WHERE {where}", values
|
||||
|
||||
def execute(self, *args):
|
||||
return self.db.cursor().execute(*args)
|
||||
return self.db.execute(*args)
|
||||
|
||||
def executemany(self, *args):
|
||||
return self.db.cursor().executemany(*args)
|
||||
return self.db.executemany(*args)
|
||||
|
||||
def begin(self):
|
||||
self.execute('begin;')
|
||||
@@ -620,7 +614,7 @@ class SQLDB:
|
||||
channel_hash IN ({','.join('?' for _ in changed_channel_keys)}) AND
|
||||
signature IS NOT NULL
|
||||
"""
|
||||
for affected_claim in self.execute(sql, changed_channel_keys.keys()):
|
||||
for affected_claim in self.execute(sql, list(changed_channel_keys.keys())):
|
||||
if affected_claim.claim_hash not in signables:
|
||||
claim_updates.append({
|
||||
'claim_hash': affected_claim.claim_hash,
|
||||
@@ -677,7 +671,7 @@ class SQLDB:
|
||||
signature_valid=CASE WHEN signature IS NOT NULL THEN 0 END,
|
||||
channel_join=NULL, canonical_url=NULL
|
||||
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
|
||||
""", spent_claims
|
||||
""", list(spent_claims)
|
||||
)
|
||||
sub_timer.stop()
|
||||
|
||||
@@ -802,12 +796,13 @@ class SQLDB:
|
||||
|
||||
def update_claimtrie(self, height, changed_claim_hashes, deleted_names, timer):
|
||||
r = timer.run
|
||||
binary_claim_hashes = list(changed_claim_hashes)
|
||||
|
||||
r(self._calculate_activation_height, height)
|
||||
r(self._update_support_amount, changed_claim_hashes)
|
||||
r(self._update_support_amount, binary_claim_hashes)
|
||||
|
||||
r(self._update_effective_amount, height, changed_claim_hashes)
|
||||
r(self._perform_overtake, height, changed_claim_hashes, list(deleted_names))
|
||||
r(self._update_effective_amount, height, binary_claim_hashes)
|
||||
r(self._perform_overtake, height, binary_claim_hashes, list(deleted_names))
|
||||
|
||||
r(self._update_effective_amount, height)
|
||||
r(self._perform_overtake, height, [], [])
|
||||
|
||||
@@ -74,9 +74,11 @@ class Env:
|
||||
self.anon_logs = self.boolean('ANON_LOGS', False)
|
||||
self.log_sessions = self.integer('LOG_SESSIONS', 3600)
|
||||
self.allow_lan_udp = self.boolean('ALLOW_LAN_UDP', False)
|
||||
self.country = self.default('COUNTRY', 'US')
|
||||
# Peer discovery
|
||||
self.peer_discovery = self.peer_discovery_enum()
|
||||
self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
|
||||
self.peer_hubs = self.extract_peer_hubs()
|
||||
self.force_proxy = self.boolean('FORCE_PROXY', False)
|
||||
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
|
||||
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
|
||||
@@ -269,3 +271,6 @@ class Env:
|
||||
return self.PD_SELF
|
||||
else:
|
||||
return self.PD_ON
|
||||
|
||||
def extract_peer_hubs(self):
|
||||
return [hub.strip() for hub in self.default('PEER_HUBS', '').split(',')]
|
||||
@@ -1,302 +0,0 @@
|
||||
# Copyright (c) 2017, Neil Booth
|
||||
#
|
||||
# All rights reserved.
|
||||
#
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining
|
||||
# a copy of this software and associated documentation files (the
|
||||
# "Software"), to deal in the Software without restriction, including
|
||||
# without limitation the rights to use, copy, modify, merge, publish,
|
||||
# distribute, sublicense, and/or sell copies of the Software, and to
|
||||
# permit persons to whom the Software is furnished to do so, subject to
|
||||
# the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
"""Representation of a peer server."""
|
||||
|
||||
from ipaddress import ip_address
|
||||
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.util import cachedproperty
|
||||
|
||||
from typing import Dict
|
||||
|
||||
|
||||
class Peer:
|
||||
|
||||
# Protocol version
|
||||
ATTRS = ('host', 'features',
|
||||
# metadata
|
||||
'source', 'ip_addr',
|
||||
'last_good', 'last_try', 'try_count')
|
||||
FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max',
|
||||
'ssl_port', 'tcp_port')
|
||||
# This should be set by the application
|
||||
DEFAULT_PORTS: Dict[str, int] = {}
|
||||
|
||||
def __init__(self, host, features, source='unknown', ip_addr=None,
|
||||
last_good=0, last_try=0, try_count=0):
|
||||
"""Create a peer given a host name (or IP address as a string),
|
||||
a dictionary of features, and a record of the source."""
|
||||
assert isinstance(host, str)
|
||||
assert isinstance(features, dict)
|
||||
assert host in features.get('hosts', {})
|
||||
self.host = host
|
||||
self.features = features.copy()
|
||||
# Canonicalize / clean-up
|
||||
for feature in self.FEATURES:
|
||||
self.features[feature] = getattr(self, feature)
|
||||
# Metadata
|
||||
self.source = source
|
||||
self.ip_addr = ip_addr
|
||||
# last_good represents the last connection that was
|
||||
# successful *and* successfully verified, at which point
|
||||
# try_count is set to 0. Failure to connect or failure to
|
||||
# verify increment the try_count.
|
||||
self.last_good = last_good
|
||||
self.last_try = last_try
|
||||
self.try_count = try_count
|
||||
# Transient, non-persisted metadata
|
||||
self.bad = False
|
||||
self.other_port_pairs = set()
|
||||
self.status = 2
|
||||
|
||||
@classmethod
|
||||
def peers_from_features(cls, features, source):
|
||||
peers = []
|
||||
if isinstance(features, dict):
|
||||
hosts = features.get('hosts')
|
||||
if isinstance(hosts, dict):
|
||||
peers = [Peer(host, features, source=source)
|
||||
for host in hosts if isinstance(host, str)]
|
||||
return peers
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, item):
|
||||
"""Deserialize from a dictionary."""
|
||||
return cls(**item)
|
||||
|
||||
def matches(self, peers):
|
||||
"""Return peers whose host matches our hostname or IP address.
|
||||
Additionally include all peers whose IP address matches our
|
||||
hostname if that is an IP address.
|
||||
"""
|
||||
candidates = (self.host.lower(), self.ip_addr)
|
||||
return [peer for peer in peers
|
||||
if peer.host.lower() in candidates
|
||||
or peer.ip_addr == self.host]
|
||||
|
||||
def __str__(self):
|
||||
return self.host
|
||||
|
||||
def update_features(self, features):
|
||||
"""Update features in-place."""
|
||||
try:
|
||||
tmp = Peer(self.host, features)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
self.update_features_from_peer(tmp)
|
||||
|
||||
def update_features_from_peer(self, peer):
|
||||
if peer != self:
|
||||
self.features = peer.features
|
||||
for feature in self.FEATURES:
|
||||
setattr(self, feature, getattr(peer, feature))
|
||||
|
||||
def connection_port_pairs(self):
|
||||
"""Return a list of (kind, port) pairs to try when making a
|
||||
connection."""
|
||||
# Use a list not a set - it's important to try the registered
|
||||
# ports first.
|
||||
pairs = [('SSL', self.ssl_port), ('TCP', self.tcp_port)]
|
||||
while self.other_port_pairs:
|
||||
pairs.append(self.other_port_pairs.pop())
|
||||
return [pair for pair in pairs if pair[1]]
|
||||
|
||||
def mark_bad(self):
|
||||
"""Mark as bad to avoid reconnects but also to remember for a
|
||||
while."""
|
||||
self.bad = True
|
||||
|
||||
def check_ports(self, other):
|
||||
"""Remember differing ports in case server operator changed them
|
||||
or removed one."""
|
||||
if other.ssl_port != self.ssl_port:
|
||||
self.other_port_pairs.add(('SSL', other.ssl_port))
|
||||
if other.tcp_port != self.tcp_port:
|
||||
self.other_port_pairs.add(('TCP', other.tcp_port))
|
||||
return bool(self.other_port_pairs)
|
||||
|
||||
@cachedproperty
|
||||
def is_tor(self):
|
||||
return self.host.endswith('.onion')
|
||||
|
||||
@cachedproperty
|
||||
def is_valid(self):
|
||||
ip = self.ip_address
|
||||
if ip:
|
||||
return ((ip.is_global or ip.is_private)
|
||||
and not (ip.is_multicast or ip.is_unspecified))
|
||||
return util.is_valid_hostname(self.host)
|
||||
|
||||
@cachedproperty
|
||||
def is_public(self):
|
||||
ip = self.ip_address
|
||||
if ip:
|
||||
return self.is_valid and not ip.is_private
|
||||
else:
|
||||
return self.is_valid and self.host != 'localhost'
|
||||
|
||||
@cachedproperty
|
||||
def ip_address(self):
|
||||
"""The host as a python ip_address object, or None."""
|
||||
try:
|
||||
return ip_address(self.host)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def bucket(self):
|
||||
if self.is_tor:
|
||||
return 'onion'
|
||||
if not self.ip_addr:
|
||||
return ''
|
||||
return tuple(self.ip_addr.split('.')[:2])
|
||||
|
||||
def serialize(self):
|
||||
"""Serialize to a dictionary."""
|
||||
return {attr: getattr(self, attr) for attr in self.ATTRS}
|
||||
|
||||
def _port(self, key):
|
||||
hosts = self.features.get('hosts')
|
||||
if isinstance(hosts, dict):
|
||||
host = hosts.get(self.host)
|
||||
port = self._integer(key, host)
|
||||
if port and 0 < port < 65536:
|
||||
return port
|
||||
return None
|
||||
|
||||
def _integer(self, key, d=None):
|
||||
d = d or self.features
|
||||
result = d.get(key) if isinstance(d, dict) else None
|
||||
if isinstance(result, str):
|
||||
try:
|
||||
result = int(result)
|
||||
except ValueError:
|
||||
pass
|
||||
return result if isinstance(result, int) else None
|
||||
|
||||
def _string(self, key):
|
||||
result = self.features.get(key)
|
||||
return result if isinstance(result, str) else None
|
||||
|
||||
@cachedproperty
|
||||
def genesis_hash(self):
|
||||
"""Returns None if no SSL port, otherwise the port as an integer."""
|
||||
return self._string('genesis_hash')
|
||||
|
||||
@cachedproperty
|
||||
def ssl_port(self):
|
||||
"""Returns None if no SSL port, otherwise the port as an integer."""
|
||||
return self._port('ssl_port')
|
||||
|
||||
@cachedproperty
|
||||
def tcp_port(self):
|
||||
"""Returns None if no TCP port, otherwise the port as an integer."""
|
||||
return self._port('tcp_port')
|
||||
|
||||
@cachedproperty
|
||||
def server_version(self):
|
||||
"""Returns the server version as a string if known, otherwise None."""
|
||||
return self._string('server_version')
|
||||
|
||||
@cachedproperty
|
||||
def pruning(self):
|
||||
"""Returns the pruning level as an integer. None indicates no
|
||||
pruning."""
|
||||
pruning = self._integer('pruning')
|
||||
if pruning and pruning > 0:
|
||||
return pruning
|
||||
return None
|
||||
|
||||
def _protocol_version_string(self, key):
|
||||
version_str = self.features.get(key)
|
||||
ptuple = util.protocol_tuple(version_str)
|
||||
return util.version_string(ptuple)
|
||||
|
||||
@cachedproperty
|
||||
def protocol_min(self):
|
||||
"""Minimum protocol version as a string, e.g., 1.0"""
|
||||
return self._protocol_version_string('protocol_min')
|
||||
|
||||
@cachedproperty
|
||||
def protocol_max(self):
|
||||
"""Maximum protocol version as a string, e.g., 1.1"""
|
||||
return self._protocol_version_string('protocol_max')
|
||||
|
||||
def to_tuple(self):
|
||||
"""The tuple ((ip, host, details) expected in response
|
||||
to a peers subscription."""
|
||||
details = self.real_name().split()[1:]
|
||||
return (self.ip_addr or self.host, self.host, details)
|
||||
|
||||
def real_name(self):
|
||||
"""Real name of this peer as used on IRC."""
|
||||
def port_text(letter, port):
|
||||
if port == self.DEFAULT_PORTS.get(letter):
|
||||
return letter
|
||||
else:
|
||||
return letter + str(port)
|
||||
|
||||
parts = [self.host, 'v' + self.protocol_max]
|
||||
if self.pruning:
|
||||
parts.append(f'p{self.pruning:d}')
|
||||
for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)):
|
||||
if port:
|
||||
parts.append(port_text(letter, port))
|
||||
return ' '.join(parts)
|
||||
|
||||
@classmethod
|
||||
def from_real_name(cls, real_name, source):
|
||||
"""Real name is a real name as on IRC, such as
|
||||
|
||||
"erbium1.sytes.net v1.0 s t"
|
||||
|
||||
Returns an instance of this Peer class.
|
||||
"""
|
||||
host = 'nohost'
|
||||
features = {}
|
||||
ports = {}
|
||||
for n, part in enumerate(real_name.split()):
|
||||
if n == 0:
|
||||
host = part
|
||||
continue
|
||||
if part[0] in ('s', 't'):
|
||||
if len(part) == 1:
|
||||
port = cls.DEFAULT_PORTS[part[0]]
|
||||
else:
|
||||
port = part[1:]
|
||||
if part[0] == 's':
|
||||
ports['ssl_port'] = port
|
||||
else:
|
||||
ports['tcp_port'] = port
|
||||
elif part[0] == 'v':
|
||||
features['protocol_max'] = features['protocol_min'] = part[1:]
|
||||
elif part[0] == 'p':
|
||||
features['pruning'] = part[1:]
|
||||
|
||||
features.update(ports)
|
||||
features['hosts'] = {host: ports}
|
||||
|
||||
return cls(host, features, source)
|
||||
@@ -1,506 +0,0 @@
|
||||
# Copyright (c) 2017-2018, Neil Booth
|
||||
#
|
||||
# All rights reserved.
|
||||
#
|
||||
# See the file "LICENCE" for information about the copyright
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Peer management."""
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import typing
|
||||
from asyncio import Event, sleep
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
from lbry.wallet.tasks import TaskGroup
|
||||
from lbry.wallet.rpc import (
|
||||
Connector, RPCSession, SOCKSProxy, Notification, handler_invocation,
|
||||
SOCKSError, RPCError
|
||||
)
|
||||
from lbry.wallet.server.peer import Peer
|
||||
from lbry.wallet.server.util import class_logger, protocol_tuple
|
||||
|
||||
PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4)
|
||||
STALE_SECS = 24 * 3600
|
||||
WAKEUP_SECS = 300
|
||||
|
||||
|
||||
class BadPeerError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def assert_good(message, result, instance):
|
||||
if not isinstance(result, instance):
|
||||
raise BadPeerError(f'{message} returned bad result type '
|
||||
f'{type(result).__name__}')
|
||||
|
||||
|
||||
class PeerSession(RPCSession):
|
||||
"""An outgoing session to a peer."""
|
||||
|
||||
async def handle_request(self, request):
|
||||
# We subscribe so might be unlucky enough to get a notification...
|
||||
if (isinstance(request, Notification) and
|
||||
request.method == 'blockchain.headers.subscribe'):
|
||||
pass
|
||||
else:
|
||||
await handler_invocation(None, request) # Raises
|
||||
|
||||
|
||||
class PeerManager:
|
||||
"""Looks after the DB of peer network servers.
|
||||
|
||||
Attempts to maintain a connection with up to 8 peers.
|
||||
Issues a 'peers.subscribe' RPC to them and tells them our data.
|
||||
"""
|
||||
def __init__(self, env, db):
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
# Initialise the Peer class
|
||||
Peer.DEFAULT_PORTS = env.coin.PEER_DEFAULT_PORTS
|
||||
self.env = env
|
||||
self.db = db
|
||||
|
||||
# Our clearnet and Tor Peers, if any
|
||||
sclass = env.coin.SESSIONCLS
|
||||
self.myselves = [Peer(ident.host, sclass.server_features(env), 'env')
|
||||
for ident in env.identities]
|
||||
self.server_version_args = sclass.server_version_args()
|
||||
# Peers have one entry per hostname. Once connected, the
|
||||
# ip_addr property is either None, an onion peer, or the
|
||||
# IP address that was connected to. Adding a peer will evict
|
||||
# any other peers with the same host name or IP address.
|
||||
self.peers: typing.Set[Peer] = set()
|
||||
self.permit_onion_peer_time = time.time()
|
||||
self.proxy = None
|
||||
self.group = TaskGroup()
|
||||
|
||||
def _my_clearnet_peer(self):
|
||||
"""Returns the clearnet peer representing this server, if any."""
|
||||
clearnet = [peer for peer in self.myselves if not peer.is_tor]
|
||||
return clearnet[0] if clearnet else None
|
||||
|
||||
def _set_peer_statuses(self):
|
||||
"""Set peer statuses."""
|
||||
cutoff = time.time() - STALE_SECS
|
||||
for peer in self.peers:
|
||||
if peer.bad:
|
||||
peer.status = PEER_BAD
|
||||
elif peer.last_good > cutoff:
|
||||
peer.status = PEER_GOOD
|
||||
elif peer.last_good:
|
||||
peer.status = PEER_STALE
|
||||
else:
|
||||
peer.status = PEER_NEVER
|
||||
|
||||
def _features_to_register(self, peer, remote_peers):
|
||||
"""If we should register ourselves to the remote peer, which has
|
||||
reported the given list of known peers, return the clearnet
|
||||
identity features to register, otherwise None.
|
||||
"""
|
||||
# Announce ourself if not present. Don't if disabled, we
|
||||
# are a non-public IP address, or to ourselves.
|
||||
if not self.env.peer_announce or peer in self.myselves:
|
||||
return None
|
||||
my = self._my_clearnet_peer()
|
||||
if not my or not my.is_public:
|
||||
return None
|
||||
# Register if no matches, or ports have changed
|
||||
for peer in my.matches(remote_peers):
|
||||
if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port:
|
||||
return None
|
||||
return my.features
|
||||
|
||||
def _permit_new_onion_peer(self):
|
||||
"""Accept a new onion peer only once per random time interval."""
|
||||
now = time.time()
|
||||
if now < self.permit_onion_peer_time:
|
||||
return False
|
||||
self.permit_onion_peer_time = now + random.randrange(0, 1200)
|
||||
return True
|
||||
|
||||
async def _import_peers(self):
|
||||
"""Import hard-coded peers from a file or the coin defaults."""
|
||||
imported_peers = self.myselves.copy()
|
||||
# Add the hard-coded ones unless only reporting ourself
|
||||
if self.env.peer_discovery != self.env.PD_SELF:
|
||||
imported_peers.extend(Peer.from_real_name(real_name, 'coins.py')
|
||||
for real_name in self.env.coin.PEERS)
|
||||
await self._note_peers(imported_peers, limit=None)
|
||||
|
||||
async def _detect_proxy(self):
|
||||
"""Detect a proxy if we don't have one and some time has passed since
|
||||
the last attempt.
|
||||
|
||||
If found self.proxy is set to a SOCKSProxy instance, otherwise
|
||||
None.
|
||||
"""
|
||||
host = self.env.tor_proxy_host
|
||||
if self.env.tor_proxy_port is None:
|
||||
ports = [9050, 9150, 1080]
|
||||
else:
|
||||
ports = [self.env.tor_proxy_port]
|
||||
while True:
|
||||
self.logger.info(f'trying to detect proxy on "{host}" '
|
||||
f'ports {ports}')
|
||||
proxy = await SOCKSProxy.auto_detect_host(host, ports, None)
|
||||
if proxy:
|
||||
self.proxy = proxy
|
||||
self.logger.info(f'detected {proxy}')
|
||||
return
|
||||
self.logger.info('no proxy detected, will try later')
|
||||
await sleep(900)
|
||||
|
||||
async def _note_peers(self, peers, limit=2, check_ports=False,
|
||||
source=None):
|
||||
"""Add a limited number of peers that are not already present."""
|
||||
new_peers = []
|
||||
for peer in peers:
|
||||
if not peer.is_public or (peer.is_tor and not self.proxy):
|
||||
continue
|
||||
|
||||
matches = peer.matches(self.peers)
|
||||
if not matches:
|
||||
new_peers.append(peer)
|
||||
elif check_ports:
|
||||
for match in matches:
|
||||
if match.check_ports(peer):
|
||||
self.logger.info(f'ports changed for {peer}')
|
||||
match.retry_event.set()
|
||||
|
||||
if new_peers:
|
||||
source = source or new_peers[0].source
|
||||
if limit:
|
||||
random.shuffle(new_peers)
|
||||
use_peers = new_peers[:limit]
|
||||
else:
|
||||
use_peers = new_peers
|
||||
for peer in use_peers:
|
||||
self.logger.info(f'accepted new peer {peer} from {source}')
|
||||
peer.retry_event = Event()
|
||||
self.peers.add(peer)
|
||||
await self.group.add(self._monitor_peer(peer))
|
||||
|
||||
async def _monitor_peer(self, peer):
|
||||
# Stop monitoring if we were dropped (a duplicate peer)
|
||||
while peer in self.peers:
|
||||
if await self._should_drop_peer(peer):
|
||||
self.peers.discard(peer)
|
||||
break
|
||||
# Figure out how long to sleep before retrying. Retry a
|
||||
# good connection when it is about to turn stale, otherwise
|
||||
# exponentially back off retries.
|
||||
if peer.try_count == 0:
|
||||
pause = STALE_SECS - WAKEUP_SECS * 2
|
||||
else:
|
||||
pause = WAKEUP_SECS * 2 ** peer.try_count
|
||||
pending, done = await asyncio.wait([peer.retry_event.wait()], timeout=pause)
|
||||
if done:
|
||||
peer.retry_event.clear()
|
||||
|
||||
async def _should_drop_peer(self, peer):
|
||||
peer.try_count += 1
|
||||
is_good = False
|
||||
for kind, port in peer.connection_port_pairs():
|
||||
peer.last_try = time.time()
|
||||
|
||||
kwargs = {}
|
||||
if kind == 'SSL':
|
||||
kwargs['ssl'] = ssl.SSLContext(ssl.PROTOCOL_TLS)
|
||||
|
||||
host = self.env.cs_host(for_rpc=False)
|
||||
if isinstance(host, list):
|
||||
host = host[0]
|
||||
|
||||
if self.env.force_proxy or peer.is_tor:
|
||||
if not self.proxy:
|
||||
return
|
||||
kwargs['proxy'] = self.proxy
|
||||
kwargs['resolve'] = not peer.is_tor
|
||||
elif host:
|
||||
# Use our listening Host/IP for outgoing non-proxy
|
||||
# connections so our peers see the correct source.
|
||||
kwargs['local_addr'] = (host, None)
|
||||
|
||||
peer_text = f'[{peer}:{port} {kind}]'
|
||||
try:
|
||||
async with Connector(PeerSession, peer.host, port,
|
||||
**kwargs) as session:
|
||||
await asyncio.wait_for(
|
||||
self._verify_peer(session, peer),
|
||||
120 if peer.is_tor else 30
|
||||
)
|
||||
is_good = True
|
||||
break
|
||||
except BadPeerError as e:
|
||||
self.logger.error(f'{peer_text} marking bad: ({e})')
|
||||
peer.mark_bad()
|
||||
break
|
||||
except RPCError as e:
|
||||
self.logger.error(f'{peer_text} RPC error: {e.message} '
|
||||
f'({e.code})')
|
||||
except (OSError, SOCKSError, ConnectionError, asyncio.TimeoutError) as e:
|
||||
self.logger.info(f'{peer_text} {e}')
|
||||
|
||||
if is_good:
|
||||
now = time.time()
|
||||
elapsed = now - peer.last_try
|
||||
self.logger.info(f'{peer_text} verified in {elapsed:.1f}s')
|
||||
peer.try_count = 0
|
||||
peer.last_good = now
|
||||
peer.source = 'peer'
|
||||
# At most 2 matches if we're a host name, potentially
|
||||
# several if we're an IP address (several instances
|
||||
# can share a NAT).
|
||||
matches = peer.matches(self.peers)
|
||||
for match in matches:
|
||||
if match.ip_address:
|
||||
if len(matches) > 1:
|
||||
self.peers.remove(match)
|
||||
# Force the peer's monitoring task to exit
|
||||
match.retry_event.set()
|
||||
elif peer.host in match.features['hosts']:
|
||||
match.update_features_from_peer(peer)
|
||||
else:
|
||||
# Forget the peer if long-term unreachable
|
||||
if peer.last_good and not peer.bad:
|
||||
try_limit = 10
|
||||
else:
|
||||
try_limit = 3
|
||||
if peer.try_count >= try_limit:
|
||||
desc = 'bad' if peer.bad else 'unreachable'
|
||||
self.logger.info(f'forgetting {desc} peer: {peer}')
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _verify_peer(self, session, peer):
|
||||
if not peer.is_tor:
|
||||
address = session.peer_address()
|
||||
if address:
|
||||
peer.ip_addr = address[0]
|
||||
|
||||
# server.version goes first
|
||||
message = 'server.version'
|
||||
result = await session.send_request(message, self.server_version_args)
|
||||
assert_good(message, result, list)
|
||||
|
||||
# Protocol version 1.1 returns a pair with the version first
|
||||
if len(result) != 2 or not all(isinstance(x, str) for x in result):
|
||||
raise BadPeerError(f'bad server.version result: {result}')
|
||||
server_version, protocol_version = result
|
||||
peer.server_version = server_version
|
||||
peer.features['server_version'] = server_version
|
||||
ptuple = protocol_tuple(protocol_version)
|
||||
|
||||
await asyncio.wait([
|
||||
self._send_headers_subscribe(session, peer, ptuple),
|
||||
self._send_server_features(session, peer),
|
||||
self._send_peers_subscribe(session, peer)
|
||||
])
|
||||
|
||||
async def _send_headers_subscribe(self, session, peer, ptuple):
|
||||
message = 'blockchain.headers.subscribe'
|
||||
result = await session.send_request(message)
|
||||
assert_good(message, result, dict)
|
||||
|
||||
our_height = self.db.db_height
|
||||
if ptuple < (1, 3):
|
||||
their_height = result.get('block_height')
|
||||
else:
|
||||
their_height = result.get('height')
|
||||
if not isinstance(their_height, int):
|
||||
raise BadPeerError(f'invalid height {their_height}')
|
||||
if abs(our_height - their_height) > 5:
|
||||
raise BadPeerError(f'bad height {their_height:,d} '
|
||||
f'(ours: {our_height:,d})')
|
||||
|
||||
# Check prior header too in case of hard fork.
|
||||
check_height = min(our_height, their_height)
|
||||
raw_header = self.db.raw_header(check_height)
|
||||
if ptuple >= (1, 4):
|
||||
ours = raw_header.hex()
|
||||
message = 'blockchain.block.header'
|
||||
theirs = await session.send_request(message, [check_height])
|
||||
assert_good(message, theirs, str)
|
||||
if ours != theirs:
|
||||
raise BadPeerError(f'our header {ours} and '
|
||||
f'theirs {theirs} differ')
|
||||
else:
|
||||
ours = self.env.coin.electrum_header(raw_header, check_height)
|
||||
ours = ours.get('prev_block_hash')
|
||||
message = 'blockchain.block.get_header'
|
||||
theirs = await session.send_request(message, [check_height])
|
||||
assert_good(message, theirs, dict)
|
||||
theirs = theirs.get('prev_block_hash')
|
||||
if ours != theirs:
|
||||
raise BadPeerError(f'our header hash {ours} and '
|
||||
f'theirs {theirs} differ')
|
||||
|
||||
async def _send_server_features(self, session, peer):
|
||||
message = 'server.features'
|
||||
features = await session.send_request(message)
|
||||
assert_good(message, features, dict)
|
||||
hosts = [host.lower() for host in features.get('hosts', {})]
|
||||
if self.env.coin.GENESIS_HASH != features.get('genesis_hash'):
|
||||
raise BadPeerError('incorrect genesis hash')
|
||||
elif peer.host.lower() in hosts:
|
||||
peer.update_features(features)
|
||||
else:
|
||||
raise BadPeerError(f'not listed in own hosts list {hosts}')
|
||||
|
||||
async def _send_peers_subscribe(self, session, peer):
|
||||
message = 'server.peers.subscribe'
|
||||
raw_peers = await session.send_request(message)
|
||||
assert_good(message, raw_peers, list)
|
||||
|
||||
# Check the peers list we got from a remote peer.
|
||||
# Each is expected to be of the form:
|
||||
# [ip_addr, hostname, ['v1.0', 't51001', 's51002']]
|
||||
# Call add_peer if the remote doesn't appear to know about us.
|
||||
try:
|
||||
real_names = [' '.join([u[1]] + u[2]) for u in raw_peers]
|
||||
peers = [Peer.from_real_name(real_name, str(peer))
|
||||
for real_name in real_names]
|
||||
except Exception:
|
||||
raise BadPeerError('bad server.peers.subscribe response')
|
||||
|
||||
await self._note_peers(peers)
|
||||
features = self._features_to_register(peer, peers)
|
||||
if not features:
|
||||
return
|
||||
self.logger.info(f'registering ourself with {peer}')
|
||||
# We only care to wait for the response
|
||||
await session.send_request('server.add_peer', [features])
|
||||
|
||||
#
|
||||
# External interface
|
||||
#
|
||||
async def discover_peers(self):
|
||||
"""Perform peer maintenance. This includes
|
||||
|
||||
1) Forgetting unreachable peers.
|
||||
2) Verifying connectivity of new peers.
|
||||
3) Retrying old peers at regular intervals.
|
||||
"""
|
||||
if self.env.peer_discovery != self.env.PD_ON:
|
||||
self.logger.info('peer discovery is disabled')
|
||||
return
|
||||
|
||||
self.logger.info(f'beginning peer discovery. Force use of '
|
||||
f'proxy: {self.env.force_proxy}')
|
||||
|
||||
self.group.add(self._detect_proxy())
|
||||
self.group.add(self._import_peers())
|
||||
|
||||
def info(self) -> typing.Dict[str, int]:
|
||||
"""The number of peers."""
|
||||
self._set_peer_statuses()
|
||||
counter = Counter(peer.status for peer in self.peers)
|
||||
return {
|
||||
'bad': counter[PEER_BAD],
|
||||
'good': counter[PEER_GOOD],
|
||||
'never': counter[PEER_NEVER],
|
||||
'stale': counter[PEER_STALE],
|
||||
'total': len(self.peers),
|
||||
}
|
||||
|
||||
async def add_localRPC_peer(self, real_name):
|
||||
"""Add a peer passed by the admin over LocalRPC."""
|
||||
await self._note_peers([Peer.from_real_name(real_name, 'RPC')])
|
||||
|
||||
async def on_add_peer(self, features, source_info):
|
||||
"""Add a peer (but only if the peer resolves to the source)."""
|
||||
if not source_info:
|
||||
self.logger.info('ignored add_peer request: no source info')
|
||||
return False
|
||||
source = source_info[0]
|
||||
peers = Peer.peers_from_features(features, source)
|
||||
if not peers:
|
||||
self.logger.info('ignored add_peer request: no peers given')
|
||||
return False
|
||||
|
||||
# Just look at the first peer, require it
|
||||
peer = peers[0]
|
||||
host = peer.host
|
||||
if peer.is_tor:
|
||||
permit = self._permit_new_onion_peer()
|
||||
reason = 'rate limiting'
|
||||
else:
|
||||
getaddrinfo = asyncio.get_event_loop().getaddrinfo
|
||||
try:
|
||||
infos = await getaddrinfo(host, 80, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
permit = False
|
||||
reason = 'address resolution failure'
|
||||
else:
|
||||
permit = any(source == info[-1][0] for info in infos)
|
||||
reason = 'source-destination mismatch'
|
||||
|
||||
if permit:
|
||||
self.logger.info(f'accepted add_peer request from {source} '
|
||||
f'for {host}')
|
||||
await self._note_peers([peer], check_ports=True)
|
||||
else:
|
||||
self.logger.warning(f'rejected add_peer request from {source} '
|
||||
f'for {host} ({reason})')
|
||||
|
||||
return permit
|
||||
|
||||
def on_peers_subscribe(self, is_tor):
|
||||
"""Returns the server peers as a list of (ip, host, details) tuples.
|
||||
|
||||
We return all peers we've connected to in the last day.
|
||||
Additionally, if we don't have onion routing, we return a few
|
||||
hard-coded onion servers.
|
||||
"""
|
||||
cutoff = time.time() - STALE_SECS
|
||||
recent = [peer for peer in self.peers
|
||||
if peer.last_good > cutoff and
|
||||
not peer.bad and peer.is_public]
|
||||
onion_peers = []
|
||||
|
||||
# Always report ourselves if valid (even if not public)
|
||||
peers = {myself for myself in self.myselves
|
||||
if myself.last_good > cutoff}
|
||||
|
||||
# Bucket the clearnet peers and select up to two from each
|
||||
buckets = defaultdict(list)
|
||||
for peer in recent:
|
||||
if peer.is_tor:
|
||||
onion_peers.append(peer)
|
||||
else:
|
||||
buckets[peer.bucket()].append(peer)
|
||||
for bucket_peers in buckets.values():
|
||||
random.shuffle(bucket_peers)
|
||||
peers.update(bucket_peers[:2])
|
||||
|
||||
# Add up to 20% onion peers (but up to 10 is OK anyway)
|
||||
random.shuffle(onion_peers)
|
||||
max_onion = 50 if is_tor else max(10, len(peers) // 4)
|
||||
|
||||
peers.update(onion_peers[:max_onion])
|
||||
|
||||
return [peer.to_tuple() for peer in peers]
|
||||
|
||||
def proxy_peername(self):
|
||||
"""Return the peername of the proxy, if there is a proxy, otherwise
|
||||
None."""
|
||||
return self.proxy.peername if self.proxy else None
|
||||
|
||||
def rpc_data(self):
|
||||
"""Peer data for the peers RPC method."""
|
||||
self._set_peer_statuses()
|
||||
descs = ['good', 'stale', 'never', 'bad']
|
||||
|
||||
def peer_data(peer):
|
||||
data = peer.serialize()
|
||||
data['status'] = descs[peer.status]
|
||||
return data
|
||||
|
||||
def peer_key(peer):
|
||||
return (peer.bad, -peer.last_good)
|
||||
|
||||
return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)]
|
||||
@@ -114,7 +114,7 @@ class Server:
|
||||
await self.start_prometheus()
|
||||
if self.env.udp_port:
|
||||
await self.bp.status_server.start(
|
||||
0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1],
|
||||
0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1], self.env.country,
|
||||
self.env.host, self.env.udp_port, self.env.allow_lan_udp
|
||||
)
|
||||
await _start_cancellable(self.bp.fetch_and_process_blocks)
|
||||
|
||||
@@ -39,7 +39,6 @@ from lbry.wallet.server import text
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, Base58Error
|
||||
from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.peers import PeerManager
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet.server.mempool import MemPool
|
||||
@@ -185,7 +184,6 @@ class SessionManager:
|
||||
self.bp = bp
|
||||
self.daemon = daemon
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = PeerManager(env, db)
|
||||
self.shutdown_event = shutdown_event
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
|
||||
@@ -209,7 +207,7 @@ class SessionManager:
|
||||
else:
|
||||
protocol_class = self.env.coin.SESSIONCLS
|
||||
protocol_factory = partial(protocol_class, self, self.db,
|
||||
self.mempool, self.peer_mgr, kind)
|
||||
self.mempool, kind)
|
||||
|
||||
host, port = args[:2]
|
||||
try:
|
||||
@@ -369,7 +367,7 @@ class SessionManager:
|
||||
'logged': logged,
|
||||
'paused': paused,
|
||||
'pid': os.getpid(),
|
||||
'peers': self.peer_mgr.info(),
|
||||
'peers': [],
|
||||
'requests': pending_requests,
|
||||
'method_counts': method_counts,
|
||||
'sessions': self.session_count(),
|
||||
@@ -436,7 +434,7 @@ class SessionManager:
|
||||
|
||||
real_name: "bch.electrumx.cash t50001 s50002" for example
|
||||
"""
|
||||
await self.peer_mgr.add_localRPC_peer(real_name)
|
||||
await self._notify_peer(real_name)
|
||||
return f"peer '{real_name}' added"
|
||||
|
||||
async def rpc_disconnect(self, session_ids):
|
||||
@@ -487,7 +485,7 @@ class SessionManager:
|
||||
|
||||
async def rpc_peers(self):
|
||||
"""Return a list of data about server peers."""
|
||||
return self.peer_mgr.rpc_data()
|
||||
return self.env.peer_hubs
|
||||
|
||||
async def rpc_query(self, items, limit):
|
||||
"""Return a list of data about server peers."""
|
||||
@@ -578,7 +576,6 @@ class SessionManager:
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
self.peer_mgr.discover_peers(),
|
||||
self._clear_stale_sessions(),
|
||||
self._log_sessions(),
|
||||
self._manage_servers()
|
||||
@@ -638,6 +635,15 @@ class SessionManager:
|
||||
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
|
||||
return self.history_cache[hashX]
|
||||
|
||||
def _notify_peer(self, peer):
|
||||
notify_tasks = [
|
||||
session.send_notification('blockchain.peers.subscribe', [peer])
|
||||
for session in self.sessions.values() if session.subscribe_peers
|
||||
]
|
||||
if notify_tasks:
|
||||
self.logger.info(f'notify {len(notify_tasks)} sessions of new peers')
|
||||
asyncio.create_task(asyncio.wait(notify_tasks))
|
||||
|
||||
async def _notify_sessions(self, height, touched, new_touched):
|
||||
"""Notify sessions about height changes and touched addresses."""
|
||||
height_changed = height != self.notified_height
|
||||
@@ -704,7 +710,7 @@ class SessionBase(RPCSession):
|
||||
request_handlers: typing.Dict[str, typing.Callable] = {}
|
||||
version = '0.5.7'
|
||||
|
||||
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
|
||||
def __init__(self, session_mgr, db, mempool, kind):
|
||||
connection = JSONRPCConnection(JSONRPCAutoDetect)
|
||||
self.env = session_mgr.env
|
||||
super().__init__(connection=connection)
|
||||
@@ -712,7 +718,6 @@ class SessionBase(RPCSession):
|
||||
self.session_mgr = session_mgr
|
||||
self.db = db
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = peer_mgr
|
||||
self.kind = kind # 'RPC', 'TCP' etc.
|
||||
self.coin = self.env.coin
|
||||
self.anon_logs = self.env.anon_logs
|
||||
@@ -904,6 +909,7 @@ class LBRYElectrumX(SessionBase):
|
||||
LBRYElectrumX.set_server_features(self.env)
|
||||
self.subscribe_headers = False
|
||||
self.subscribe_headers_raw = False
|
||||
self.subscribe_peers = False
|
||||
self.connection.max_response_size = self.env.max_send
|
||||
self.hashX_subs = {}
|
||||
self.sv_seen = False
|
||||
@@ -1084,7 +1090,8 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def peers_subscribe(self):
|
||||
"""Return the server peers as a list of (ip, host, details) tuples."""
|
||||
return self.peer_mgr.on_peers_subscribe(self.is_tor())
|
||||
self.subscribe_peers = True
|
||||
return self.env.peer_hubs
|
||||
|
||||
async def address_status(self, hashX):
|
||||
"""Returns an address status.
|
||||
@@ -1327,11 +1334,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def banner(self):
|
||||
"""Return the server banner text."""
|
||||
banner = f'You are connected to an {self.version} server.'
|
||||
|
||||
if self.is_tor():
|
||||
banner_file = self.env.tor_banner_file
|
||||
else:
|
||||
banner_file = self.env.banner_file
|
||||
banner_file = self.env.banner_file
|
||||
if banner_file:
|
||||
try:
|
||||
with codecs.open(banner_file, 'r', 'utf-8') as f:
|
||||
|
||||
+46
-18
@@ -4,6 +4,7 @@ from time import perf_counter
|
||||
import logging
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
from lbry.utils import LRUCache, is_valid_public_ipv4
|
||||
from lbry.schema.attrs import country_str_to_int, country_int_to_str
|
||||
# from prometheus_client import Counter
|
||||
|
||||
|
||||
@@ -13,6 +14,9 @@ _MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
|
||||
_PAD_BYTES = b'\x00' * 64
|
||||
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
|
||||
class SPVPing(NamedTuple):
|
||||
magic: int
|
||||
protocol_version: int
|
||||
@@ -22,8 +26,8 @@ class SPVPing(NamedTuple):
|
||||
return struct.pack(b'!lB64s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(protocol_version=1) -> bytes:
|
||||
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
|
||||
def make() -> bytes:
|
||||
return SPVPing(_MAGIC, PROTOCOL_VERSION, _PAD_BYTES).encode()
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
@@ -33,24 +37,40 @@ class SPVPing(NamedTuple):
|
||||
return decoded
|
||||
|
||||
|
||||
PONG_ENCODING = b'!BBL32s4sH'
|
||||
|
||||
|
||||
class SPVPong(NamedTuple):
|
||||
protocol_version: int
|
||||
flags: int
|
||||
height: int
|
||||
tip: bytes
|
||||
source_address_raw: bytes
|
||||
country: int
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!BBl32s4s', *self)
|
||||
return struct.pack(PONG_ENCODING, *self)
|
||||
|
||||
@staticmethod
|
||||
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
|
||||
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
|
||||
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
|
||||
def encode_address(address: str):
|
||||
return bytes(int(b) for b in address.split("."))
|
||||
|
||||
@classmethod
|
||||
def make(cls, flags: int, height: int, tip: bytes, source_address: str, country: str) -> bytes:
|
||||
return SPVPong(
|
||||
PROTOCOL_VERSION, flags, height, tip,
|
||||
cls.encode_address(source_address),
|
||||
country_str_to_int(country)
|
||||
).encode()
|
||||
|
||||
@classmethod
|
||||
def make_sans_source_address(cls, flags: int, height: int, tip: bytes, country: str) -> Tuple[bytes, bytes]:
|
||||
pong = cls.make(flags, height, tip, '0.0.0.0', country)
|
||||
return pong[:38], pong[42:]
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
|
||||
return cls(*struct.unpack(PONG_ENCODING, packet[:44]))
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
@@ -60,23 +80,30 @@ class SPVPong(NamedTuple):
|
||||
def ip_address(self) -> str:
|
||||
return ".".join(map(str, self.source_address_raw))
|
||||
|
||||
@property
|
||||
def country_name(self):
|
||||
return country_int_to_str(self.country)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
|
||||
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()})"
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()}, country={self.country_name})"
|
||||
|
||||
|
||||
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10,
|
||||
allow_localhost: bool = False, allow_lan: bool = False):
|
||||
def __init__(
|
||||
self, height: int, tip: bytes, country: str,
|
||||
throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10,
|
||||
allow_localhost: bool = False, allow_lan: bool = False
|
||||
):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self._height = height
|
||||
self._tip = tip
|
||||
self._flags = 0
|
||||
self._cached_response = None
|
||||
self._country = country
|
||||
self._left_cache = self._right_cache = None
|
||||
self.update_cached_response()
|
||||
self._throttle = LRUCache(throttle_cache_size)
|
||||
self._should_log = LRUCache(throttle_cache_size)
|
||||
@@ -85,7 +112,9 @@ class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
self._allow_lan = allow_lan
|
||||
|
||||
def update_cached_response(self):
|
||||
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
|
||||
self._left_cache, self._right_cache = SPVPong.make_sans_source_address(
|
||||
self._flags, max(0, self._height), self._tip, self._country
|
||||
)
|
||||
|
||||
def set_unavailable(self):
|
||||
self._flags &= 0b11111110
|
||||
@@ -112,7 +141,7 @@ class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
return False
|
||||
|
||||
def make_pong(self, host):
|
||||
return self._cached_response + bytes(int(b) for b in host.split("."))
|
||||
return self._left_cache + SPVPong.encode_address(host) + self._right_cache
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
if self.should_throttle(addr[0]):
|
||||
@@ -144,13 +173,13 @@ class StatusServer:
|
||||
def __init__(self):
|
||||
self._protocol: Optional[SPVServerStatusProtocol] = None
|
||||
|
||||
async def start(self, height: int, tip: bytes, interface: str, port: int, allow_lan: bool = False):
|
||||
async def start(self, height: int, tip: bytes, country: str, interface: str, port: int, allow_lan: bool = False):
|
||||
if self.is_running:
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
|
||||
self._protocol = SPVServerStatusProtocol(
|
||||
height, tip, allow_localhost=interface == '127.0.0.1', allow_lan=allow_lan
|
||||
height, tip, country, allow_localhost=interface == '127.0.0.1', allow_lan=allow_lan
|
||||
)
|
||||
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
|
||||
log.info("started udp status server on %s:%i", interface, port)
|
||||
@@ -178,13 +207,12 @@ class StatusServer:
|
||||
|
||||
|
||||
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, responses: asyncio.Queue):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self.responses = responses
|
||||
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
|
||||
self._ping_packet = SPVPing.make()
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
try:
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# need this to avoid circular import
|
||||
PROTOCOL_MIN = (0, 54, 0)
|
||||
PROTOCOL_MAX = (0, 99, 0)
|
||||
PROTOCOL_MAX = (0, 199, 0)
|
||||
@@ -1401,9 +1401,6 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
|
||||
|
||||
async def test_repost(self):
|
||||
sql = self.conductor.spv_node.server.bp.sql
|
||||
sql.execute(sql.TAG_INDEXES)
|
||||
|
||||
await self.channel_create('@goodies', '1.0')
|
||||
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies', tags=['foo', 'gaming'])
|
||||
claim_id = self.get_claim_id(tx)
|
||||
@@ -1425,7 +1422,6 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['foo'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['foo'], claim_type=['stream', 'repost']), 0)
|
||||
# "common" / indexed tags work too
|
||||
self.assertIn('gaming', sql.TAG_INDEXES) # if this breaks, next test doesn't make sense
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['gaming'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['gaming'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['gaming'], claim_type=['stream', 'repost']), 0)
|
||||
|
||||
@@ -9,6 +9,7 @@ from lbry.wallet.orchstr8.node import SPVNode
|
||||
from lbry.wallet.rpc import RPCSession
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
from lbry.conf import Config
|
||||
|
||||
|
||||
class NetworkTests(IntegrationTestCase):
|
||||
@@ -24,7 +25,7 @@ class NetworkTests(IntegrationTestCase):
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '0.99.0',
|
||||
'protocol_max': '0.199.0',
|
||||
'protocol_min': '0.54.0',
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
@@ -49,7 +50,7 @@ class NetworkTests(IntegrationTestCase):
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '0.99.0',
|
||||
'protocol_max': '0.199.0',
|
||||
'protocol_min': '0.54.0',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
@@ -68,8 +69,8 @@ class ReconnectTests(IntegrationTestCase):
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
await node2.start(self.blockchain)
|
||||
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
self.ledger.network.config['default_servers'].reverse()
|
||||
self.ledger.network.config['explicit_servers'].append((node2.hostname, node2.port))
|
||||
self.ledger.network.config['explicit_servers'].reverse()
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
@@ -138,6 +139,19 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_timeout_and_concurrency_propagated_from_config(self):
|
||||
conf = Config()
|
||||
self.assertEqual(self.ledger.network.client.timeout, 30)
|
||||
self.assertEqual(self.ledger.network.client.concurrency, 32)
|
||||
conf.hub_timeout = 123.0
|
||||
conf.concurrent_hub_requests = 42
|
||||
conf.known_hubs = self.ledger.config['known_hubs']
|
||||
conf.wallet_dir = self.ledger.config['data_path']
|
||||
self.manager.config = conf
|
||||
await self.manager.reset()
|
||||
self.assertEqual(self.ledger.network.client.timeout, 123)
|
||||
self.assertEqual(self.ledger.network.client.concurrency, 42)
|
||||
|
||||
# async def test_online_but_still_unavailable(self):
|
||||
# # Edge case. See issue #2445 for context
|
||||
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
@@ -113,3 +113,79 @@ class TestESSync(CommandTestCase):
|
||||
self.assertTrue(await make_es_index(db.search_index))
|
||||
await db.search_index.start()
|
||||
await resync()
|
||||
|
||||
|
||||
class TestHubDiscovery(CommandTestCase):
|
||||
|
||||
async def test_hub_discovery(self):
|
||||
us_final_node = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
await us_final_node.start(self.blockchain, extraconf={"COUNTRY": "US"})
|
||||
self.addCleanup(us_final_node.stop)
|
||||
final_node_host = f"{us_final_node.hostname}:{us_final_node.port}"
|
||||
|
||||
kp_final_node = SPVNode(self.conductor.spv_module, node_number=3)
|
||||
await kp_final_node.start(self.blockchain, extraconf={"COUNTRY": "KP"})
|
||||
self.addCleanup(kp_final_node.stop)
|
||||
kp_final_node_host = f"{kp_final_node.hostname}:{kp_final_node.port}"
|
||||
|
||||
relay_node = SPVNode(self.conductor.spv_module, node_number=4)
|
||||
await relay_node.start(self.blockchain, extraconf={
|
||||
"COUNTRY": "FR",
|
||||
"PEER_HUBS": ",".join([kp_final_node_host, final_node_host])
|
||||
})
|
||||
relay_node_host = f"{relay_node.hostname}:{relay_node.port}"
|
||||
self.addCleanup(relay_node.stop)
|
||||
|
||||
self.assertEqual(list(self.daemon.conf.known_hubs), [])
|
||||
self.assertEqual(
|
||||
self.daemon.ledger.network.client.server_address_and_port,
|
||||
('127.0.0.1', 50002)
|
||||
)
|
||||
|
||||
# connect to relay hub which will tell us about the final hubs
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [relay_node_host])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
self.assertEqual(
|
||||
self.daemon.conf.known_hubs.filter(), {
|
||||
(relay_node.hostname, relay_node.port): {"country": "FR"},
|
||||
(us_final_node.hostname, us_final_node.port): {}, # discovered from relay but not contacted yet
|
||||
(kp_final_node.hostname, kp_final_node.port): {}, # discovered from relay but not contacted yet
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', relay_node.port)
|
||||
)
|
||||
|
||||
# use known_hubs to connect to final US hub
|
||||
self.daemon.jsonrpc_settings_clear('lbryum_servers')
|
||||
self.daemon.conf.jurisdiction = "US"
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
self.assertEqual(
|
||||
self.daemon.conf.known_hubs.filter(), {
|
||||
(relay_node.hostname, relay_node.port): {"country": "FR"},
|
||||
(us_final_node.hostname, us_final_node.port): {"country": "US"},
|
||||
(kp_final_node.hostname, kp_final_node.port): {"country": "KP"},
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', us_final_node.port)
|
||||
)
|
||||
|
||||
# connection to KP jurisdiction
|
||||
self.daemon.conf.jurisdiction = "KP"
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
self.assertEqual(
|
||||
self.daemon.ledger.network.client.server_address_and_port, ('127.0.0.1', kp_final_node.port)
|
||||
)
|
||||
|
||||
kp_final_node.server.session_mgr._notify_peer('127.0.0.1:9988')
|
||||
await self.daemon.ledger.network.on_hub.first
|
||||
await asyncio.sleep(0.5) # wait for above event to be processed by other listeners
|
||||
self.assertEqual(
|
||||
self.daemon.conf.known_hubs.filter(), {
|
||||
(relay_node.hostname, relay_node.port): {"country": "FR"},
|
||||
(us_final_node.hostname, us_final_node.port): {"country": "US"},
|
||||
(kp_final_node.hostname, kp_final_node.port): {"country": "KP"},
|
||||
('127.0.0.1', 9988): {}
|
||||
}
|
||||
)
|
||||
@@ -24,10 +24,10 @@ class TestExchangeRateManager(AsyncioTestCase):
|
||||
self.assertLessEqual(len(failures), 1, f"feed failures: {failures}. Please check exchange rate feeds!")
|
||||
lbc = manager.convert_currency('USD', 'LBC', Decimal('1.0'))
|
||||
self.assertGreaterEqual(lbc, 2.0)
|
||||
self.assertLessEqual(lbc, 15.0)
|
||||
self.assertLessEqual(lbc, 60.0)
|
||||
lbc = manager.convert_currency('BTC', 'LBC', Decimal('0.01'))
|
||||
self.assertGreaterEqual(lbc, 1_000)
|
||||
self.assertLessEqual(lbc, 4_000)
|
||||
self.assertLessEqual(lbc, 20_000)
|
||||
|
||||
async def test_it_handles_feed_being_offline(self):
|
||||
class FakeFeed(MarketFeed):
|
||||
|
||||
@@ -15,7 +15,7 @@ class AddressManagement(CommandTestCase):
|
||||
class SettingsManagement(CommandTestCase):
|
||||
|
||||
async def test_settings(self):
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('127.0.0.1', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('localhost', 50002))
|
||||
|
||||
setting = self.daemon.jsonrpc_settings_set('lbryum_servers', ['server:50001'])
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('server', 50001))
|
||||
|
||||
@@ -9,7 +9,6 @@ from decimal import Decimal
|
||||
from lbry.file.file_manager import FileManager
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from lbry.testcase import get_fake_exchange_rate_manager
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import InsufficientFundsError
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadSDTimeoutError, DownloadDataTimeoutError
|
||||
from lbry.wallet import WalletManager, Wallet, Ledger, Transaction, Input, Output, Database
|
||||
@@ -140,11 +139,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.sd_hash = descriptor.sd_hash
|
||||
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, self.client_wallet_dir,
|
||||
balance, fee)
|
||||
analytics_manager = AnalyticsManager(
|
||||
self.client_config,
|
||||
binascii.hexlify(generate_id()).decode(),
|
||||
binascii.hexlify(generate_id()).decode()
|
||||
)
|
||||
analytics_manager = AnalyticsManager(self.client_config)
|
||||
self.stream_manager = StreamManager(
|
||||
self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client),
|
||||
|
||||
@@ -47,6 +47,26 @@ class ConfigurationTests(unittest.TestCase):
|
||||
c.persisted = {}
|
||||
self.assertEqual(c.test_str, 'the default')
|
||||
|
||||
def test_is_set(self):
|
||||
c = TestConfig()
|
||||
self.assertEqual(c.test_str, 'the default')
|
||||
self.assertFalse(TestConfig.test_str.is_set(c))
|
||||
c.test_str = 'new value'
|
||||
self.assertEqual(c.test_str, 'new value')
|
||||
self.assertTrue(TestConfig.test_str.is_set(c))
|
||||
|
||||
def test_is_set_to_default(self):
|
||||
c = TestConfig()
|
||||
self.assertEqual(TestConfig.test_str.default, 'the default')
|
||||
self.assertFalse(TestConfig.test_str.is_set(c))
|
||||
self.assertFalse(TestConfig.test_str.is_set_to_default(c))
|
||||
c.test_str = 'new value'
|
||||
self.assertTrue(TestConfig.test_str.is_set(c))
|
||||
self.assertFalse(TestConfig.test_str.is_set_to_default(c))
|
||||
c.test_str = 'the default'
|
||||
self.assertTrue(TestConfig.test_str.is_set(c))
|
||||
self.assertTrue(TestConfig.test_str.is_set_to_default(c))
|
||||
|
||||
def test_arguments(self):
|
||||
parser = argparse.ArgumentParser()
|
||||
TestConfig.contribute_to_argparse(parser)
|
||||
@@ -255,3 +275,38 @@ class ConfigurationTests(unittest.TestCase):
|
||||
args = parser.parse_args(['--string-choice', 'c'])
|
||||
c = TestConfig.create_from_arguments(args)
|
||||
self.assertEqual("c", c.string_choice)
|
||||
|
||||
def test_known_hubs_list(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
hubs = Config(config=os.path.join(temp_dir, 'settings.yml'), wallet_dir=temp_dir).known_hubs
|
||||
|
||||
self.assertEqual(hubs.serialized, {})
|
||||
self.assertEqual(list(hubs), [])
|
||||
self.assertFalse(hubs)
|
||||
hubs.set('new.hub.io:99', {'jurisdiction': 'us'})
|
||||
self.assertTrue(hubs)
|
||||
|
||||
self.assertFalse(hubs.exists)
|
||||
hubs.save()
|
||||
self.assertTrue(hubs.exists)
|
||||
|
||||
hubs = Config(config=os.path.join(temp_dir, 'settings.yml'), wallet_dir=temp_dir).known_hubs
|
||||
self.assertEqual(list(hubs), [('new.hub.io', 99)])
|
||||
self.assertEqual(hubs.serialized, {'new.hub.io:99': {'jurisdiction': 'us'}})
|
||||
|
||||
hubs.set('any.hub.io:99', {})
|
||||
hubs.set('oth.hub.io:99', {'jurisdiction': 'other'})
|
||||
self.assertEqual(list(hubs), [('new.hub.io', 99), ('any.hub.io', 99), ('oth.hub.io', 99)])
|
||||
self.assertEqual(hubs.filter(), {
|
||||
('new.hub.io', 99): {'jurisdiction': 'us'},
|
||||
('oth.hub.io', 99): {'jurisdiction': 'other'},
|
||||
('any.hub.io', 99): {}
|
||||
})
|
||||
self.assertEqual(hubs.filter(foo="bar"), {})
|
||||
self.assertEqual(hubs.filter(jurisdiction="us"), {
|
||||
('new.hub.io', 99): {'jurisdiction': 'us'}
|
||||
})
|
||||
self.assertEqual(hubs.filter(jurisdiction="us", match_none=True), {
|
||||
('new.hub.io', 99): {'jurisdiction': 'us'},
|
||||
('any.hub.io', 99): {}
|
||||
})
|
||||
@@ -1,6 +1,6 @@
|
||||
import time
|
||||
import struct
|
||||
import apsw
|
||||
import sqlite3
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from typing import Tuple, List, Dict, Union, Type, Optional
|
||||
@@ -21,19 +21,20 @@ from lbry.wallet import Ledger, RegTestLedger
|
||||
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
|
||||
|
||||
class SQLiteOperationalError(apsw.Error):
|
||||
class SQLiteOperationalError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query errored')
|
||||
self.metrics = metrics
|
||||
|
||||
|
||||
class SQLiteInterruptedError(apsw.InterruptError):
|
||||
class SQLiteInterruptedError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query interrupted')
|
||||
self.metrics = metrics
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
@@ -63,7 +64,7 @@ ORDER_FIELDS = {
|
||||
|
||||
@dataclass
|
||||
class ReaderState:
|
||||
db: apsw.Connection
|
||||
db: sqlite3.Connection
|
||||
stack: List[List]
|
||||
metrics: Dict
|
||||
is_tracking_metrics: bool
|
||||
@@ -90,7 +91,7 @@ class ReaderState:
|
||||
self.db.interrupt()
|
||||
return
|
||||
|
||||
self.db.setprogresshandler(interruptor, 100)
|
||||
self.db.set_progress_handler(interruptor, 100)
|
||||
|
||||
def get_resolve_censor(self) -> Censor:
|
||||
return Censor(Censor.RESOLVE)
|
||||
@@ -105,13 +106,13 @@ ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
def row_factory(cursor, row):
|
||||
return {
|
||||
k[0]: (set(row[i].split(',')) if k[0] == 'tags' else row[i])
|
||||
for i, k in enumerate(cursor.getdescription())
|
||||
for i, k in enumerate(cursor.description)
|
||||
}
|
||||
|
||||
|
||||
def initializer(log, _path, _ledger_name, query_timeout, _measure=False, block_and_filter=None):
|
||||
db = apsw.Connection(_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
|
||||
db.setrowtrace(row_factory)
|
||||
db = sqlite3.connect(_path, isolation_level=None, uri=True)
|
||||
db.row_factory = row_factory
|
||||
if block_and_filter:
|
||||
blocked_streams, blocked_channels, filtered_streams, filtered_channels = block_and_filter
|
||||
else:
|
||||
@@ -184,31 +185,12 @@ def execute_query(sql, values, row_offset: int, row_limit: int, censor: Censor)
|
||||
context = ctx.get()
|
||||
context.set_query_timeout()
|
||||
try:
|
||||
c = context.db.cursor()
|
||||
def row_filter(cursor, row):
|
||||
nonlocal row_offset
|
||||
row = row_factory(cursor, row)
|
||||
if len(row) > 1 and censor.censor(row):
|
||||
return
|
||||
if row_offset:
|
||||
row_offset -= 1
|
||||
return
|
||||
return row
|
||||
c.setrowtrace(row_filter)
|
||||
i, rows = 0, []
|
||||
for row in c.execute(sql, values):
|
||||
i += 1
|
||||
rows.append(row)
|
||||
if i >= row_limit:
|
||||
break
|
||||
return rows
|
||||
except apsw.Error as err:
|
||||
rows = context.db.execute(sql, values).fetchall()
|
||||
return rows[row_offset:row_limit]
|
||||
except sqlite3.OperationalError as err:
|
||||
plain_sql = interpolate(sql, values)
|
||||
if context.is_tracking_metrics:
|
||||
context.metrics['execute_query'][-1]['sql'] = plain_sql
|
||||
if isinstance(err, apsw.InterruptError):
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
|
||||
raise SQLiteInterruptedError(context.metrics)
|
||||
context.log.exception('failed running query', exc_info=err)
|
||||
raise SQLiteOperationalError(context.metrics)
|
||||
|
||||
|
||||
Reference in new issue
Block a user