Compare commits
71
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abfe0be81e | ||
|
|
820b7a6753 | ||
|
|
8bb6f328dc | ||
|
|
7f03b13579 | ||
|
|
09008ea991 | ||
|
|
ee234212e6 | ||
|
|
277b243f52 | ||
|
|
0973ac753f | ||
|
|
f41cfbfb97 | ||
|
|
a70980c81a | ||
|
|
12a962f656 | ||
|
|
5936444f3e | ||
|
|
c7d42f00c6 | ||
|
|
b1318a9958 | ||
|
|
d2560d260c | ||
|
|
f40a61cf9a | ||
|
|
e0623578bf | ||
|
|
ead8daaa14 | ||
|
|
23b4b9e230 | ||
|
|
947017e334 | ||
|
|
7fd0d6507f | ||
|
|
ad1e9ef086 | ||
|
|
2f1d08e417 | ||
|
|
55f4eb80ba | ||
|
|
7c160ff65e | ||
|
|
fc5d5faaed | ||
|
|
5394f1763c | ||
|
|
050b67c9d6 | ||
|
|
4b87cb45ee | ||
|
|
f0e1db319c | ||
|
|
3950715237 | ||
|
|
ae9ba14b59 | ||
|
|
ad6c6fbe35 | ||
|
|
ffa5c20c88 | ||
|
|
5a6218eeca | ||
|
|
200531dd96 | ||
|
|
15a2f048ac | ||
|
|
d317a4042c | ||
|
|
c2d717aba5 | ||
|
|
ef74777df1 | ||
|
|
f6dbf99fb5 | ||
|
|
0b2d9e15b4 | ||
|
|
d14e5e75e8 | ||
|
|
df11ef34d2 | ||
|
|
4fc619262c | ||
|
|
2750eee86d | ||
|
|
71cb03345b | ||
|
|
22c8ca6c40 | ||
|
|
237a8965eb | ||
|
|
7d872c7863 | ||
|
|
0d08858dfb | ||
|
|
5c6d63c93d | ||
|
|
c8c10d2bb0 | ||
|
|
f4d3e9ea6a | ||
|
|
3713d3488d | ||
|
|
9b46d03c91 | ||
|
|
fe4e738580 | ||
|
|
205b0c4263 | ||
|
|
78eef25f1a | ||
|
|
c9305fd070 | ||
|
|
20774280b9 | ||
|
|
74bd2557af | ||
|
|
605fc8ecd8 | ||
|
|
6b745c53dc | ||
|
|
98c60b38fc | ||
|
|
be4515ec9a | ||
|
|
dcb1b64696 | ||
|
|
1190a60c7a | ||
|
|
82ecf207c6 | ||
|
|
84c7f1bf58 | ||
|
|
5ac453fe07 |
No files matched your search
+3
-2
@@ -84,14 +84,15 @@ test:json-api:
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.59.0"
|
||||
__version__ = "0.60.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+11
-10
@@ -558,6 +558,7 @@ class Config(CLIConfig):
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
@@ -573,15 +574,15 @@ class Config(CLIConfig):
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv1.lbry.com', 50001),
|
||||
('spv2.lbry.com', 50001),
|
||||
('spv3.lbry.com', 50001),
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
('spv15.lbry.com', 50001),
|
||||
('spv16.lbry.com', 50001),
|
||||
('spv17.lbry.com', 50001),
|
||||
('spv18.lbry.com', 50001),
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
@@ -601,7 +602,7 @@ class Config(CLIConfig):
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
|
||||
@@ -57,6 +57,9 @@ Code | Name | Message
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
@@ -242,6 +242,27 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
self.address = address
|
||||
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentWalletLockedError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, daily_fee, max_fee):
|
||||
self.daily_fee = daily_fee
|
||||
self.max_fee = max_fee
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
|
||||
+3
-4
@@ -256,10 +256,9 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
loggly_handler = get_loggly_handler(conf)
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
|
||||
@@ -110,7 +110,6 @@ class AnalyticsManager:
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
@@ -118,20 +117,24 @@ class AnalyticsManager:
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.conf.share_usage_data
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
if self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
@@ -154,7 +157,7 @@ class AnalyticsManager:
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
if self.enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,6 +30,7 @@ log = logging.getLogger(__name__)
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
@@ -160,6 +162,34 @@ class WalletComponent(Component):
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class WalletServerPaymentsComponent(Component):
|
||||
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
depends_on = [WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.usage_payment_service = WalletServerPayer(
|
||||
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
|
||||
)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[WalletServerPayer]:
|
||||
return self.usage_payment_service
|
||||
|
||||
async def start(self):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
|
||||
|
||||
async def stop(self):
|
||||
await self.usage_payment_service.stop()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'max_fee': self.usage_payment_service.max_fee,
|
||||
'running': self.usage_payment_service.running
|
||||
}
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
@@ -297,6 +297,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
|
||||
self.conf = conf
|
||||
self.platform_info = system_info.get_platform()
|
||||
self._video_file_analyzer = VideoFileAnalyzer(conf)
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
@@ -307,7 +308,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
skip_components=conf.components_to_skip or []
|
||||
)
|
||||
self.component_startup_task = None
|
||||
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
|
||||
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
|
||||
|
||||
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
|
||||
rpc_app = web.Application()
|
||||
@@ -357,7 +358,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@classmethod
|
||||
def get_api_definitions(cls):
|
||||
prefix = 'jsonrpc_'
|
||||
not_grouped = ['routing_table_get']
|
||||
not_grouped = ['routing_table_get', 'ffmpeg_find']
|
||||
api = {
|
||||
'groups': {
|
||||
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
|
||||
@@ -449,7 +450,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def start(self):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
|
||||
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
|
||||
await self.analytics_manager.send_server_startup()
|
||||
await self.rpc_runner.setup()
|
||||
await self.streaming_runner.setup()
|
||||
@@ -764,6 +765,26 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
asyncio.get_event_loop().call_later(0, shutdown)
|
||||
return "Shutting down"
|
||||
|
||||
async def jsonrpc_ffmpeg_find(self):
|
||||
"""
|
||||
Get ffmpeg installation information
|
||||
|
||||
Usage:
|
||||
ffmpeg_find
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of ffmpeg information
|
||||
{
|
||||
'available': (bool) found ffmpeg,
|
||||
'which': (str) path to ffmpeg,
|
||||
'analyze_audio_volume': (bool) should ffmpeg analyze audio
|
||||
}
|
||||
"""
|
||||
return await self._video_file_analyzer.status(recheck=True)
|
||||
|
||||
async def jsonrpc_status(self):
|
||||
"""
|
||||
Get daemon status
|
||||
@@ -894,9 +915,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'build': (str) "dev" | "qa" | "rc" | "release",
|
||||
}
|
||||
"""
|
||||
platform_info = system_info.get_platform()
|
||||
log.info("Get version info: %s", json.dumps(platform_info))
|
||||
return platform_info
|
||||
return self.platform_info
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
|
||||
|
||||
@@ -294,7 +294,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
|
||||
@@ -3,10 +3,12 @@ import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
import typing
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
@@ -36,17 +38,19 @@ class JsonFormatter(logging.Formatter):
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
def __init__(self, loggly_token: str, config: 'Config'):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.cookies = {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._config = config
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._config.share_usage_data
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
@@ -62,12 +66,14 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry:
|
||||
if self._loop.is_running() and retry and self.enabled:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
@@ -83,7 +89,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler():
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
|
||||
def get_loggly_handler(config):
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
|
||||
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee):
|
||||
raw_content_fee, fully_reflected):
|
||||
return {
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
@@ -129,7 +129,8 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
),
|
||||
"fully_reflected": fully_reflected
|
||||
}
|
||||
|
||||
|
||||
@@ -138,11 +139,14 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
|
||||
"select file.rowid, file.*, stream.*, c.* "
|
||||
"select file.rowid, file.*, stream.*, c.*, "
|
||||
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
|
||||
" is null then 0 else 1 end as fully_reflected "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"order by c.rowid desc").fetchall():
|
||||
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -152,7 +156,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
_get_lbry_file_stream_dict(
|
||||
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee
|
||||
raw_content_fee, fully_reflected
|
||||
)
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
|
||||
+11
-8
@@ -19,6 +19,7 @@ class VideoFileAnalyzer:
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._checked_ffmpeg = False
|
||||
|
||||
async def _execute(self, command, arguments):
|
||||
args = shlex.split(arguments)
|
||||
@@ -48,18 +49,20 @@ class VideoFileAnalyzer:
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
async def status(self, reset=False):
|
||||
async def status(self, reset=False, recheck=False):
|
||||
if reset:
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
|
||||
installed = True
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
installed = False
|
||||
|
||||
if self._checked_ffmpeg and not recheck:
|
||||
installed = self._ffmpeg_installed
|
||||
else:
|
||||
installed = True
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
installed = False
|
||||
self._checked_ffmpeg = True
|
||||
return {
|
||||
"available": installed,
|
||||
"which": self._which,
|
||||
|
||||
@@ -111,6 +111,10 @@ class ManagedStream:
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
return self.fully_reflected.is_set()
|
||||
|
||||
@property
|
||||
def descriptor(self) -> StreamDescriptor:
|
||||
return self.downloader.descriptor
|
||||
@@ -429,9 +433,9 @@ class ManagedStream:
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
if sent_sd: # reflector needed the sd blob
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed:
|
||||
if not sent_sd and not needed: # reflector already has the stream
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
|
||||
@@ -116,7 +116,7 @@ class StreamManager:
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
added_on: Optional[int], fully_reflected: bool):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -127,6 +127,8 @@ class StreamManager:
|
||||
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
|
||||
analytics_manager=self.analytics_manager, added_on=added_on
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
@@ -159,7 +161,7 @@ class StreamManager:
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
file_info['added_on']
|
||||
file_info['added_on'], file_info['fully_reflected']
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
|
||||
@@ -253,6 +253,11 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
lambda e: e.tx.id == txid
|
||||
)
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.address == address
|
||||
)
|
||||
|
||||
def on_transaction_address(self, tx, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.tx.id == tx.id and e.address == address
|
||||
|
||||
+11
-7
@@ -134,16 +134,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
col, op = col[:-len('__not_like')], 'NOT LIKE'
|
||||
elif key.endswith('__in') or key.endswith('__not_in'):
|
||||
if key.endswith('__in'):
|
||||
col, op = col[:-len('__in')], 'IN'
|
||||
col, op, one_val_op = col[:-len('__in')], 'IN', '='
|
||||
else:
|
||||
col, op = col[:-len('__not_in')], 'NOT IN'
|
||||
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
|
||||
if constraint:
|
||||
if isinstance(constraint, (list, set, tuple)):
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
if len(constraint) == 1:
|
||||
values[f'{key}{tag}'] = next(iter(constraint))
|
||||
sql.append(f'{col} {one_val_op} :{key}{tag}')
|
||||
else:
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
elif isinstance(constraint, str):
|
||||
sql.append(f'{col} {op} ({constraint})')
|
||||
else:
|
||||
|
||||
@@ -630,7 +630,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
|
||||
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
|
||||
outputs = Outputs.from_base64(await query)
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs = []
|
||||
if len(outputs.txs) > 0:
|
||||
txs: List[Transaction] = await asyncio.gather(*(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
from time import perf_counter
|
||||
from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
@@ -68,7 +69,14 @@ class ClientSession(BaseClientSession):
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
return request.result()
|
||||
try:
|
||||
return request.result()
|
||||
except ConnectionResetError:
|
||||
log.error(
|
||||
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
|
||||
self.server[0], method, len(args), len(json.dumps(args))
|
||||
)
|
||||
raise
|
||||
except (RPCError, ProtocolError) as e:
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
|
||||
@@ -173,6 +173,9 @@ class SPVNode:
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
conf = {
|
||||
'DESCRIPTION': '',
|
||||
'PAYMENT_ADDRESS': '',
|
||||
'DAILY_FEE': '0',
|
||||
'DB_DIRECTORY': self.data_path,
|
||||
'DAEMON_URL': blockchain_node.rpc_url,
|
||||
'REORG_LIMIT': '100',
|
||||
|
||||
@@ -386,9 +386,11 @@ class RPCSession(SessionBase):
|
||||
while not self.is_closing():
|
||||
try:
|
||||
message = await self.framer.receive_message()
|
||||
except MemoryError as e:
|
||||
self.logger.warning(f'{e!r}')
|
||||
continue
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
self._close()
|
||||
return
|
||||
|
||||
self.last_recv = time.perf_counter()
|
||||
self.recv_count += 1
|
||||
|
||||
@@ -164,6 +164,7 @@ class BlockProcessor:
|
||||
# is consistent with self.height
|
||||
self.state_lock = asyncio.Lock()
|
||||
|
||||
self.search_cache = {}
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
# cancellations from shutdown don't lose work - when the task
|
||||
@@ -191,6 +192,8 @@ class BlockProcessor:
|
||||
if hprevs == chain:
|
||||
start = time.perf_counter()
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
@@ -729,7 +732,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
|
||||
self.sql: SQLDB = self.db.sql
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.search_cache = {}
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
self.sql.begin()
|
||||
@@ -744,8 +746,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
'repost': 3
|
||||
}
|
||||
|
||||
STREAM_TYPES = {
|
||||
|
||||
@@ -266,7 +266,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
else:
|
||||
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
|
||||
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
@@ -282,15 +282,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'channel_ids' in constraints:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = [
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
]
|
||||
}
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = [
|
||||
not_channel_ids_binary = {
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
]
|
||||
}
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
@@ -320,13 +320,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'stream_types' in constraints:
|
||||
stream_types = constraints.pop('stream_types')
|
||||
if stream_types:
|
||||
constraints['claim.stream_type__in'] = [
|
||||
constraints['claim.stream_type__in'] = {
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
]
|
||||
}
|
||||
if 'media_types' in constraints:
|
||||
media_types = constraints.pop('media_types')
|
||||
if media_types:
|
||||
constraints['claim.media_type__in'] = media_types
|
||||
constraints['claim.media_type__in'] = set(media_types)
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
@@ -418,8 +418,6 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
total = count_claims(**constraints)
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
context = ctx.get()
|
||||
search_censor = context.get_search_censor()
|
||||
txo_rows = search_claims(search_censor, **constraints)
|
||||
|
||||
@@ -94,6 +94,7 @@ class SQLDB:
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -129,35 +130,42 @@ class SQLDB:
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
-- naked order bys (no filters)
|
||||
create unique index if not exists claim_release_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
|
||||
|
||||
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
|
||||
-- claim_type filter + order by
|
||||
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
|
||||
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
|
||||
|
||||
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, effective_amount, claim_hash);
|
||||
-- stream_type filter + order by
|
||||
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
|
||||
create unique index if not exists stream_type_trending_idx on claim (stream_type, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists stream_type_effective_amount_idx on claim (stream_type, effective_amount, claim_hash);
|
||||
|
||||
create unique index if not exists filter_stream_duration_idx on claim (duration, trending_global, trending_mixed, claim_hash);
|
||||
-- channel_hash filter + order by
|
||||
create unique index if not exists channel_hash_release_idx on claim (channel_hash, release_time, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists channel_hash_effective_amount_idx on claim (channel_hash, effective_amount, claim_hash);
|
||||
|
||||
-- duration filter + order by
|
||||
create unique index if not exists duration_release_idx on claim (duration, release_time, claim_hash);
|
||||
create unique index if not exists duration_trending_idx on claim (duration, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists duration_effective_amount_idx on claim (duration, effective_amount, claim_hash);
|
||||
|
||||
-- fee_amount + order by
|
||||
create unique index if not exists fee_amount_release_idx on claim (fee_amount, release_time, claim_hash);
|
||||
create unique index if not exists fee_amount_trending_idx on claim (fee_amount, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists fee_amount_effective_amount_idx on claim (fee_amount, effective_amount, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
@@ -360,6 +368,7 @@ class SQLDB:
|
||||
if isinstance(fee.amount, Decimal):
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['repost']
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
@@ -72,16 +72,18 @@ class Env:
|
||||
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
|
||||
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
|
||||
# The electrum client takes the empty string as unspecified
|
||||
self.payment_address = self.default('PAYMENT_ADDRESS', '')
|
||||
self.donation_address = self.default('DONATION_ADDRESS', '')
|
||||
# Server limits to help prevent DoS
|
||||
self.max_send = self.integer('MAX_SEND', 1000000)
|
||||
self.max_receive = self.integer('MAX_RECEIVE', 1000000)
|
||||
self.max_subs = self.integer('MAX_SUBS', 250000)
|
||||
self.max_sessions = self.sane_max_sessions()
|
||||
self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
self.daily_fee = self.string_amount('DAILY_FEE', '0')
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
@@ -107,6 +109,14 @@ class Env:
|
||||
raise cls.Error(f'required envvar {envvar} not set')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def string_amount(cls, envvar, default):
|
||||
value = environ.get(envvar, default)
|
||||
amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
|
||||
if len(value) > 0 and not amount_pattern.fullmatch(value):
|
||||
raise cls.Error(f'{value} is not a valid amount for {envvar}')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def integer(cls, envvar, default):
|
||||
value = environ.get(envvar)
|
||||
|
||||
@@ -30,6 +30,7 @@ from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
|
||||
from lbry.wallet.rpc.framing import NewlineFramer
|
||||
import lbry.wallet.server.version as VERSION
|
||||
|
||||
from lbry.wallet.rpc import (
|
||||
@@ -621,6 +622,7 @@ class SessionBase(RPCSession):
|
||||
|
||||
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
|
||||
connection = JSONRPCConnection(JSONRPCAutoDetect)
|
||||
self.env = session_mgr.env
|
||||
super().__init__(connection=connection)
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.session_mgr = session_mgr
|
||||
@@ -628,7 +630,6 @@ class SessionBase(RPCSession):
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = peer_mgr
|
||||
self.kind = kind # 'RPC', 'TCP' etc.
|
||||
self.env = session_mgr.env
|
||||
self.coin = self.env.coin
|
||||
self.anon_logs = self.env.anon_logs
|
||||
self.txs_sent = 0
|
||||
@@ -641,6 +642,9 @@ class SessionBase(RPCSession):
|
||||
async def notify(self, touched, height_changed):
|
||||
pass
|
||||
|
||||
def default_framer(self):
|
||||
return NewlineFramer(self.env.max_receive)
|
||||
|
||||
def peer_address_str(self, *, for_log=True):
|
||||
"""Returns the peer's IP address and port as a human-readable
|
||||
string, respecting anon logs if the output is for a log."""
|
||||
@@ -800,6 +804,7 @@ class LBRYElectrumX(SessionBase):
|
||||
'blockchain.transaction.get_merkle': cls.transaction_merkle,
|
||||
'server.add_peer': cls.add_peer,
|
||||
'server.banner': cls.banner,
|
||||
'server.payment_address': cls.payment_address,
|
||||
'server.donation_address': cls.donation_address,
|
||||
'server.features': cls.server_features_async,
|
||||
'server.peers.subscribe': cls.peers_subscribe,
|
||||
@@ -854,7 +859,8 @@ class LBRYElectrumX(SessionBase):
|
||||
'protocol_max': max_str,
|
||||
'genesis_hash': env.coin.GENESIS_HASH,
|
||||
'description': env.description,
|
||||
'payment_address': env.donation_address,
|
||||
'payment_address': env.payment_address,
|
||||
'donation_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
}
|
||||
@@ -1362,11 +1368,16 @@ class LBRYElectrumX(SessionBase):
|
||||
('$SERVER_VERSION', self.version),
|
||||
('$DAEMON_VERSION', daemon_version),
|
||||
('$DAEMON_SUBVERSION', network_info['subversion']),
|
||||
('$PAYMENT_ADDRESS', self.env.payment_address),
|
||||
('$DONATION_ADDRESS', self.env.donation_address),
|
||||
]:
|
||||
banner = banner.replace(*pair)
|
||||
return banner
|
||||
|
||||
async def payment_address(self):
|
||||
"""Return the payment address as a string, empty if there is none."""
|
||||
return self.env.payment_address
|
||||
|
||||
async def donation_address(self):
|
||||
"""Return the donation address as a string, empty if there is none."""
|
||||
return self.env.donation_address
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from lbry.error import (
|
||||
ServerPaymentFeeAboveMaxAllowedError,
|
||||
ServerPaymentInvalidAddressError,
|
||||
ServerPaymentWalletLockedError
|
||||
)
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.transaction import Output, Transaction
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WalletServerPayer:
|
||||
def __init__(self, payment_period=24 * 60 * 60, max_fee='1.0', analytics_manager=None):
|
||||
self.ledger = None
|
||||
self.wallet = None
|
||||
self.running = False
|
||||
self.task = None
|
||||
self.payment_period = payment_period
|
||||
self.analytics_manager = analytics_manager
|
||||
self.max_fee = max_fee
|
||||
self._on_payment_controller = StreamController()
|
||||
self.on_payment = self._on_payment_controller.stream
|
||||
self.on_payment.listen(None, on_error=lambda e: logging.warning(e.args[0]))
|
||||
|
||||
async def pay(self):
|
||||
while self.running:
|
||||
await asyncio.sleep(self.payment_period)
|
||||
features = await self.ledger.network.retriable_call(self.ledger.network.get_server_features)
|
||||
address = features['payment_address']
|
||||
amount = str(features['daily_fee'])
|
||||
if not address or not amount:
|
||||
continue
|
||||
|
||||
if not self.ledger.is_valid_address(address):
|
||||
self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
|
||||
continue
|
||||
|
||||
if self.wallet.is_locked:
|
||||
self._on_payment_controller.add_error(ServerPaymentWalletLockedError())
|
||||
continue
|
||||
|
||||
amount = lbc_to_dewies(features['daily_fee']) # check that this is in lbc and not dewies
|
||||
limit = lbc_to_dewies(self.max_fee)
|
||||
if amount > limit:
|
||||
self._on_payment_controller.add_error(
|
||||
ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee)
|
||||
)
|
||||
continue
|
||||
|
||||
tx = await Transaction.create(
|
||||
[],
|
||||
[Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
|
||||
self.wallet.get_accounts_or_all(None),
|
||||
self.wallet.get_account_or_default(None)
|
||||
)
|
||||
|
||||
await self.ledger.broadcast(tx)
|
||||
if self.analytics_manager:
|
||||
await self.analytics_manager.send_credits_sent()
|
||||
self._on_payment_controller.add(tx)
|
||||
|
||||
async def start(self, ledger=None, wallet=None):
|
||||
if lbc_to_dewies(self.max_fee) < 1:
|
||||
return
|
||||
self.ledger = ledger
|
||||
self.wallet = wallet
|
||||
self.running = True
|
||||
self.task = asyncio.ensure_future(self.pay())
|
||||
self.task.add_done_callback(lambda _: log.info("Stopping wallet server payments."))
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.task.cancel()
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# usage: update_dev_wallet_server.sh <host to update>
|
||||
TARGET_HOST=$1
|
||||
|
||||
SCRIPTS_DIR=`dirname $0`
|
||||
LBRY_DIR=`dirname $SCRIPTS_DIR`
|
||||
|
||||
# build the image
|
||||
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
|
||||
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
|
||||
|
||||
# push the image to the server
|
||||
ssh $TARGET_HOST docker image prune --force
|
||||
docker save $IMAGE | ssh $TARGET_HOST docker load
|
||||
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
|
||||
|
||||
# restart the wallet server
|
||||
ssh $TARGET_HOST docker-compose down
|
||||
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
|
||||
@@ -12,7 +12,7 @@ async def main():
|
||||
try:
|
||||
hostname = sys.argv[1]
|
||||
except IndexError:
|
||||
hostname = 'spv1.lbry.com'
|
||||
hostname = 'spv11.lbry.com'
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
client = ClientSession(network=None, server=(hostname, 50001))
|
||||
|
||||
@@ -32,6 +32,7 @@ disable=
|
||||
too-many-branches,
|
||||
too-many-arguments,
|
||||
too-many-statements,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-instance-attributes,
|
||||
protected-access,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import os.path
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
from lbry.error import InsufficientFundsError
|
||||
|
||||
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
|
||||
@@ -73,6 +73,19 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
(result['txid'], result['claim_id'])
|
||||
)
|
||||
|
||||
async def test_disconnect_on_memory_error(self):
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
] * 23828
|
||||
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
|
||||
|
||||
# 23829 claim ids makes the request just large enough
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
] * 23829
|
||||
with self.assertRaises(ConnectionResetError):
|
||||
await self.claim_search(claim_ids=claim_ids)
|
||||
|
||||
async def test_basic_claim_search(self):
|
||||
await self.create_channel()
|
||||
channel_txo = self.channel['outputs'][0]
|
||||
@@ -309,12 +322,14 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
@@ -322,7 +337,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
|
||||
# stream_type
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
@@ -401,6 +416,8 @@ class ClaimCommands(ClaimTestCase):
|
||||
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
|
||||
|
||||
async def test_claim_stream_channel_list_with_resolve(self):
|
||||
self.assertListEqual([], await self.claim_list(resolve=True))
|
||||
|
||||
await self.channel_create()
|
||||
await self.stream_create()
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
|
||||
import lbry
|
||||
@@ -29,15 +28,20 @@ class NetworkTests(IntegrationTestCase):
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
'daily_fee': 0,
|
||||
'donation_address': '',
|
||||
'daily_fee': '0',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
address = (await self.account.get_addresses(limit=1))[0]
|
||||
os.environ.update({
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'DONATION_ADDRESS': address,
|
||||
'DAILY_FEE': '42'})
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node,
|
||||
extraconf={
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'PAYMENT_ADDRESS': payment_address,
|
||||
'DONATION_ADDRESS': donation_address,
|
||||
'DAILY_FEE': '42'
|
||||
}
|
||||
)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
@@ -47,8 +51,9 @@ class NetworkTests(IntegrationTestCase):
|
||||
'protocol_min': '0.54.0',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
'payment_address': address,
|
||||
'daily_fee': 42,
|
||||
'payment_address': payment_address,
|
||||
'donation_address': donation_address,
|
||||
'daily_fee': '42',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
|
||||
@@ -2,8 +2,10 @@ import asyncio
|
||||
|
||||
import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.testcase import IntegrationTestCase
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
|
||||
|
||||
class TestSessions(IntegrationTestCase):
|
||||
@@ -49,3 +51,46 @@ class TestSegwitServer(IntegrationTestCase):
|
||||
|
||||
async def test_at_least_it_starts(self):
|
||||
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
|
||||
|
||||
|
||||
class TestUsagePayment(CommandTestCase):
|
||||
async def test_single_server_payment(self):
|
||||
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
|
||||
wallet_pay_service.payment_period = 1
|
||||
# only starts with a positive max key fee
|
||||
wallet_pay_service.max_fee = "0.0"
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
self.assertFalse(wallet_pay_service.running)
|
||||
wallet_pay_service.max_fee = "1.0"
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
self.assertTrue(wallet_pay_service.running)
|
||||
await wallet_pay_service.stop()
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
|
||||
address = await self.blockchain.get_raw_change_address()
|
||||
_, history = await self.ledger.get_local_status_and_history(address)
|
||||
self.assertEqual(history, [])
|
||||
|
||||
node = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.1"})
|
||||
self.addCleanup(node.stop)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.1")
|
||||
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
|
||||
|
||||
await node.stop(False)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.0")
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
|
||||
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
|
||||
self.assertEqual(tx.outputs[0].amount, 100000000)
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
|
||||
@@ -7,7 +7,7 @@ from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
|
||||
@@ -22,7 +22,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(conf)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
@@ -11,6 +12,7 @@ class AddressManagement(CommandTestCase):
|
||||
self.assertItemCount(single, 1)
|
||||
self.assertEqual(single['items'][0], addresses['items'][11])
|
||||
|
||||
|
||||
class SettingsManagement(CommandTestCase):
|
||||
|
||||
async def test_settings(self):
|
||||
@@ -21,5 +23,18 @@ class SettingsManagement(CommandTestCase):
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
|
||||
|
||||
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
|
||||
|
||||
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
|
||||
# tests that changing share_usage_data propagates to the relevant properties
|
||||
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
|
||||
loggly = get_loggly_handler(self.daemon.conf)
|
||||
self.addCleanup(loggly.close)
|
||||
self.assertFalse(self.daemon.analytics_manager.enabled)
|
||||
self.assertFalse(loggly.enabled)
|
||||
self.daemon.jsonrpc_settings_set('share_usage_data', True)
|
||||
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
|
||||
self.assertTrue(self.daemon.analytics_manager.enabled)
|
||||
self.assertTrue(loggly.enabled)
|
||||
self.daemon.jsonrpc_settings_set('share_usage_data', False)
|
||||
@@ -27,6 +27,7 @@ class TestComponentManager(AsyncioTestCase):
|
||||
components.HashAnnouncerComponent,
|
||||
components.PeerProtocolServerComponent,
|
||||
components.StreamManagerComponent,
|
||||
components.WalletServerPaymentsComponent
|
||||
]
|
||||
]
|
||||
self.component_manager = ComponentManager(Config())
|
||||
|
||||
@@ -122,6 +122,10 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
|
||||
|
||||
class TestStreamManager(BlobExchangeTestBase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.client_config.share_usage_data = True
|
||||
|
||||
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
|
||||
+12
-6
@@ -64,8 +64,9 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
self.assertTrue(log.isEnabledFor(logging.INFO))
|
||||
self.assertFalse(log.isEnabledFor(logging.DEBUG))
|
||||
self.assertFalse(log.isEnabledFor(logging.DEBUG))
|
||||
self.assertEqual(len(log.handlers), 1)
|
||||
self.assertEqual(len(log.handlers), 2)
|
||||
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
|
||||
self.assertFalse(log.handlers[1].enabled)
|
||||
|
||||
async with get_logger(["start", "--verbose"]) as log:
|
||||
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
|
||||
@@ -80,16 +81,22 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
|
||||
|
||||
async def test_loggly(self):
|
||||
async with get_logger(["start"], share_usage_data=False) as log:
|
||||
self.assertEqual(len(log.getChild("lbry").handlers), 2) # file and console
|
||||
async with get_logger(["start"]) as log: # default share_usage_data=True
|
||||
async with get_logger(["start"]) as log: # default share_usage_data=False
|
||||
log = log.getChild("lbry")
|
||||
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
|
||||
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
self.assertFalse(log.handlers[2].enabled)
|
||||
async with get_logger(["start"], share_usage_data=True) as log:
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 3)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
async with get_logger(["start"], share_usage_data=True) as log: # explicit share_usage_data=True
|
||||
self.assertTrue(log.handlers[2].enabled)
|
||||
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 3)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
self.assertFalse(log.handlers[2].enabled)
|
||||
|
||||
async def test_quiet(self):
|
||||
async with get_logger(["start"]) as log: # default is loud
|
||||
@@ -100,7 +107,6 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 2)
|
||||
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
|
||||
self.assertIsNot(type(log.handlers[1]), logging.StreamHandler)
|
||||
|
||||
|
||||
class CLITest(AsyncioTestCase):
|
||||
|
||||
@@ -98,6 +98,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__in': {'abc123'}}),
|
||||
('txo.name = :txo_name__in0', {
|
||||
'txo_name__in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
|
||||
('txo.age IN (SELECT age from ages_table)', {})
|
||||
@@ -118,6 +124,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__not_in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
|
||||
('txo.name != :txo_name__not_in0', {
|
||||
'txo_name__not_in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
|
||||
('txo.age NOT IN (SELECT age from ages_table)', {})
|
||||
|
||||
Reference in new issue
Block a user