Compare commits

...
Author SHA1 Message Date
Alex Grintsvayg dd979f800d add config option to skip transaction verification 2020-03-16 14:06:18 -04:00
Lex Berezhny 6814f2e38c Merge pull request #2857 from eggplantbren/master
Trending algorithm with time delay and variable decay rate
2020-03-11 18:34:52 -04:00
Brendon J. Brewer 5f043b9a78 variable decay 2020-03-12 10:33:15 +13:00
Lex Berezhny c9092cd1c7 v0.64.0 2020-03-09 13:32:19 -04:00
Lex Berezhny 86cc65d894 Merge pull request #2848 from lbryio/claim_transaction_list_filtering
added new `txo_list` command with filtering for `--claim_id`, claim `--name` and `--is_received`/`--is_not_received`, also commands `claim_list`/`stream_list`/`channel_list`/`support_list` are based on `txo_list` and thus support most of the new filters
2020-03-08 23:33:38 -04:00
Lex Berezhny 8ef2647fa9 is_received 2020-03-08 23:11:03 -04:00
Lex Berezhny 3ff9e99416 added txo_list command 2020-03-08 16:22:49 -04:00
Lex Berezhny 1731046011 Merge pull request #2846 from lbryio/txo_is_spent
added `is_spent` attribute to transaction outputs
2020-03-06 21:10:30 -05:00
Lex Berezhny c3d96184b6 fix test 2020-03-06 20:35:26 -05:00
Lex Berezhny c255c606a7 added is_spent attribute to transaction outputs 2020-03-06 20:12:38 -05:00
Jack Robison cc69faa1fd v0.63.0 2020-03-06 15:58:34 -05:00
Jack Robison 0f53cd86c8 Merge branch 'debug_find_ffmpeg' 2020-03-06 15:54:24 -05:00
Brannon King 212c8f188d only change LD vars if running from bundle
missed a word


moved is_running_from_bundle
2020-03-06 15:53:09 -05:00
Brannon King dec248adec repair env modified by pyinstaller
see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
2020-03-06 15:53:09 -05:00
Brannon King 98e6a066f4 add unit test and debug output 2020-03-06 15:53:09 -05:00
Jack Robison c90db54a3d Revert "v0.63.0"
This reverts commit 927d1569
2020-03-06 15:51:42 -05:00
Jack Robison 927d156933 v0.63.0 2020-03-06 14:49:53 -05:00
Jack Robison c71f91073f Merge pull request #2845 from lbryio/temp-disable-windows-ffmpeg
fix DHT setup and disable ffmpeg on windows temporarily
2020-03-06 14:20:18 -05:00
Jack Robison ab28387692 remove ProactorEventLoop and disable ffmpeg on windows temporarily 2020-03-06 13:45:34 -05:00
Lex Berezhny c4905d02b9 Merge pull request #2793 from lbryio/dont_validate_non_video
don't validate and repair files that aren't videos; add maximum video bit rate setting
2020-03-04 09:16:37 -05:00
Brannon King ee39880fb5 fix items from review 2020-03-03 21:38:30 -07:00
Brannon King 926b3e56b9 take a hint, lint 2020-03-03 21:38:30 -07:00
Brannon King e060df5367 hide ValueError 2020-03-03 21:38:30 -07:00
Brannon King a3294d4a0d make bit_rate check support maximum 2020-03-03 21:38:30 -07:00
Brannon King 19ce0ab246 ogg -> ogv, ignore files that aren't video 2020-03-03 21:38:30 -07:00
Lex Berezhny 75a1cc0d33 Merge pull request #2840 from lbryio/status_returns_server_features
`status` command returns features available of currently connected wallet server, including trending algorithm
2020-03-03 20:47:04 -05:00
Lex Berezhny 05228529b0 fix tests 2020-03-03 20:26:07 -05:00
Lex Berezhny 3cbeadfbc3 lbrynet status command returns features available of currenty connected wallet server, including thetrending algorithm used 2020-03-03 20:04:57 -05:00
Lex Berezhny 3d4938c0e2 Merge pull request #2839 from lbryio/file_analysis_join_path
use `os.path.join()` instead of string addition when searching for file analysis binaries
2020-03-03 10:48:06 -05:00
Lex Berezhny 1b850b8a2b dont prepend path when doing which ffmpeg 2020-03-02 21:11:11 -05:00
Lex Berezhny ebd33f1869 use os.path.join instead of string addition when searching for file analysis binaries 2020-03-02 20:34:54 -05:00
Jack Robison ac846b4df3 v0.62.0 2020-03-01 20:29:32 -05:00
Jack Robison 0d427c9b90 Merge pull request #2836 from lbryio/wallet-server-lru-caches
Add LRU caches for blocks and block hashes in the wallet server
2020-03-01 15:19:28 -05:00
Jack Robison dc6194f862 perf counter 2020-03-01 15:17:21 -05:00
Jack Robison de50214e1f block lru cache 2020-03-01 15:17:21 -05:00
Jack Robison 5ec4a88c35 block hash lru cache 2020-03-01 15:17:05 -05:00
Jack Robison 31c141e757 Merge pull request #2835 from lbryio/wallet-server-threadpools
Improve wallet server performance with separate thread pools for leveldb, the block processor, and mempool
2020-03-01 15:15:20 -05:00
Jack Robison 9d5760d899 fix test 2020-03-01 14:41:14 -05:00
Jack Robison 7945e1ea3c mempool threadpool 2020-03-01 12:40:13 -05:00
Jack Robison d94c40e371 leveldb threadpool 2020-03-01 12:40:13 -05:00
Jack Robison 31f22122e8 block processor threadpool 2020-03-01 12:40:13 -05:00
Lex Berezhny 506582aa2b Merge pull request #2834 from lbryio/list_resolve_includes_local_data
when using `--resolve` with local claim `*_list` commands, update resolved result with local metadata (such as `is_mine`)
2020-02-29 15:48:49 -05:00
Lex Berezhny a02b251c9b when using --resolve with local claim list commands, update resolved result with local metadata (such as is_mine and is_mine) 2020-02-29 09:42:11 -05:00
Lex Berezhny 34cbb6fa79 Merge pull request #2831 from lbryio/claim_type_index_improvement
update `claim_type_*` db indexes to be more performant
2020-02-27 14:27:24 -05:00
Lex Berezhny 33679b56fd update claim_type_* indexes to be more peformant 2020-02-27 12:07:38 -05:00
Lex Berezhny 8897c4d560 Merge pull request #2816 from osilkin98/removes-anonymous
removes ability to comment anonymously
2020-02-27 10:37:47 -05:00
Oleg Silkin f0c18ccbe7 Removes testing for anonymous comments, fixes up existing asserts 2020-02-26 19:59:38 -05:00
Oleg Silkin e560d83c51 drops support for commenting without a channel 2020-02-26 19:58:03 -05:00
Lex Berezhny eca7addc67 v0.61.0 2020-02-24 10:01:03 -05:00
Lex Berezhny 263f3ba5c9 Merge pull request #2828 from lbryio/update_lbrycrd
update lbrycrd in tests to v0.17.4.3
2020-02-22 14:47:48 -05:00
Lex Berezhny f514123ef0 update lbrycrd 2020-02-22 13:42:28 -05:00
Lex Berezhny 9f461db0d0 Merge pull request #2827 from lbryio/repost_inherit_tags
reposts inherit the tags of the original claim, filtered/blocked claims cause their reposts to also be filtered/blocked
2020-02-22 13:04:31 -05:00
Lex Berezhny 7de8670616 blocked original repost causes all reposts to also be blocked 2020-02-22 12:49:20 -05:00
Lex Berezhny 8c7908b200 reposts inherit the tags of the original claim 2020-02-22 12:23:11 -05:00
Lex Berezhny e7722e039f Merge pull request #2826 from lbryio/fix_not_channel_ids
`claim_search --not_channel_ids` excludes the channel itself and not just claims in the channel
2020-02-21 23:01:00 -05:00
Lex Berezhny 039bc0208a fix tests 2020-02-21 22:22:28 -05:00
Lex Berezhny d57900a069 claim_search --not_channel_ids excludes the channel itself and not just claims in the channel 2020-02-21 21:59:46 -05:00
Jack Robison c1153302aa v0.60.1 2020-02-21 16:13:30 -05:00
Victor Shyba 1086a3297f check its broadcasted and use a retriable call 2020-02-21 17:51:46 -03:00
Jack Robison 5c613934ca Revert "v0.60.1"
This reverts commit 7f03b13579.
2020-02-21 15:05:33 -05:00
Jack Robison 6d7e9092f8 Merge pull request #2825 from lbryio/tx_from_stream_controller
use the stream controller tx instead of local history
2020-02-21 15:04:57 -05:00
Victor Shyba 8bb6f328dc use the stream controller tx instead of local history 2020-02-21 16:41:23 -03:00
Jack Robison 7f03b13579 v0.60.1 2020-02-21 14:14:15 -05:00
Jack Robison 09008ea991 Merge pull request #2824 from lbryio/skip_wallet_server_payment
max wallet server payment fee defaults to 0 and component doesnt start when its not a positive value
2020-02-21 13:27:58 -05:00
Jack Robison ee234212e6 Merge branch 'master' into skip_wallet_server_payment 2020-02-21 13:26:38 -05:00
Jack Robison 277b243f52 Merge pull request #2772 from lbryio/reflector-status
Add `is_fully_reflected` to `file_list` response
2020-02-21 13:21:36 -05:00
Jack Robison 0973ac753f add is_fully_reflected to file_list response 2020-02-21 12:49:19 -05:00
Victor Shyba f41cfbfb97 max wallet server payment fee defaults to 0 and component doesnt start in that case 2020-02-21 14:47:09 -03:00
Lex Berezhny a70980c81a reverting v0.60.1 release 2020-02-21 12:43:43 -05:00
Lex Berezhny 12a962f656 Merge pull request #2822 from lbryio/stream_controller_payment
use stream controller and documented errors for wallet server payments
2020-02-21 12:34:07 -05:00
Lex Berezhny 5936444f3e more reliable wait for wallet server payment test 2020-02-21 12:18:44 -05:00
Lex Berezhny c7d42f00c6 shorten error message for ServerPaymentFeeAboveMaxAllowed 2020-02-21 10:16:56 -05:00
Lex Berezhny b1318a9958 minor cleanup 2020-02-21 10:05:46 -05:00
Victor Shyba d2560d260c use stream controller and documented errors 2020-02-21 00:04:37 -03:00
Jack Robison f40a61cf9a v0.60.1 2020-02-20 17:56:54 -05:00
Jack Robison e0623578bf Merge pull request #2821 from lbryio/ffmpeg-find
add `ffmpeg_find` api to check ffmpeg installation status, don't recheck for ffmpeg in `status`
2020-02-20 17:48:36 -05:00
Jack Robison ead8daaa14 doc 2020-02-20 17:27:18 -05:00
Jack Robison 23b4b9e230 add ffmpeg_find api, don't recheck for it in status 2020-02-20 16:43:41 -05:00
Jack Robison 947017e334 Merge pull request #2820 from lbryio/disconnect-on-oversized
disconnect from client sending message larger than MAX_RECEIVE
2020-02-20 16:24:26 -05:00
Jack Robison 7fd0d6507f disconnect from client sending message larger than MAX_RECEIVE 2020-02-20 16:08:21 -05:00
Jack Robison ad1e9ef086 Merge pull request #2819 from lbryio/max-receive
add MAX_RECEIVE setting to wallet server
2020-02-20 14:38:11 -05:00
Jack Robison 2f1d08e417 add MAX_RECEIVE wallet server setting 2020-02-20 14:11:16 -05:00
Lex Berezhny 55f4eb80ba Merge pull request #2815 from lbryio/apply_share_usage_data_live
apply share_usage_data as its set, without restarting
2020-02-20 13:35:47 -05:00
Victor Shyba 7c160ff65e remove unused parameters 2020-02-20 14:38:13 -03:00
Victor Shyba fc5d5faaed use conf directly isntead of lambda 2020-02-20 14:27:39 -03:00
Victor Shyba 5394f1763c simplify external ip logic 2020-02-20 12:12:40 -03:00
Victor Shyba 050b67c9d6 apply share_usage_data as its set 2020-02-20 12:12:40 -03:00
Lex Berezhny 4b87cb45ee Merge pull request #2707 from lbryio/pay_wallet_servers
wallet servers requiring a daily fee will automatically get paid by client
2020-02-18 17:40:23 -05:00
Victor Shyba f0e1db319c make wallet server payments a component 2020-02-18 19:18:30 -03:00
Victor Shyba 3950715237 add max_wallet_server_fee conf to limit daily wallet server payments 2020-02-18 18:10:58 -03:00
Victor Shyba ae9ba14b59 use both donation and payment addresses separately 2020-02-18 18:10:58 -03:00
Victor Shyba ad6c6fbe35 fixes from review and add analytics 2020-02-18 18:10:58 -03:00
Victor Shyba ffa5c20c88 fix hanging test 2020-02-18 18:10:58 -03:00
Victor Shyba 5a6218eeca improve tests, fix types 2020-02-18 18:10:58 -03:00
Victor Shyba 200531dd96 wait for tx instead 2020-02-18 18:10:58 -03:00
Victor Shyba 15a2f048ac pay wallet server test and initial prototype 2020-02-18 18:10:58 -03:00
Victor Shyba d317a4042c change daily_fee to be lbc instead of dewies 2020-02-18 18:10:58 -03:00
Victor Shyba c2d717aba5 old code from #2683 2020-02-18 18:10:58 -03:00
Lex Berezhny ef74777df1 v0.60.0 2020-02-17 17:52:11 -05:00
Lex Berezhny f6dbf99fb5 Merge pull request #2811 from lbryio/index_fixes
wallet server indexes
2020-02-17 17:43:55 -05:00
Lex Berezhny 0b2d9e15b4 added order bys for fee_amount 2020-02-17 17:24:48 -05:00
Lex Berezhny d14e5e75e8 all three order bys for duration filter 2020-02-17 17:24:48 -05:00
Lex Berezhny df11ef34d2 index cleanup 2020-02-17 17:24:48 -05:00
Lex Berezhny 4fc619262c replacing indexes on trending_global, trending_mixed to be trending_group, trending_mixed 2020-02-17 17:24:48 -05:00
Lex Berezhny 2750eee86d Merge pull request #2810 from lbryio/no_default_order_by
removed default `order_by` for `claim_search`
2020-02-17 17:04:41 -05:00
Lex Berezhny 71cb03345b removed default order_by for claim_search 2020-02-17 16:26:29 -05:00
Lex Berezhny 22c8ca6c40 Merge pull request #2806 from lbryio/privacyDefault
change default privacy setting
2020-02-17 16:21:34 -05:00
Victor Shyba 237a8965eb fix tests 2020-02-17 18:00:23 -03:00
jessop 7d872c7863 change default privacy setting 2020-02-13 18:56:11 -05:00
Jack Robison 0d08858dfb v0.59.2 2020-02-13 17:22:21 -05:00
54 changed files with 1610 additions and 464 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.59.1"
__version__ = "0.64.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+14 -4
View File
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 18 -vf "format=yuv420p"')
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 192k')
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
class CLIConfig(TranscodeConfig):
@@ -558,6 +563,7 @@ class Config(CLIConfig):
"Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
)
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
# reflector settings
reflect_streams = Toggle(
@@ -601,7 +607,7 @@ class Config(CLIConfig):
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
"Whether to share usage stats and diagnostic info with LBRY.", False,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
@@ -621,6 +627,10 @@ class Config(CLIConfig):
"only disable this if file_x commands are not needed", True
)
skip_transaction_verification = Toggle(
"Enable to skip transaction verification if you trust the wallet server you are connected to", False
)
@property
def streaming_host(self):
return self.streaming_server.split(':')[0]
+3
View File
@@ -57,6 +57,9 @@ Code | Name | Message
420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
+21
View File
@@ -242,6 +242,27 @@ class IncompatibleWalletServerError(WalletError):
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
self.address = address
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
class ServerPaymentWalletLockedError(WalletError):
def __init__(self):
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
def __init__(self, daily_fee, max_fee):
self.daily_fee = daily_fee
self.max_fee = max_fee
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
class BlobError(BaseError):
"""
**Blobs**
+3 -8
View File
@@ -3,7 +3,6 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@@ -256,16 +255,12 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
else:
logger.getChild('lbry').setLevel(logging.DEBUG)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
loggly_handler = get_loggly_handler(conf)
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
+9 -6
View File
@@ -110,7 +110,6 @@ class AnalyticsManager:
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = _make_context(system_info.get_platform())
self.installation_id = installation_id
@@ -118,20 +117,24 @@ class AnalyticsManager:
self.task: typing.Optional[asyncio.Task] = None
self.external_ip: typing.Optional[str] = None
@property
def enabled(self):
return self.conf.share_usage_data
@property
def is_started(self):
return self.task is not None
async def start(self):
if self._enabled and self.task is None:
self.external_ip = await utils.get_external_ip()
if self.task is None:
self.task = asyncio.create_task(self.run())
async def run(self):
while True:
await self._send_heartbeat()
if self.enabled:
self.external_ip = await utils.get_external_ip()
await self._send_heartbeat()
await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self):
if self.task is not None and not self.task.done():
@@ -154,7 +157,7 @@ class AnalyticsManager:
async def track(self, event: typing.Dict):
"""Send a single tracking event"""
if self._enabled:
if self.enabled:
log.debug('Sending track event: %s', event)
await self._post(event)
+31
View File
@@ -21,6 +21,7 @@ from lbry.extras.daemon.component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager
from lbry.wallet.usage_payment import WalletServerPayer
log = logging.getLogger(__name__)
@@ -29,6 +30,7 @@ log = logging.getLogger(__name__)
DATABASE_COMPONENT = "database"
BLOB_COMPONENT = "blob_manager"
WALLET_COMPONENT = "wallet"
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_MANAGER_COMPONENT = "stream_manager"
@@ -119,6 +121,7 @@ class WalletComponent(Component):
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
'servers': [
{
'host': session.server[0],
@@ -160,6 +163,34 @@ class WalletComponent(Component):
self.wallet_manager = None
class WalletServerPaymentsComponent(Component):
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
depends_on = [WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.usage_payment_service = WalletServerPayer(
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
)
@property
def component(self) -> typing.Optional[WalletServerPayer]:
return self.usage_payment_service
async def start(self):
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
async def stop(self):
await self.usage_payment_service.stop()
async def get_status(self):
return {
'max_fee': self.usage_payment_service.max_fee,
'running': self.usage_payment_service.running
}
class BlobComponent(Component):
component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT]
+135 -70
View File
@@ -21,9 +21,10 @@ from prometheus_client import generate_latest as prom_generate_latest
from google.protobuf.message import DecodeError
from lbry.wallet import (
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
Transaction, Output, Input, Account
Transaction, Output, Input, Account, database
)
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES
from lbry import utils
from lbry.conf import Config, Setting, NOT_SET
@@ -358,7 +359,7 @@ class Daemon(metaclass=JSONRPCServerType):
@classmethod
def get_api_definitions(cls):
prefix = 'jsonrpc_'
not_grouped = ['routing_table_get']
not_grouped = ['routing_table_get', 'ffmpeg_find']
api = {
'groups': {
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
@@ -765,6 +766,26 @@ class Daemon(metaclass=JSONRPCServerType):
asyncio.get_event_loop().call_later(0, shutdown)
return "Shutting down"
async def jsonrpc_ffmpeg_find(self):
"""
Get ffmpeg installation information
Usage:
ffmpeg_find
Options:
None
Returns:
(dict) Dictionary of ffmpeg information
{
'available': (bool) found ffmpeg,
'which': (str) path to ffmpeg,
'analyze_audio_volume': (bool) should ffmpeg analyze audio
}
"""
return await self._video_file_analyzer.status(recheck=True)
async def jsonrpc_status(self):
"""
Get daemon status
@@ -2147,19 +2168,20 @@ class Daemon(metaclass=JSONRPCServerType):
"""
@requires(WALLET_COMPONENT)
def jsonrpc_claim_list(
self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
def jsonrpc_claim_list(self, claim_type=None, **kwargs):
"""
List my stream and channel claims.
Usage:
claim_list [--claim_type=<claim_type>...]
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
Options:
--claim_type=<claim_type> : (str) claim type: channel, stream, repost, collection
--claim_type=<claim_type> : (str or list) claim type: channel, stream, repost, collection
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -2168,15 +2190,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
claims = account.get_claims
claim_count = account.get_claim_count
else:
claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts)
claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
kwargs['unspent'] = True
return self.jsonrpc_txo_list(**kwargs)
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
@@ -2679,15 +2695,18 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_channel_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
def jsonrpc_channel_list(self, *args, **kwargs):
"""
List my channel claims.
Usage:
channel_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--name=<name>...] [--claim_id=<claim_id>...]
[--page=<page>] [--page_size=<page_size>] [--resolve]
Options:
--name=<name> : (str or list) channel name
--claim_id=<claim_id> : (str or list) channel id
--account_id=<account_id> : (str) id of the account to use
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -2696,15 +2715,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
channels = account.get_channels
channel_count = account.get_channel_count
else:
channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts)
channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(channels, channel_count, page, page_size, resolve=resolve)
kwargs['type'] = 'channel'
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None):
@@ -3122,8 +3135,9 @@ class Daemon(metaclass=JSONRPCServerType):
f"Use --allow-duplicate-name flag to override."
)
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
file_path = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
tx = await Transaction.claim_create(
@@ -3420,15 +3434,18 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_stream_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
def jsonrpc_stream_list(self, *args, **kwargs):
"""
List my stream claims.
Usage:
stream_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>] [--resolve]
[--name=<name>...] [--claim_id=<claim_id>...]
[--page=<page>] [--page_size=<page_size>] [--resolve]
Options:
--name=<name> : (str or list) stream name
--claim_id=<claim_id> : (str or list) stream id
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -3437,15 +3454,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
streams = account.get_streams
stream_count = account.get_stream_count
else:
streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts)
stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(streams, stream_count, page, page_size, resolve=resolve)
kwargs['type'] = 'stream'
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
DHT_COMPONENT, DATABASE_COMPONENT)
@@ -3891,15 +3902,19 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_support_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
def jsonrpc_support_list(self, *args, tips=None, **kwargs):
"""
List supports and tips in my control.
Usage:
support_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--name=<name>...] [--claim_id=<claim_id>...] [--tips]
[--page=<page>] [--page_size=<page_size>]
Options:
--name=<name> : (str or list) claim name
--claim_id=<claim_id> : (str or list) claim id
--tips : (bool) only show tips (is_received=true)
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -3907,15 +3922,12 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
supports = account.get_supports
support_count = account.get_support_count
else:
supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts)
support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(supports, support_count, page, page_size)
kwargs['type'] = 'support'
kwargs['unspent'] = True
kwargs['include_is_received'] = True
if tips is True:
kwargs['is_received'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
@requires(WALLET_COMPONENT)
async def jsonrpc_support_abandon(
@@ -4082,12 +4094,71 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.wallet_manager.get_transaction(txid)
TXO_DOC = """
List transaction outputs.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_txo_list(
self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin
claim_id=None, name=None, unspent=False,
include_is_received=False, is_received=None, is_not_received=None,
wallet_id=None, page=None, page_size=None, resolve=False):
"""
List my transaction outputs.
Usage:
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
[--include_is_received] [--is_received] [--is_not_received]
[--wallet_id=<wallet_id>] [--include_is_received] [--is_received]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
Options:
--type=<type> : (str or list) claim type: stream, channel, support,
purchase, collection, repost, other
--txid=<txid> : (str or list) transaction id of outputs
--unspent : (bool) hide spent outputs, show only unspent ones
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--include_is_received : (bool) calculate the is_received property and
include in output, this happens automatically if you
use the --is_received or --is_not_received filters
--is_received : (bool) only return txos sent from others to this account
--is_not_received : (bool) only return txos created by this account
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--resolve : (bool) resolves each claim to provide additional metadata
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
claims = account.get_txos
claim_count = account.get_txo_count
else:
claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts)
claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts)
constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received}
if is_received is True:
constraints['is_received'] = True
elif is_not_received is True:
constraints['is_received'] = False
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
database.constrain_single_or_list(constraints, 'claim_name', name)
return paginate_rows(claims, claim_count, page, page_size, **constraints)
UTXO_DOC = """
Unspent transaction management.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_utxo_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
def jsonrpc_utxo_list(self, *args, **kwargs):
"""
List unspent transaction outputs
@@ -4103,15 +4174,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
utxos = account.get_utxos
utxo_count = account.get_utxo_count
else:
utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts)
utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(utxos, utxo_count, page, page_size)
kwargs['type'] = ['other', 'purchase']
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
@requires(WALLET_COMPONENT)
async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None):
@@ -4564,7 +4629,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
comment_create (<comment> | --comment=<comment>)
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
[--channel_id=<channel_id>] [--channel_name=<channel_name>]
(--channel_id=<channel_id> | --channel_name=<channel_name>)
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
@@ -4592,23 +4657,23 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_body = {
'comment': comment.strip(),
'claim_id': claim_id,
'parent_id': parent_id,
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
}
channel = await self.get_channel_or_none(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
if channel:
comment_body.update({
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(comment_body, channel)
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
if 'signature' in response:
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
return response
@requires(WALLET_COMPONENT)
+8 -1
View File
@@ -25,6 +25,8 @@ def encode_txo_doc():
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_received': "true if txo was sent from external account to this account",
'is_spent': "true if txo is spent",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'purchase'",
'name': "when type is 'claim' or 'support', this is the claim name",
@@ -168,6 +170,10 @@ class JSONResponseEncoder(JSONEncoder):
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_received is not None:
output['is_received'] = txo.is_received
if txo.is_spent is not None:
output['is_spent'] = txo.is_spent
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
@@ -294,7 +300,8 @@ class JSONResponseEncoder(JSONEncoder):
'added_on': managed_stream.added_on,
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
'is_fully_reflected': managed_stream.is_fully_reflected
}
def encode_claim(self, claim):
+15 -9
View File
@@ -3,10 +3,12 @@ import json
import logging.handlers
import traceback
import typing
from aiohttp.client_exceptions import ClientError
import aiohttp
from lbry import utils, __version__
if typing.TYPE_CHECKING:
from lbry.conf import Config
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
@@ -36,17 +38,19 @@ class JsonFormatter(logging.Formatter):
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
def __init__(self, loggly_token: str, config: 'Config'):
super().__init__()
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
self.cookies = {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
self._config = config
@property
def enabled(self):
return self._config.share_usage_data
@staticmethod
def get_full_message(record):
@@ -62,12 +66,14 @@ class HTTPSLogglyHandler(logging.Handler):
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
if self._loop.is_running() and retry:
if self._loop.is_running() and retry and self.enabled:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
if not self.enabled:
return
try:
asyncio.ensure_future(self._emit(record), loop=self._loop)
except RuntimeError: # TODO: use a second loop
@@ -83,7 +89,7 @@ class HTTPSLogglyHandler(logging.Handler):
pass
def get_loggly_handler():
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
def get_loggly_handler(config):
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
handler.setFormatter(JsonFormatter())
return handler
+8 -4
View File
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee):
raw_content_fee, fully_reflected):
return {
"rowid": rowid,
"added_on": added_on,
@@ -129,7 +129,8 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
"saved_file": bool(saved_file),
"content_fee": None if not raw_content_fee else Transaction(
binascii.unhexlify(raw_content_fee)
)
),
"fully_reflected": fully_reflected
}
@@ -138,11 +139,14 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
signed_claims = {}
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.* "
"select file.rowid, file.*, stream.*, c.*, "
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
" is null then 0 else 1 end as fully_reflected "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"order by c.rowid desc").fetchall():
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -152,7 +156,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
_get_lbry_file_stream_dict(
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee
raw_content_fee, fully_reflected
)
)
for claim_name, claim_id in _batched_select(
+68 -42
View File
@@ -6,24 +6,42 @@ import pathlib
import re
import shlex
import shutil
import platform
import lbry.utils
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
DISABLED = platform.system() == "Windows"
class VideoFileAnalyzer:
def _replace_or_pop_env(self, variable):
if variable + '_ORIG' in self._env_copy:
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
else:
self._env_copy.pop(variable, None)
def __init__(self, conf: TranscodeConfig):
self._conf = conf
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
self._env_copy = dict(os.environ)
if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment:
self._replace_or_pop_env('LD_LIBRARY_PATH')
async def _execute(self, command, arguments):
if DISABLED:
return "Disabled on Windows", -1
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
@@ -31,10 +49,10 @@ class VideoFileAnalyzer:
try:
version, code = await self._execute(name, "-version")
except Exception as e:
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
code = -1
version = ""
version = str(e)
if code != 0 or not version.startswith(name):
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
return version
@@ -44,22 +62,24 @@ class VideoFileAnalyzer:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def status(self, reset=False):
async def status(self, reset=False, recheck=False):
if reset:
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
if self._checked_ffmpeg and not recheck:
installed = self._ffmpeg_installed
else:
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
self._checked_ffmpeg = True
return {
"available": installed,
"which": self._which,
@@ -92,24 +112,21 @@ class VideoFileAnalyzer:
return ""
@staticmethod
def _verify_bitrate(scan_data: json):
if "bit_rate" not in scan_data["format"]:
def _verify_bitrate(self, scan_data: json, file_path):
bit_rate_max = float(self._conf.video_bitrate_maximum)
if bit_rate_max <= 0:
return ""
bit_rate = float(scan_data["format"]["bit_rate"])
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
pixels = -1.0
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
pieces = stream["r_frame_rate"].split('/', 1)
frame_rate = float(pieces[0]) if len(pieces) == 1 \
else float(pieces[0]) / float(pieces[1])
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
if "bit_rate" in scan_data["format"]:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
if pixels > 0.0 and pixels / bit_rate < 3.0:
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
return ""
@@ -173,6 +190,9 @@ class VideoFileAnalyzer:
# https://developers.google.com/media/vp9/settings/vod/
return int(-0.011 * height + 40)
def _get_video_scaler(self):
return self._conf.video_scaler
async def _get_video_encoder(self, scan_data):
# use what the user said if it's there:
# if it's not there, use h264 if we can because it's way faster than the others
@@ -252,12 +272,12 @@ class VideoFileAnalyzer:
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogg"
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogg"
return "ogv"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
@@ -269,31 +289,35 @@ class VideoFileAnalyzer:
scan_data = json.loads(result)
except Exception as e:
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
if validate:
raise Exception(f'Invalid video file: {file_path}')
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
return
raise ValueError(f'Absent or unreadable video file: {file_path}')
if "format" not in scan_data:
if validate:
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
raise ValueError(f'Assuming image file at: {file_path}')
return scan_data
async def verify_or_repair(self, validate, repair, file_path):
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair:
return file_path
await self._verify_ffmpeg_installed()
scan_data = await self._get_scan_data(validate, file_path)
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
video_msg = self._verify_video_encoding(scan_data)
audio_msg = self._verify_audio_encoding(scan_data)
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
@@ -319,6 +343,8 @@ class VideoFileAnalyzer:
if video_msg or bitrate_msg:
video_encoder = await self._get_video_encoder(scan_data)
transcode_command.append(video_encoder)
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
transcode_command.append(self._get_video_scaler())
else:
transcode_command.append("copy")
+1
View File
@@ -148,6 +148,7 @@ types_map = {
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'),
'.ogg': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
+3 -1
View File
@@ -38,7 +38,9 @@ class Censor:
for claim_hash, lookup in (
(row['claim_hash'], self.streams),
(row['claim_hash'], self.channels),
(row['channel_hash'], self.channels)):
(row['channel_hash'], self.channels),
(row['reposted_claim_hash'], self.streams),
(row['reposted_claim_hash'], self.channels)):
censoring_channel_hash = lookup.get(claim_hash)
if censoring_channel_hash:
was_censored = True
+6 -2
View File
@@ -111,6 +111,10 @@ class ManagedStream:
self.started_writing = asyncio.Event(loop=self.loop)
self.finished_write_attempt = asyncio.Event(loop=self.loop)
@property
def is_fully_reflected(self) -> bool:
return self.fully_reflected.is_set()
@property
def descriptor(self) -> StreamDescriptor:
return self.downloader.descriptor
@@ -429,9 +433,9 @@ class ManagedStream:
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
if sent_sd:
if sent_sd: # reflector needed the sd blob
sent.append(self.sd_hash)
if not sent_sd and not needed:
if not sent_sd and not needed: # reflector already has the stream
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+4 -2
View File
@@ -116,7 +116,7 @@ class StreamManager:
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int]):
added_on: Optional[int], fully_reflected: bool):
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err:
@@ -127,6 +127,8 @@ class StreamManager:
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
analytics_manager=self.analytics_manager, added_on=added_on
)
if fully_reflected:
stream.fully_reflected.set()
self.streams[sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
@@ -159,7 +161,7 @@ class StreamManager:
file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'],
file_info['claim'], file_info['content_fee'],
file_info['added_on']
file_info['added_on'], file_info['fully_reflected']
)))
if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop)
+20 -2
View File
@@ -253,6 +253,11 @@ class IntegrationTestCase(AsyncioTestCase):
lambda e: e.tx.id == txid
)
def on_address_update(self, address):
return self.ledger.on_transaction.where(
lambda e: e.address == address
)
def on_transaction_address(self, tx, address):
return self.ledger.on_transaction.where(
lambda e: e.tx.id == tx.id and e.address == address
@@ -543,8 +548,15 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
)
async def support_abandon(self, *args, confirm=True, **kwargs):
if 'blocking' not in kwargs:
kwargs['blocking'] = False
return await self.confirm_and_render(
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
)
async def resolve(self, uri):
return await self.out(self.daemon.jsonrpc_resolve(uri))
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
@@ -552,6 +564,9 @@ class CommandTestCase(IntegrationTestCase):
async def file_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
async def txo_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
@@ -561,9 +576,12 @@ class CommandTestCase(IntegrationTestCase):
async def channel_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
async def transaction_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
@staticmethod
def get_claim_id(tx):
return tx['outputs'][0]['claim_id']
def assertItemCount(self, result, count): # pylint: disable=invalid-name
self.assertEqual(result['total_items'], count)
self.assertEqual(count, result['total_items'])
+6
View File
@@ -4,6 +4,7 @@ import datetime
import random
import socket
import string
import sys
import json
import typing
import asyncio
@@ -276,3 +277,8 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
return response['data']['ip']
except Exception:
return
def is_running_from_bundle():
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+6
View File
@@ -467,6 +467,12 @@ class Account:
'max_receiving_gap': receiving_gap,
}
def get_txos(self, **constraints):
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
def get_txo_count(self, **constraints):
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
def get_utxos(self, **constraints):
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
+9 -4
View File
@@ -6,6 +6,7 @@ COIN = 100*CENT
TIMEOUT = 30.0
TXO_TYPES = {
"other": 0,
"stream": 1,
"channel": 2,
"support": 3,
@@ -14,9 +15,13 @@ TXO_TYPES = {
"repost": 6,
}
CLAIM_TYPE_NAMES = [
'stream',
'channel',
'collection',
'repost',
]
CLAIM_TYPES = [
TXO_TYPES['stream'],
TXO_TYPES['channel'],
TXO_TYPES['collection'],
TXO_TYPES['repost'],
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
]
+65 -28
View File
@@ -107,7 +107,9 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
if not key:
sql.append(constraint)
continue
if key.startswith('$'):
if key.startswith('$$'):
col, key = col[2:], key[1:]
elif key.startswith('$'):
values[key] = constraint
continue
if key.endswith('__not'):
@@ -221,6 +223,19 @@ def rows_to_dict(rows, fields):
return []
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
if value is not None:
if isinstance(value, list):
value = [convert(v) for v in value]
if len(value) == 1:
constraints[column] = value[0]
elif len(value) > 1:
constraints[f"{column}__in"] = value
else:
constraints[column] = convert(value)
return constraints
class SQLiteMixin:
SCHEMA_VERSION: Optional[str] = None
@@ -350,6 +365,7 @@ class Database(SQLiteMixin):
create index if not exists txo_txid_idx on txo (txid);
create index if not exists txo_address_idx on txo (address);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_claim_name_idx on txo (claim_name);
create index if not exists txo_txo_type_idx on txo (txo_type);
"""
@@ -474,20 +490,21 @@ class Database(SQLiteMixin):
async def select_transactions(self, cols, accounts=None, **constraints):
if not {'txid', 'txid__in'}.intersection(constraints):
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
constraints.update({
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
where, values = constraints_to_sql({
'$$account_address.account__in': [a.public_key.address for a in accounts]
})
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
where = f" WHERE account_address.account IN ({account_values})"
constraints['txid__in'] = f"""
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where}
UNION
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where}
"""
constraints.update(values)
return await self.db.execute_fetchall(
*query(f"SELECT {cols} FROM tx", **constraints)
)
TXO_NOT_MINE = Output(None, None, is_my_account=False)
async def get_transactions(self, wallet=None, **constraints):
tx_rows = await self.select_transactions(
'txid, raw, height, position, is_verified',
@@ -538,7 +555,7 @@ class Database(SQLiteMixin):
if _txo:
txo.update_annotations(_txo)
else:
txo.update_annotations(None)
txo.update_annotations(self.TXO_NOT_MINE)
for tx in txs:
txos = tx.outputs
@@ -560,13 +577,33 @@ class Database(SQLiteMixin):
if txs:
return txs[0]
async def select_txos(self, cols, **constraints):
async def select_txos(self, cols, wallet=None, include_is_received=False, **constraints):
if include_is_received:
assert wallet is not None, 'cannot use is_recieved filter without wallet argument'
account_in_wallet, values = constraints_to_sql({
'$$account__in#is_received': [a.public_key.address for a in wallet.accounts]
})
cols += f""",
NOT EXISTS(
SELECT 1 FROM txi JOIN account_address USING (address)
WHERE txi.txid=txo.txid AND {account_in_wallet}
) as is_received
"""
constraints.update(values)
sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)"
if 'accounts' in constraints:
sql += " JOIN account_address USING (address)"
return await self.db.execute_fetchall(*query(sql, **constraints))
async def get_txos(self, wallet=None, no_tx=False, **constraints):
@staticmethod
def constrain_unspent(constraints):
constraints['is_reserved'] = False
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, **constraints):
include_is_received = include_is_received or 'is_received' in constraints
if unspent:
self.constrain_unspent(constraints)
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
if 'order_by' not in constraints:
constraints['order_by'] = [
@@ -577,9 +614,9 @@ class Database(SQLiteMixin):
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
select group_concat(account||"|"||chain) from account_address
where account_address.address=txo.address
)
), exists(select 1 from txi where txi.txoid=txo.txoid)
""",
**constraints
wallet=wallet, include_is_received=include_is_received, **constraints
)
txos = []
txs = {}
@@ -599,6 +636,9 @@ class Database(SQLiteMixin):
txo = txs[row[0]].outputs[row[5]]
row_accounts = dict(a.split('|') for a in row[8].split(','))
account_match = set(row_accounts) & my_accounts
txo.is_spent = bool(row[9])
if include_is_received:
txo.is_received = bool(row[10])
if account_match:
txo.is_my_account = True
txo.is_change = row_accounts[account_match.pop()] == '1'
@@ -634,33 +674,28 @@ class Database(SQLiteMixin):
return txos
async def get_txo_count(self, **constraints):
async def get_txo_count(self, unspent=False, **constraints):
constraints['include_is_received'] = 'is_received' in constraints
constraints.pop('resolve', None)
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
if unspent:
self.constrain_unspent(constraints)
count = await self.select_txos('count(*)', **constraints)
return count[0][0]
@staticmethod
def constrain_utxo(constraints):
constraints['is_reserved'] = False
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
def get_utxos(self, **constraints):
self.constrain_utxo(constraints)
return self.get_txos(**constraints)
return self.get_txos(unspent=True, **constraints)
def get_utxo_count(self, **constraints):
self.constrain_utxo(constraints)
return self.get_txo_count(**constraints)
return self.get_txo_count(unspent=True, **constraints)
async def get_balance(self, wallet=None, accounts=None, **constraints):
assert wallet or accounts, \
"'wallet' or 'accounts' constraints required to calculate balance"
constraints['accounts'] = accounts or wallet.accounts
self.constrain_utxo(constraints)
self.constrain_unspent(constraints)
balance = await self.select_txos('SUM(amount)', **constraints)
return balance[0][0] or 0
@@ -743,11 +778,13 @@ class Database(SQLiteMixin):
@staticmethod
def constrain_claims(constraints):
if {'txo_type', 'txo_type__in'}.intersection(constraints):
return
claim_types = constraints.pop('claim_type', None)
if isinstance(claim_types, str) and claim_types:
claim_types = [claim_types]
if isinstance(claim_types, list) and claim_types:
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
if claim_types:
constrain_single_or_list(
constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x]
)
else:
constraints['txo_type__in'] = CLAIM_TYPES
+29 -9
View File
@@ -262,6 +262,15 @@ class Ledger(metaclass=LedgerRegistry):
self.constraint_spending_utxos(constraints)
return self.db.get_utxo_count(**constraints)
async def get_txos(self, resolve=False, **constraints):
txos = await self.db.get_txos(**constraints)
if resolve:
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
return txos
def get_txo_count(self, **constraints):
return self.db.get_txo_count(**constraints)
def get_transactions(self, **constraints):
return self.db.get_transactions(**constraints)
@@ -558,7 +567,7 @@ class Ledger(metaclass=LedgerRegistry):
cache_item = self._tx_cache[txid] = TransactionCacheItem()
elif cache_item.tx is not None and \
cache_item.tx.height >= remote_height and \
(cache_item.tx.is_verified or remote_height < 1):
(cache_item.tx.is_verified or self.config.get('skip_transaction_verification') or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
async with cache_item.lock:
@@ -580,6 +589,8 @@ class Ledger(metaclass=LedgerRegistry):
async def maybe_verify_transaction(self, tx, remote_height):
tx.height = remote_height
if self.config.get("skip_transaction_verification"):
return
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
@@ -735,10 +746,13 @@ class Ledger(metaclass=LedgerRegistry):
async def _resolve_for_local_results(self, accounts, txos):
results = []
response = await self.resolve(accounts, [txo.permanent_url for txo in txos])
response = await self.resolve(
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
)
for txo in txos:
resolved = response[txo.permanent_url]
resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None
if isinstance(resolved, Output):
resolved.update_annotations(txo)
results.append(resolved)
else:
if isinstance(resolved, dict) and 'error' in resolved:
@@ -843,7 +857,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_update_outputs:
if is_my_inputs: # updating my own claim
@@ -862,7 +877,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
else: # someone sent us their claim
item['update_info'].append({
@@ -871,7 +887,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_support_outputs:
item['support_info'].append({
@@ -881,7 +898,8 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': not is_my_inputs,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
if is_my_inputs:
for txo in tx.other_support_outputs:
@@ -892,7 +910,8 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_abandon_outputs:
item['abandon_info'].append({
@@ -909,7 +928,8 @@ class Ledger(metaclass=LedgerRegistry):
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.purchased_claim_id,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
history.append(item)
return history
+1
View File
@@ -184,6 +184,7 @@ class WalletManager:
'auto_connect': True,
'default_servers': config.lbryum_servers,
'data_path': config.wallet_dir,
'skip_transaction_verification': config.skip_transaction_verification,
}
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
+13 -1
View File
@@ -1,5 +1,6 @@
import logging
import asyncio
import json
from time import perf_counter
from operator import itemgetter
from typing import Dict, Optional, Tuple
@@ -68,7 +69,14 @@ class ClientSession(BaseClientSession):
log.info("timeout sending %s to %s:%i", method, *self.server)
raise asyncio.TimeoutError
if done:
return request.result()
try:
return request.result()
except ConnectionResetError:
log.error(
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
self.server[0], method, len(args), len(json.dumps(args))
)
raise
except (RPCError, ProtocolError) as e:
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
@@ -154,6 +162,7 @@ class Network:
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
@@ -181,17 +190,20 @@ class Network:
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
async def start(self):
+3
View File
@@ -173,6 +173,9 @@ class SPVNode:
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
self.data_path = tempfile.mkdtemp()
conf = {
'DESCRIPTION': '',
'PAYMENT_ADDRESS': '',
'DAILY_FEE': '0',
'DB_DIRECTORY': self.data_path,
'DAEMON_URL': blockchain_node.rpc_url,
'REORG_LIMIT': '100',
+5 -3
View File
@@ -386,9 +386,11 @@ class RPCSession(SessionBase):
while not self.is_closing():
try:
message = await self.framer.receive_message()
except MemoryError as e:
self.logger.warning(f'{e!r}')
continue
except MemoryError:
self.logger.warning('received oversized message from %s:%s, dropping connection',
self._address[0], self._address[1])
self._close()
return
self.last_recv = time.perf_counter()
self.recv_count += 1
+5 -1
View File
@@ -1,6 +1,7 @@
import time
import asyncio
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import lbry
from lbry.schema.claim import Claim
@@ -145,6 +146,7 @@ class BlockProcessor:
self.blocks_event = asyncio.Event()
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
self.logger = class_logger(__name__, self.__class__.__name__)
self.executor = ThreadPoolExecutor(1)
# Meta
self.next_cache_check = 0
@@ -165,6 +167,7 @@ class BlockProcessor:
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -173,7 +176,7 @@ class BlockProcessor:
# consistent and not being updated elsewhere.
async def run_in_thread_locked():
async with self.state_lock:
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
return await asyncio.shield(run_in_thread_locked())
async def check_and_advance_blocks(self, raw_blocks):
@@ -657,6 +660,7 @@ class BlockProcessor:
self.logger.info('flushing to DB for a clean shutdown...')
await self.flush(True)
self.db.close()
self.executor.shutdown(wait=True)
def force_chain_reorg(self, count):
"""Force a reorg of the given number of blocks.
+17 -1
View File
@@ -3,6 +3,7 @@ import itertools
import json
import time
from functools import wraps
from pylru import lrucache
import aiohttp
@@ -42,6 +43,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(1000000)
self._block_cache = lrucache(100000)
async def close(self):
if self.connector:
@@ -219,12 +222,25 @@ class Daemon:
async def block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
if first + count < (self.cached_height() or 0) - 200:
return await self._cached_block_hex_hashes(first, count)
params_iterable = ((h, ) for h in range(first, first + count))
return await self._send_vector('getblockhash', params_iterable)
async def _cached_block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
cached = self._block_hash_cache.get((first, count))
if cached:
return cached
params_iterable = ((h, ) for h in range(first, first + count))
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
return self._block_hash_cache[(first, count)]
async def deserialised_block(self, hex_hash):
"""Return the deserialised block with the given hex hash."""
return await self._send_single('getblock', (hex_hash, True))
if not self._block_cache.get(hex_hash):
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
return self._block_cache[hex_hash]
async def raw_blocks(self, hex_hashes):
"""Return the raw binary blocks with the given hex hashes."""
+32 -12
View File
@@ -291,6 +291,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
not_channel_ids_binary = {
unhexlify(ncid)[::-1] for ncid in not_channel_ids
}
constraints['claim.claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
@@ -316,7 +317,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
constraints['claim.txo_hash'] = tx_hash + struct.pack('<I', nout)
if 'claim_type' in constraints:
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
claim_types = constraints.pop('claim_type')
if isinstance(claim_types, str):
claim_types = [claim_types]
if claim_types:
constraints['claim.claim_type__in'] = {
CLAIM_TYPES[claim_type] for claim_type in claim_types
}
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
@@ -418,8 +425,6 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
context = ctx.get()
search_censor = context.get_search_censor()
txo_rows = search_claims(search_censor, **constraints)
@@ -485,6 +490,14 @@ def resolve_url(raw_url):
return channel
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
CASE WHEN claim.claim_type = {CLAIM_TYPES['repost']}
THEN claim.reposted_claim_hash
ELSE claim.claim_hash
END
"""
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
@@ -504,7 +517,8 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
AND tag = '{item}'
)
"""
@@ -517,7 +531,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
AND tag IN ({values})
)
"""
@@ -531,14 +545,16 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
@@ -557,15 +573,17 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$all_{attr}{i}' for i in range(len(all_items))
)
if for_count:
constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
constraints[f'#_all_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
)
"""
else:
constraints[f'#_all_{attr}'] = f"""
{len(all_items)}=(
SELECT count(*) FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
@@ -578,14 +596,16 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
+3 -2
View File
@@ -1,8 +1,9 @@
from . import zscore
from . import ar
from . import variable_decay
TRENDING_ALGORITHMS = {
'zscore': zscore,
'ar': ar
'ar': ar,
'variable_decay': variable_decay
}
@@ -0,0 +1,431 @@
"""
Delayed AR with variable decay rate.
The spike height function is also simpler.
"""
import copy
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
# How frequently to write trending values to the db
SAVE_INTERVAL = 10
# Renormalisation interval
RENORM_INTERVAL = 1000
# Assertion
assert RENORM_INTERVAL % SAVE_INTERVAL == 0
# Decay coefficient per renormalisation interval
DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
# Log trending calculations?
TRENDING_LOG = True
def install(connection):
"""
Install the trending algorithm.
"""
check_trending_values(connection)
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
f.close()
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
reset them. This will allow resyncing from a standard snapshot.
"""
c = connection.cursor()
needs_reset = False
for row in c.execute("SELECT COUNT(*) num FROM claim WHERE trending_global <> 0;"):
if row[0] != 0:
needs_reset = True
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True, end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def spike_height(x, x_old):
"""
Compute the size of a trending spike (normed - constant units).
"""
# Sign of trending spike
sign = 1.0
if x < x_old:
sign = -1.0
# Magnitude
mag = abs(x**0.25 - x_old**0.25)
# Minnow boost
mag *= 1.0 + 2E4/(x + 100.0)**2
return sign*mag
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
def test_trending():
"""
Quick trending test for claims with different support patterns.
Actually use the run() function.
"""
# Create a fake "claims.db" for testing
# pylint: disable=I1101
dbc = apsw.Connection(":memory:")
db = dbc.cursor()
# Create table
db.execute("""
BEGIN;
CREATE TABLE claim (claim_hash TEXT PRIMARY KEY,
amount REAL NOT NULL DEFAULT 0.0,
support_amount REAL NOT NULL DEFAULT 0.0,
trending_mixed REAL NOT NULL DEFAULT 0.0);
COMMIT;
""")
# Insert initial states of claims
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
for key in stuff:
l.append((key, stuff[key]))
return l
db.executemany("""
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
# Main loop
for height in range(1, 1000):
# One-off supports
if height == 1:
everything["huge_whale"] += 5E5
everything["medium_whale"] += 5E4
everything["small_whale"] += 5E3
# Every block
if height < 500:
everything["huge_whale_botted"] += 5E5/500
everything["minnow"] += 1
# Remove supports
if height == 500:
for key in everything:
everything[key] = 0.01
# Whack into the db
db.executemany("""
UPDATE claim SET amount = 1E8*? WHERE claim_hash = ?;
""", [(y, x) for (x, y) in to_list_of_tuples(everything)])
# Call run()
run(db, height, height, everything.keys())
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trending_data.claims:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
test_trending()
+27 -20
View File
@@ -130,35 +130,42 @@ class SQLDB:
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
-- naked order bys (no filters)
create unique index if not exists claim_release_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_idx on claim (trending_group, trending_mixed, claim_hash);
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
-- claim_type filter + order by
create unique index if not exists claim_type_release_idx on claim (release_time, claim_type, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (trending_group, trending_mixed, claim_type, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (effective_amount, claim_type, claim_hash);
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_global, trending_mixed, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, effective_amount, claim_hash);
-- stream_type filter + order by
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
create unique index if not exists stream_type_trending_idx on claim (stream_type, trending_group, trending_mixed, claim_hash);
create unique index if not exists stream_type_effective_amount_idx on claim (stream_type, effective_amount, claim_hash);
create unique index if not exists filter_stream_duration_idx on claim (duration, trending_global, trending_mixed, claim_hash);
-- channel_hash filter + order by
create unique index if not exists channel_hash_release_idx on claim (channel_hash, release_time, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_group, trending_mixed, claim_hash);
create unique index if not exists channel_hash_effective_amount_idx on claim (channel_hash, effective_amount, claim_hash);
-- duration filter + order by
create unique index if not exists duration_release_idx on claim (duration, release_time, claim_hash);
create unique index if not exists duration_trending_idx on claim (duration, trending_group, trending_mixed, claim_hash);
create unique index if not exists duration_effective_amount_idx on claim (duration, effective_amount, claim_hash);
-- fee_amount + order by
create unique index if not exists fee_amount_release_idx on claim (fee_amount, release_time, claim_hash);
create unique index if not exists fee_amount_trending_idx on claim (fee_amount, trending_group, trending_mixed, claim_hash);
create unique index if not exists fee_amount_effective_amount_idx on claim (fee_amount, effective_amount, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
@@ -859,7 +866,7 @@ class LBRYLevelDB(LevelDB):
super().__init__(*args, **kwargs)
path = os.path.join(self.env.db_dir, 'claims.db')
trending = []
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
for algorithm_name in self.env.trending_algorithms:
if algorithm_name in TRENDING_ALGORITHMS:
trending.append(TRENDING_ALGORITHMS[algorithm_name])
self.sql = SQLDB(
+12 -1
View File
@@ -37,6 +37,7 @@ class Env:
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
@@ -72,16 +73,18 @@ class Env:
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
# The electrum client takes the empty string as unspecified
self.payment_address = self.default('PAYMENT_ADDRESS', '')
self.donation_address = self.default('DONATION_ADDRESS', '')
# Server limits to help prevent DoS
self.max_send = self.integer('MAX_SEND', 1000000)
self.max_receive = self.integer('MAX_RECEIVE', 1000000)
self.max_subs = self.integer('MAX_SUBS', 250000)
self.max_sessions = self.sane_max_sessions()
self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
self.description = self.default('DESCRIPTION', '')
self.daily_fee = self.integer('DAILY_FEE', 0)
self.daily_fee = self.string_amount('DAILY_FEE', '0')
# Identities
clearnet_identity = self.clearnet_identity()
@@ -107,6 +110,14 @@ class Env:
raise cls.Error(f'required envvar {envvar} not set')
return value
@classmethod
def string_amount(cls, envvar, default):
value = environ.get(envvar, default)
amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
if len(value) > 0 and not amount_pattern.fullmatch(value):
raise cls.Error(f'{value} is not a valid amount for {envvar}')
return value
@classmethod
def integer(cls, envvar, default):
value = environ.get(envvar)
+8 -6
View File
@@ -20,7 +20,7 @@ from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.wallet.server import util
@@ -63,6 +63,7 @@ class LevelDB:
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
@@ -139,6 +140,7 @@ class LevelDB:
def close(self):
self.utxo_db.close()
self.history.close_db()
self.executor.shutdown(wait=True)
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -406,7 +408,7 @@ class LevelDB:
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(None, read_headers)
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -445,7 +447,7 @@ class LevelDB:
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await asyncio.get_event_loop().run_in_executor(None, read_history)
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
@@ -611,7 +613,7 @@ class LevelDB:
return utxos
while True:
utxos = await asyncio.get_event_loop().run_in_executor(None, read_utxos)
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
@@ -664,5 +666,5 @@ class LevelDB:
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await asyncio.get_event_loop().run_in_executor(None, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(None, lookup_utxos, hashX_pairs)
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
+15 -10
View File
@@ -6,12 +6,13 @@
# and warranty status of this software.
"""Mempool handling."""
import os
import asyncio
import itertools
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
import attr
@@ -105,14 +106,15 @@ class MemPool:
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
self.logger.info('beginning processing of daemon mempool. '
'This can take some time...')
start = time.time()
start = time.perf_counter()
await synchronized_event.wait()
elapsed = time.time() - start
elapsed = time.perf_counter() - start
self.logger.info(f'synced in {elapsed:.2f}s')
while True:
self.logger.info(f'{len(self.txs):,d} txs '
@@ -121,12 +123,15 @@ class MemPool:
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(None, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
try:
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
finally:
self.executor.shutdown(wait=True)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
@@ -289,7 +294,7 @@ class MemPool:
return txs
# Thread this potentially slow operation so as not to block
tx_map = await asyncio.get_event_loop().run_in_executor(None, deserialize_txs)
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
+14 -2
View File
@@ -30,6 +30,7 @@ from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
from lbry.wallet.rpc.framing import NewlineFramer
import lbry.wallet.server.version as VERSION
from lbry.wallet.rpc import (
@@ -621,6 +622,7 @@ class SessionBase(RPCSession):
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
connection = JSONRPCConnection(JSONRPCAutoDetect)
self.env = session_mgr.env
super().__init__(connection=connection)
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.session_mgr = session_mgr
@@ -628,7 +630,6 @@ class SessionBase(RPCSession):
self.mempool = mempool
self.peer_mgr = peer_mgr
self.kind = kind # 'RPC', 'TCP' etc.
self.env = session_mgr.env
self.coin = self.env.coin
self.anon_logs = self.env.anon_logs
self.txs_sent = 0
@@ -641,6 +642,9 @@ class SessionBase(RPCSession):
async def notify(self, touched, height_changed):
pass
def default_framer(self):
return NewlineFramer(self.env.max_receive)
def peer_address_str(self, *, for_log=True):
"""Returns the peer's IP address and port as a human-readable
string, respecting anon logs if the output is for a log."""
@@ -800,6 +804,7 @@ class LBRYElectrumX(SessionBase):
'blockchain.transaction.get_merkle': cls.transaction_merkle,
'server.add_peer': cls.add_peer,
'server.banner': cls.banner,
'server.payment_address': cls.payment_address,
'server.donation_address': cls.donation_address,
'server.features': cls.server_features_async,
'server.peers.subscribe': cls.peers_subscribe,
@@ -854,9 +859,11 @@ class LBRYElectrumX(SessionBase):
'protocol_max': max_str,
'genesis_hash': env.coin.GENESIS_HASH,
'description': env.description,
'payment_address': env.donation_address,
'payment_address': env.payment_address,
'donation_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
async def server_features_async(self):
@@ -1362,11 +1369,16 @@ class LBRYElectrumX(SessionBase):
('$SERVER_VERSION', self.version),
('$DAEMON_VERSION', daemon_version),
('$DAEMON_SUBVERSION', network_info['subversion']),
('$PAYMENT_ADDRESS', self.env.payment_address),
('$DONATION_ADDRESS', self.env.donation_address),
]:
banner = banner.replace(*pair)
return banner
async def payment_address(self):
"""Return the payment address as a string, empty if there is none."""
return self.env.payment_address
async def donation_address(self):
"""Return the donation address as a string, empty if there is none."""
return self.env.donation_address
+9 -4
View File
@@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator:
class Output(InputOutput):
__slots__ = (
'amount', 'script', 'is_change', 'is_my_account',
'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account',
'channel', 'private_key', 'meta',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims',
@@ -215,13 +215,16 @@ class Output(InputOutput):
def __init__(self, amount: int, script: OutputScript,
tx_ref: TXRef = None, position: int = None,
is_change: Optional[bool] = None, is_my_account: Optional[bool] = None,
is_change: Optional[bool] = None, is_spent: Optional[bool] = None,
is_received: Optional[bool] = None, is_my_account: Optional[bool] = None,
channel: Optional['Output'] = None, private_key: Optional[str] = None
) -> None:
super().__init__(tx_ref, position)
self.amount = amount
self.script = script
self.is_change = is_change
self.is_spent = is_spent
self.is_received = is_received
self.is_my_account = is_my_account
self.channel = channel
self.private_key = private_key
@@ -234,10 +237,12 @@ class Output(InputOutput):
def update_annotations(self, annotated):
if annotated is None:
self.is_change = False
self.is_my_account = False
self.is_change = None
self.is_spent = None
self.is_my_account = None
else:
self.is_change = annotated.is_change
self.is_spent = annotated.is_spent
self.is_my_account = annotated.is_my_account
self.channel = annotated.channel if annotated else None
self.private_key = annotated.private_key if annotated else None
+78
View File
@@ -0,0 +1,78 @@
import asyncio
import logging
from lbry.error import (
ServerPaymentFeeAboveMaxAllowedError,
ServerPaymentInvalidAddressError,
ServerPaymentWalletLockedError
)
from lbry.wallet.dewies import lbc_to_dewies
from lbry.wallet.stream import StreamController
from lbry.wallet.transaction import Output, Transaction
log = logging.getLogger(__name__)
class WalletServerPayer:
def __init__(self, payment_period=24 * 60 * 60, max_fee='1.0', analytics_manager=None):
self.ledger = None
self.wallet = None
self.running = False
self.task = None
self.payment_period = payment_period
self.analytics_manager = analytics_manager
self.max_fee = max_fee
self._on_payment_controller = StreamController()
self.on_payment = self._on_payment_controller.stream
self.on_payment.listen(None, on_error=lambda e: logging.warning(e.args[0]))
async def pay(self):
while self.running:
await asyncio.sleep(self.payment_period)
features = await self.ledger.network.retriable_call(self.ledger.network.get_server_features)
address = features['payment_address']
amount = str(features['daily_fee'])
if not address or not amount:
continue
if not self.ledger.is_valid_address(address):
self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
continue
if self.wallet.is_locked:
self._on_payment_controller.add_error(ServerPaymentWalletLockedError())
continue
amount = lbc_to_dewies(features['daily_fee']) # check that this is in lbc and not dewies
limit = lbc_to_dewies(self.max_fee)
if amount > limit:
self._on_payment_controller.add_error(
ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee)
)
continue
tx = await Transaction.create(
[],
[Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
self.wallet.get_accounts_or_all(None),
self.wallet.get_account_or_default(None)
)
await self.ledger.broadcast(tx)
if self.analytics_manager:
await self.analytics_manager.send_credits_sent()
self._on_payment_controller.add(tx)
async def start(self, ledger=None, wallet=None):
if lbc_to_dewies(self.max_fee) < 1:
return
self.ledger = ledger
self.wallet = wallet
self.running = True
self.task = asyncio.ensure_future(self.pay())
self.task.add_done_callback(lambda _: log.info("Stopping wallet server payments."))
async def stop(self):
if self.running:
self.running = False
self.task.cancel()
+2 -2
View File
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
try:
await analyzer.verify_or_repair(True, False, video_file)
print("No concerns. Ship it!")
except FileNotFoundError as e:
print(str(e))
except (FileNotFoundError, ValueError) as e:
print("Analysis failed.", str(e))
except Exception as e:
print(str(e))
transcode = input("Would you like to make a repaired clone now? [y/N] ")
+1 -1
View File
@@ -333,7 +333,7 @@ class Examples(CommandTestCase):
)
reply = await r(
'You can r',
'Use the parent_id param to make replies',
'comment', 'create',
'--comment="I have photographic evidence confirming Sasquatch exists"',
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
@@ -4,7 +4,6 @@ import logging
from binascii import unhexlify
from urllib.request import urlopen
from lbry.error import InsufficientFundsError
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
@@ -73,6 +72,19 @@ class ClaimSearchCommand(ClaimTestCase):
(result['txid'], result['claim_id'])
)
async def test_disconnect_on_memory_error(self):
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23828
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23829
with self.assertRaises(ConnectionResetError):
await self.claim_search(claim_ids=claim_ids)
async def test_basic_claim_search(self):
await self.create_channel()
channel_txo = self.channel['outputs'][0]
@@ -250,6 +262,7 @@ class ClaimSearchCommand(ClaimTestCase):
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
chan3_id = self.get_claim_id(await self.channel_create('@chan3'))
chan4 = await self.channel_create('@chan4', '0.1')
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
@@ -272,6 +285,7 @@ class ClaimSearchCommand(ClaimTestCase):
await match([claim6, claim5, claim4, claim1], not_channel_ids=[chan1_id], claim_type='stream')
await match([claim6, claim3, claim2, claim1], not_channel_ids=[chan2_id], claim_type='stream')
await match([claim6, claim1], not_channel_ids=[chan1_id, chan2_id], claim_type='stream')
await match([claim6, claim1, chan4], not_channel_ids=[chan1_id, chan2_id])
# not_channel_ids + valid_channel_signature
await match([claim5, claim4, claim3, claim2, claim1],
@@ -384,12 +398,116 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([], text='cloud')
class TransactionCommands(ClaimTestCase):
async def test_transaction_list(self):
channel_id = self.get_claim_id(await self.channel_create())
await self.channel_update(channel_id, bid='0.5')
await self.channel_abandon(claim_id=channel_id)
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
await self.stream_abandon(claim_id=stream_id)
r = await self.transaction_list()
self.assertEqual(7, len(r))
self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id'])
self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id'])
self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id'])
self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id'])
self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id'])
self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id'])
class TransactionOutputCommands(ClaimTestCase):
async def test_txo_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
await self.channel_update(channel_id, bid='0.5')
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
# type filtering
r = await self.txo_list(type='channel')
self.assertEqual(2, len(r))
self.assertEqual('channel', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('channel', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type='stream')
self.assertEqual(2, len(r))
self.assertEqual('stream', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('stream', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type=['stream', 'channel'])
self.assertEqual(4, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(claim_id=[stream_id, channel_id])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, _, channel_name, _ = (c['name'] for c in r)
r = await self.txo_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.txo_list(name=stream_name)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=[stream_name, channel_name])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=['beef'])
self.assertEqual(0, len(r))
r = await self.txo_list()
self.assertEqual(9, len(r))
await self.stream_abandon(claim_id=stream_id)
r = await self.txo_list()
self.assertEqual(10, len(r))
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertTrue(r[0]['is_spent'])
self.assertTrue(r[1]['is_spent'])
async def test_txo_list_received_filtering(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create(claim_address=address2)
r = await self.txo_list(include_is_received=True)
self.assertEqual(2, len(r))
self.assertFalse(r[0]['is_received'])
self.assertTrue(r[1]['is_received'])
rt = await self.txo_list(is_not_received=True)
self.assertEqual(1, len(rt))
self.assertEqual(rt[0], r[0])
rf = await self.txo_list(is_received=True)
self.assertEqual(1, len(rf))
self.assertEqual(rf[0], r[1])
r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id)
self.assertEqual(1, len(r))
self.assertTrue(r[0]['is_received'])
class ClaimCommands(ClaimTestCase):
async def test_claim_list_type_filtering(self):
await self.channel_create()
await self.stream_create()
async def test_claim_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
@@ -402,6 +520,31 @@ class ClaimCommands(ClaimTestCase):
self.assertEqual(2, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.claim_list(claim_id=stream_id)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(claim_id=[stream_id, channel_id])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, channel_name = (c['name'] for c in r)
r = await self.claim_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.claim_list(name=stream_name)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=[stream_name, channel_name])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=['beef'])
self.assertEqual(0, len(r))
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
@@ -473,6 +616,9 @@ class ClaimCommands(ClaimTestCase):
self.assertTrue(r[0]['meta']['is_controlling'])
self.assertTrue(r[1]['meta']['is_controlling'])
# check that metadata is transfered
self.assertTrue(r[0]['is_mine'])
class ChannelCommands(CommandTestCase):
@@ -814,8 +960,11 @@ class StreamCommands(ClaimTestCase):
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
async def test_repost(self):
sql = self.conductor.spv_node.server.bp.sql
sql.execute(sql.TAG_INDEXES)
await self.channel_create('@goodies', '1.0')
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies')
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies', tags=['foo', 'gaming'])
claim_id = self.get_claim_id(tx)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
@@ -826,6 +975,16 @@ class StreamCommands(ClaimTestCase):
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
# tags are inherited (non-common / indexed tags)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['foo'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['foo'], claim_type=['stream', 'repost']), 0)
# "common" / indexed tags work too
self.assertIn('gaming', sql.TAG_INDEXES) # if this breaks, next test doesn't make sense
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['gaming'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['gaming'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['gaming'], claim_type=['stream', 'repost']), 0)
await self.channel_create('@reposting-goodies', '1.0')
await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies')
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2)
@@ -845,25 +1004,29 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(search['reposted_claim']['meta']['reposted'], 2)
self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies')
resolved = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
resolved = await self.out(
self.daemon.jsonrpc_resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
)
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
async def test_filtering_channels_for_removing_content(self):
await self.channel_create('@some_channel', '1.0')
await self.stream_create('good_content', '1.1', channel_name='@some_channel', tags=['good'])
await self.channel_create('@some_channel', '0.1')
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
bad_content_id = self.get_claim_id(
await self.stream_create('bad_content', '1.1', channel_name='@some_channel', tags=['bad'])
await self.stream_create('bad_content', '0.1', channel_name='@some_channel', tags=['bad'])
)
filtering_channel_id = self.get_claim_id(
await self.channel_create('@filtering', '1.0')
await self.channel_create('@filtering', '0.1')
)
self.conductor.spv_node.server.db.sql.filtering_channel_hashes.add(
unhexlify(filtering_channel_id)[::-1]
)
await self.stream_repost(bad_content_id, 'filter1', '1.1', channel_name='@filtering')
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_streams))
await self.stream_repost(bad_content_id, 'filter1', '0.1', channel_name='@filtering')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_streams))
# search for blocked content directly
# search for filtered content directly
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content'))
blocked = result['blocked']
self.assertEqual([], result['items'])
@@ -872,40 +1035,84 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# search channel containing blocked content
# search inside channel containing filtered content
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
blocked = result['blocked']
filtered = result['blocked']
self.assertEqual(1, len(result['items']))
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual(1, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(1, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# content was filtered by not_tag before censoring
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
self.assertEqual(0, len(result['items']))
self.assertEqual({"channels": [], "total": 0}, result['blocked'])
# filtered content can still be resolved
result = await self.resolve('lbry://@some_channel/bad_content')
self.assertEqual(bad_content_id, result['claim_id'])
blocking_channel_id = self.get_claim_id(
await self.channel_create('@blocking', '1.0')
await self.channel_create('@blocking', '0.1')
)
self.conductor.spv_node.server.db.sql.blocking_channel_hashes.add(
unhexlify(blocking_channel_id)[::-1]
)
# filtered content can still be resolved
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
self.assertEqual(bad_content_id, result['lbry://@some_channel/bad_content']['claim_id'])
await self.stream_repost(bad_content_id, 'block1', '1.1', channel_name='@blocking')
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_streams))
await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_streams))
# blocked content is not resolveable
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
error = result['lbry://@some_channel/bad_content']['error']
error = (await self.resolve('lbry://@some_channel/bad_content'))['error']
self.assertEqual(error['name'], 'BLOCKED')
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
# a filtered/blocked channel impacts all content inside it
bad_channel_id = self.get_claim_id(
await self.channel_create('@bad_channel', '0.1', tags=['bad-stuff'])
)
worse_content_id = self.get_claim_id(
await self.stream_create('worse_content', '0.1', channel_name='@bad_channel', tags=['bad-stuff'])
)
# check search before filtering channel
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
self.assertEqual(2, result['total_items'])
self.assertEqual('worse_content', result['items'][0]['name'])
self.assertEqual('@bad_channel', result['items'][1]['name'])
# filter channel out
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_channels))
await self.stream_repost(bad_channel_id, 'filter2', '0.1', channel_name='@filtering')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_channels))
# same claim search as previous now returns 0 results
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
filtered = result['blocked']
self.assertEqual(0, len(result['items']))
self.assertEqual(3, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(3, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# filtered channel should still resolve
result = await self.resolve('lbry://@bad_channel')
self.assertEqual(bad_channel_id, result['claim_id'])
result = await self.resolve('lbry://@bad_channel/worse_content')
self.assertEqual(worse_content_id, result['claim_id'])
# block channel
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_channels))
await self.stream_repost(bad_channel_id, 'block2', '0.1', channel_name='@blocking')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_channels))
# channel, claim in channel or claim individually no longer resolve
self.assertEqual((await self.resolve('lbry://@bad_channel'))['error']['name'], 'BLOCKED')
self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED')
self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED')
async def test_publish_updates_file_list(self):
tx = await self.stream_create(title='created')
txo = tx['outputs'][0]
@@ -1285,11 +1492,12 @@ class StreamCommands(ClaimTestCase):
tx = await self.stream_create(bid='2.5') # creates new claim
claim_id = self.get_claim_id(tx)
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['claim_info']), 1)
self.assertEqual(txs[0]['confirmations'], 1)
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
await self.assertBalance(self.account, '7.479893')
@@ -1299,19 +1507,23 @@ class StreamCommands(ClaimTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
await self.stream_update(claim_id, bid='1.0') # updates previous claim
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['update_info']), 1)
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['update_info'][0]['is_spent'])
self.assertTrue(txs[1]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.0002165')
await self.assertBalance(self.account, '8.9796765')
await self.stream_abandon(claim_id)
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[1]['update_info'][0]['is_spent'])
self.assertTrue(txs[2]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
await self.assertBalance(self.account, '9.9795695')
@@ -1361,15 +1573,14 @@ class StreamCommands(ClaimTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
r = await self.resolve('lbry://@abc/foo')
self.assertEqual(
r['lbry://@abc/foo']['claim_id'],
r['claim_id'],
self.get_claim_id(tx3)
)
# publishing again clears channel
tx4 = await self.publish('foo', languages='uk-UA', tags=['Anime', 'anime '])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
r = await self.resolve('lbry://foo')
claim = r['lbry://foo']
claim = await self.resolve('lbry://foo')
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
self.assertNotIn('signing_channel', claim)
self.assertEqual(claim['value']['languages'], ['uk-UA'])
@@ -1411,22 +1622,22 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '3.9998585')
# verify that the incoming tip is marked correctly as is_tip=True in account1
txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items']
txs = await self.transaction_list(account_id=self.account.id)
self.assertEqual(len(txs[0]['support_info']), 1)
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[0]['support_info'][0]['is_tip'])
self.assertFalse(txs[0]['support_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '1.0')
self.assertEqual(txs[0]['fee'], '0.0')
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = (await self.out(
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
))['items']
txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '-1.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
@@ -1442,14 +1653,24 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '1.999717')
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items']
txs2 = await self.transaction_list(wallet_id='wallet2')
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertFalse(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '0.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
# abandoning the tip increases balance and shows tip as spent
await self.support_abandon(claim_id)
await self.assertBalance(self.account, '4.979662')
txs = await self.transaction_list(account_id=self.account.id)
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(len(txs[1]['support_info']), 1)
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
class CollectionCommands(CommandTestCase):
+21 -12
View File
@@ -1,5 +1,4 @@
import logging
import os
import asyncio
import lbry
@@ -29,15 +28,22 @@ class NetworkTests(IntegrationTestCase):
'pruning': None,
'description': '',
'payment_address': '',
'daily_fee': 0,
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'donation_address': '',
'daily_fee': '0',
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
address = (await self.account.get_addresses(limit=1))[0]
os.environ.update({
'DESCRIPTION': 'Fastest server in the west.',
'DONATION_ADDRESS': address,
'DAILY_FEE': '42'})
await self.conductor.spv_node.start(self.conductor.blockchain_node)
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
self.conductor.blockchain_node,
extraconf={
'DESCRIPTION': 'Fastest server in the west.',
'PAYMENT_ADDRESS': payment_address,
'DONATION_ADDRESS': donation_address,
'DAILY_FEE': '42'
}
)
await self.ledger.network.on_connected.first
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
@@ -47,9 +53,12 @@ class NetworkTests(IntegrationTestCase):
'protocol_min': '0.54.0',
'pruning': None,
'description': 'Fastest server in the west.',
'payment_address': address,
'daily_fee': 42,
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'payment_address': payment_address,
'donation_address': donation_address,
'daily_fee': '42',
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
@@ -148,7 +148,7 @@ class PurchaseCommandTests(CommandTestCase):
url = result[0]['canonical_url']
resolve = await self.resolve(url)
self.assertEqual(result[0]['claim_id'], resolve[url]['purchase_receipt']['claim_id'])
self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id'])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
await self.daemon.jsonrpc_get('lbry://a')
@@ -12,7 +12,7 @@ from lbry.crypto.base58 import Base58
class BaseResolveTestCase(CommandTestCase):
async def assertResolvesToClaimId(self, name, claim_id):
other = (await self.resolve(name))[name]
other = await self.resolve(name)
if claim_id is None:
self.assertIn('error', other)
self.assertEqual(other['error']['name'], 'NOT_FOUND')
@@ -29,23 +29,20 @@ class ResolveCommand(BaseResolveTestCase):
# resolving a channel @abc
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertEqual(response['lbry://@abc']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['value_type'], 'channel')
self.assertEqual(response['lbry://@abc']['meta']['claims_in_channel'], 0)
self.assertEqual(response['name'], '@abc')
self.assertEqual(response['value_type'], 'channel')
self.assertEqual(response['meta']['claims_in_channel'], 0)
await self.stream_create('foo', '0.01', channel_id=channel_id)
await self.stream_create('foo2', '0.01', channel_id=channel_id)
# resolving a channel @abc with some claims in it
response['lbry://@abc']['confirmations'] += 2
response['lbry://@abc']['meta']['claims_in_channel'] = 2
response['confirmations'] += 2
response['meta']['claims_in_channel'] = 2
self.assertEqual(response, await self.resolve('lbry://@abc'))
# resolving claim foo within channel @abc
response = await self.resolve('lbry://@abc/foo')
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
claim = response['lbry://@abc/foo']
claim = await self.resolve('lbry://@abc/foo')
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
@@ -60,13 +57,12 @@ class ResolveCommand(BaseResolveTestCase):
)
# resolving claim foo by itself
self.assertEqual(claim, (await self.resolve('lbry://foo'))['lbry://foo'])
self.assertEqual(claim, await self.resolve('lbry://foo'))
# resolving from the given permanent url
permanent_url = response['lbry://@abc/foo']['permanent_url']
self.assertEqual(claim, (await self.resolve(permanent_url))[permanent_url])
self.assertEqual(claim, await self.resolve(claim['permanent_url']))
# resolving multiple at once
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
response = await self.out(self.daemon.jsonrpc_resolve(['lbry://foo', 'lbry://foo2']))
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
claim = response['lbry://foo2']
self.assertEqual(claim['name'], 'foo2')
@@ -81,7 +77,7 @@ class ResolveCommand(BaseResolveTestCase):
# resolve handles invalid data
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
await self.generate(1)
response = await self.resolve("lbry://gibberish")
response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish"))
self.assertSetEqual({'lbry://gibberish'}, set(response))
claim = response['lbry://gibberish']
self.assertEqual(claim['name'], 'gibberish')
@@ -89,10 +85,10 @@ class ResolveCommand(BaseResolveTestCase):
# resolve retries
await self.conductor.spv_node.stop()
resolving_future = asyncio.ensure_future(self.resolve('foo'))
resolve_task = asyncio.create_task(self.resolve('foo'))
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.on_ready.first
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
self.assertIsNotNone((await resolve_task)['claim_id'])
async def test_winning_by_effective_amount(self):
# first one remains winner unless something else changes
@@ -182,20 +178,18 @@ class ResolveCommand(BaseResolveTestCase):
# Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {
'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}
}
})
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
self.assertEqual(
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}},
await self.resolve('lbry://@abc/on-channel-claim')
)
response = await self.resolve('lbry://on-channel-claim')
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = (await self.resolve(direct_uri))[direct_uri]
response = await self.resolve(direct_uri)
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
await self.stream_abandon(claim_id=orphan_claim_id)
@@ -205,7 +199,7 @@ class ResolveCommand(BaseResolveTestCase):
valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id'])
# resolves normally
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
self.assertTrue(response['is_channel_signature_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.stream_create(
@@ -214,8 +208,8 @@ class ResolveCommand(BaseResolveTestCase):
# it still resolves! but to the older claim
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
self.assertEqual(response[uri]['txid'], valid_claim['txid'])
self.assertTrue(response['is_channel_signature_valid'])
self.assertEqual(response['txid'], valid_claim['txid'])
claims = await self.claim_search(name='on-channel-claim')
self.assertEqual(2, len(claims))
self.assertEqual(
@@ -235,8 +229,8 @@ class ResolveCommand(BaseResolveTestCase):
r1 = await self.resolve(f'lbry://{one}')
r2 = await self.resolve(f'lbry://{two}')
self.assertEqual(winner_id, r1[f'lbry://{one}']['claim_id'])
self.assertEqual(winner_id, r2[f'lbry://{two}']['claim_id'])
self.assertEqual(winner_id, r1['claim_id'])
self.assertEqual(winner_id, r2['claim_id'])
async def test_resolve_old_claim(self):
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
@@ -249,7 +243,7 @@ class ResolveCommand(BaseResolveTestCase):
await self.confirm_tx(tx.id)
response = await self.resolve('@olds/example')
self.assertTrue(response['@olds/example']['is_channel_signature_valid'])
self.assertTrue(response['is_channel_signature_valid'])
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
tx = await Transaction.claim_create(
@@ -260,16 +254,14 @@ class ResolveCommand(BaseResolveTestCase):
await self.confirm_tx(tx.id)
response = await self.resolve('bad_example')
self.assertFalse(response['bad_example']['is_channel_signature_valid'])
response = await self.resolve('@olds/bad_example')
self.assertEqual(response, {
'@olds/bad_example': {
'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "@olds/bad_example".',
}
}
})
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual(
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "@olds/bad_example".',
}},
await self.resolve('@olds/bad_example')
)
class ResolveAfterReorg(BaseResolveTestCase):
@@ -289,36 +281,36 @@ class ResolveAfterReorg(BaseResolveTestCase):
channel_id = self.get_claim_id(
await self.channel_create(channel_name, '0.01')
)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertNotIn('error', await self.resolve(channel_name))
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertNotIn('error', await self.resolve(channel_name))
stream_name = 'foo'
stream_id = self.get_claim_id(
await self.stream_create(stream_name, '0.01', channel_id=channel_id)
)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(stream_name))
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(stream_name))
await self.support_create(stream_id, '0.01')
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(stream_name))
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(stream_name))
await self.stream_abandon(stream_id)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
self.assertNotIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
await self.channel_abandon(channel_id)
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
self.assertIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
await self.reorg(206)
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
self.assertIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
def generate_signed_legacy(address: bytes, output: Output):
@@ -2,8 +2,10 @@ import asyncio
import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.testcase import IntegrationTestCase
from lbry.testcase import IntegrationTestCase, CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
class TestSessions(IntegrationTestCase):
@@ -49,3 +51,46 @@ class TestSegwitServer(IntegrationTestCase):
async def test_at_least_it_starts(self):
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
wallet_pay_service.payment_period = 1
# only starts with a positive max key fee
wallet_pay_service.max_fee = "0.0"
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
self.assertFalse(wallet_pay_service.running)
wallet_pay_service.max_fee = "1.0"
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
self.assertTrue(wallet_pay_service.running)
await wallet_pay_service.stop()
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
address = await self.blockchain.get_raw_change_address()
_, history = await self.ledger.get_local_status_and_history(address)
self.assertEqual(history, [])
node = SPVNode(self.conductor.spv_module, node_number=2)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.1"})
self.addCleanup(node.stop)
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
+9 -11
View File
@@ -40,7 +40,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And is the channel resolvable and empty?
response = await self.resolve('lbry://@spam')
self.assertEqual(response['lbry://@spam']['value_type'], 'channel')
self.assertEqual(response['value_type'], 'channel')
# "What goes well with spam?" ponders Chris...
# "A hovercraft with eels!" he exclaims.
@@ -64,7 +64,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# Also checks that his new story can be found on the blockchain before
# giving the link to all his friends.
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(response['lbry://@spam/hovercraft']['value_type'], 'stream')
self.assertEqual(response['value_type'], 'stream')
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
await self.generate(5)
@@ -84,13 +84,12 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(abandon['txid'])
# And now checks that the claim doesn't resolve anymore.
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(
response['lbry://@spam/hovercraft'],
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@spam/hovercraft".'
}}
}},
await self.resolve('lbry://@spam/hovercraft')
)
# After abandoning he just waits for his LBCs to be returned to his account
@@ -140,8 +139,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And check if his support showed up
resolve_result = await self.resolve(uri)
# It obviously did! Because, blockchain baby \O/
self.assertEqual(resolve_result[uri]['amount'], '1.0')
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.2')
self.assertEqual(resolve_result['amount'], '1.0')
self.assertEqual(resolve_result['meta']['effective_amount'], '1.2')
await self.generate(5)
# Now he also wanted to support the original creator of the Award Winning Novel
@@ -154,7 +153,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And again checks if it went to the just right place
resolve_result = await self.resolve(uri)
# Which it obviously did. Because....?????
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.5')
self.assertEqual(resolve_result['meta']['effective_amount'], '1.5')
await self.generate(5)
# Seeing the ravishing success of his novel Chris adds support to his claim too
@@ -188,11 +187,10 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(abandon['txid'])
# He them checks that the claim doesn't resolve anymore.
response = await self.resolve(uri)
self.assertEqual(
response[uri],
{'error': {
'name': 'NOT_FOUND',
'text': f'Could not find claim at "{uri}".'
}}
}},
await self.resolve(uri)
)
+2 -2
View File
@@ -7,7 +7,7 @@ from lbry.extras import cli
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
@@ -22,7 +22,7 @@ class CLIIntegrationTest(AsyncioTestCase):
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)
@@ -278,7 +278,9 @@ class CommentCommands(CommandTestCase):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
empty_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
self.assertEqual(0, len(empty_list['items']))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
@@ -296,66 +298,41 @@ class CommentCommands(CommandTestCase):
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(2, len(comments))
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
comment='Anonymous comment'
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
async def test02_unsigned_comment_list(self):
stream = (await self.stream_create())['outputs'][0]
comments = []
num_items = 28
for i in range(num_items):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(comment_list['total_items'], num_items)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 0)
async def test03_signed_comments_list(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
self.assertIn('comment_id', comment)
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
comment_temp = comments.pop()
self.assertEqual(comment['comment'], comment_temp['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
@@ -366,12 +343,14 @@ class CommentCommands(CommandTestCase):
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], abandoned)
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
@@ -436,12 +415,12 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test06_comment_list_test(self):
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=claim_id,
@@ -451,44 +430,48 @@ class CommentCommands(CommandTestCase):
owner_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth_id,
channel_id=moth['claim_id'],
)
other_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
anon_comment = await self.daemon.jsonrpc_comment_create(
claim_id=claim_id,
comment='Anonymous comment'
)
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
for field in list_fields:
self.assertIn(field, normal_list)
self.assertEqual(normal_list['total_items'], 4)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(normal_list)
)
self.assertEqual(normal_list['total_items'], 3)
self.assertTrue(normal_list['has_hidden_comments'])
for i, cmnt in enumerate(all_comments):
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(hidden)
)
self.assertTrue(hidden['has_hidden_comments'])
for field in list_fields:
self.assertIn(field, hidden)
self.assertEqual(hidden['total_items'], 1)
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
for field in list_fields:
self.assertIn(field, visible)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(visible)
)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
for field in list_fields:
self.assertIn(field, valid_list)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(valid_list)
)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
self.assertEqual(len(valid_list['items']), len(normal_list['items']))
async def test07_edit_comments(self):
luda = (await self.channel_create('@Ludacris'))['outputs'][0]
@@ -537,18 +520,3 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
# editing an anonymous comment
anon_comment = await self.daemon.jsonrpc_comment_create(
comment='fast and furiouuuuuus',
claim_id=claim_id
)
anon_cid = anon_comment.get('comment_id')
self.assertIsNotNone(anon_cid)
with self.assertRaises(ValueError):
await self.daemon.jsonrpc_comment_update(
comment='drift drift drift',
comment_id=anon_cid
)
@@ -1,3 +1,4 @@
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.testcase import CommandTestCase
@@ -11,6 +12,7 @@ class AddressManagement(CommandTestCase):
self.assertItemCount(single, 1)
self.assertEqual(single['items'][0], addresses['items'][11])
class SettingsManagement(CommandTestCase):
async def test_settings(self):
@@ -23,3 +25,16 @@ class SettingsManagement(CommandTestCase):
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
# tests that changing share_usage_data propagates to the relevant properties
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
loggly = get_loggly_handler(self.daemon.conf)
self.addCleanup(loggly.close)
self.assertFalse(self.daemon.analytics_manager.enabled)
self.assertFalse(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', True)
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
self.assertTrue(self.daemon.analytics_manager.enabled)
self.assertTrue(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', False)
+7 -2
View File
@@ -93,6 +93,11 @@ class TranscodeValidation(ClaimTestCase):
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_max_bit_rate(self):
self.conf.video_bitrate_maximum = 100
with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
async def test_video_format(self):
file_name = self.make_name("bad_video_format_1")
if not file_name.exists():
@@ -130,7 +135,7 @@ class TranscodeValidation(ClaimTestCase):
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "ogg")
self.assertEqual(extension, "ogv")
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
extension = self.analyzer._get_best_container_extension(scan_data, "")
@@ -143,7 +148,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("webm", extension)
extension = self.analyzer._get_best_container_extension("", "libtheora")
self.assertEqual("ogg", extension)
self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/"
@@ -27,6 +27,7 @@ class TestComponentManager(AsyncioTestCase):
components.HashAnnouncerComponent,
components.PeerProtocolServerComponent,
components.StreamManagerComponent,
components.WalletServerPaymentsComponent
]
]
self.component_manager = ComponentManager(Config())
+4
View File
@@ -122,6 +122,10 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
class TestStreamManager(BlobExchangeTestBase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.client_config.share_usage_data = True
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
file_path = os.path.join(self.server_dir, "test_file")
with open(file_path, 'wb') as f:
+12 -6
View File
@@ -64,8 +64,9 @@ class CLILoggingTest(AsyncioTestCase):
self.assertTrue(log.isEnabledFor(logging.INFO))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 1)
self.assertEqual(len(log.handlers), 2)
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
self.assertFalse(log.handlers[1].enabled)
async with get_logger(["start", "--verbose"]) as log:
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
@@ -80,16 +81,22 @@ class CLILoggingTest(AsyncioTestCase):
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
async def test_loggly(self):
async with get_logger(["start"], share_usage_data=False) as log:
self.assertEqual(len(log.getChild("lbry").handlers), 2) # file and console
async with get_logger(["start"]) as log: # default share_usage_data=True
async with get_logger(["start"]) as log: # default share_usage_data=False
log = log.getChild("lbry")
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=True) as log:
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
async with get_logger(["start"], share_usage_data=True) as log: # explicit share_usage_data=True
self.assertTrue(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async def test_quiet(self):
async with get_logger(["start"]) as log: # default is loud
@@ -100,7 +107,6 @@ class CLILoggingTest(AsyncioTestCase):
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 2)
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
self.assertIsNot(type(log.handlers[1]), logging.StreamHandler)
class CLITest(AsyncioTestCase):