Compare commits

..
Author SHA1 Message Date
Victor Shyba abfe0be81e do a reatriable call so the service doesnt die 2020-02-21 17:26:31 -03:00
Victor Shyba 820b7a6753 check its broadcasted 2020-02-21 17:26:04 -03:00
35 changed files with 379 additions and 778 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.64.0"
__version__ = "0.60.1"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+3 -8
View File
@@ -467,17 +467,12 @@ class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
'libx264 -crf 18 -vf "format=yuv420p"')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 160k')
'aac -b:a 192k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
class CLIConfig(TranscodeConfig):
+4
View File
@@ -3,6 +3,7 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@@ -261,6 +262,9 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
-1
View File
@@ -121,7 +121,6 @@ class WalletComponent(Component):
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
'servers': [
{
'host': session.server[0],
+69 -114
View File
@@ -21,10 +21,9 @@ from prometheus_client import generate_latest as prom_generate_latest
from google.protobuf.message import DecodeError
from lbry.wallet import (
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
Transaction, Output, Input, Account, database
Transaction, Output, Input, Account
)
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES
from lbry import utils
from lbry.conf import Config, Setting, NOT_SET
@@ -2168,20 +2167,19 @@ class Daemon(metaclass=JSONRPCServerType):
"""
@requires(WALLET_COMPONENT)
def jsonrpc_claim_list(self, claim_type=None, **kwargs):
def jsonrpc_claim_list(
self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
"""
List my stream and channel claims.
Usage:
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...]
claim_list [--claim_type=<claim_type>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
Options:
--claim_type=<claim_type> : (str or list) claim type: channel, stream, repost, collection
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--claim_type=<claim_type> : (str) claim type: channel, stream, repost, collection
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -2190,9 +2188,15 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
kwargs['unspent'] = True
return self.jsonrpc_txo_list(**kwargs)
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
claims = account.get_claims
claim_count = account.get_claim_count
else:
claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts)
claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
@@ -2695,18 +2699,15 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_channel_list(self, *args, **kwargs):
def jsonrpc_channel_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
"""
List my channel claims.
Usage:
channel_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--name=<name>...] [--claim_id=<claim_id>...]
[--page=<page>] [--page_size=<page_size>] [--resolve]
Options:
--name=<name> : (str or list) channel name
--claim_id=<claim_id> : (str or list) channel id
--account_id=<account_id> : (str) id of the account to use
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -2715,9 +2716,15 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = 'channel'
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
channels = account.get_channels
channel_count = account.get_channel_count
else:
channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts)
channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(channels, channel_count, page, page_size, resolve=resolve)
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None):
@@ -3135,9 +3142,8 @@ class Daemon(metaclass=JSONRPCServerType):
f"Use --allow-duplicate-name flag to override."
)
file_path = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
tx = await Transaction.claim_create(
@@ -3434,18 +3440,15 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_stream_list(self, *args, **kwargs):
def jsonrpc_stream_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
"""
List my stream claims.
Usage:
stream_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--name=<name>...] [--claim_id=<claim_id>...]
[--page=<page>] [--page_size=<page_size>] [--resolve]
[--page=<page>] [--page_size=<page_size>] [--resolve]
Options:
--name=<name> : (str or list) stream name
--claim_id=<claim_id> : (str or list) stream id
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -3454,9 +3457,15 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = 'stream'
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
streams = account.get_streams
stream_count = account.get_stream_count
else:
streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts)
stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(streams, stream_count, page, page_size, resolve=resolve)
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
DHT_COMPONENT, DATABASE_COMPONENT)
@@ -3902,19 +3911,15 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
def jsonrpc_support_list(self, *args, tips=None, **kwargs):
def jsonrpc_support_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
"""
List supports and tips in my control.
Usage:
support_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
[--name=<name>...] [--claim_id=<claim_id>...] [--tips]
[--page=<page>] [--page_size=<page_size>]
Options:
--name=<name> : (str or list) claim name
--claim_id=<claim_id> : (str or list) claim id
--tips : (bool) only show tips (is_received=true)
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -3922,12 +3927,15 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = 'support'
kwargs['unspent'] = True
kwargs['include_is_received'] = True
if tips is True:
kwargs['is_received'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
supports = account.get_supports
support_count = account.get_support_count
else:
supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts)
support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(supports, support_count, page, page_size)
@requires(WALLET_COMPONENT)
async def jsonrpc_support_abandon(
@@ -4094,71 +4102,12 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.wallet_manager.get_transaction(txid)
TXO_DOC = """
List transaction outputs.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_txo_list(
self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin
claim_id=None, name=None, unspent=False,
include_is_received=False, is_received=None, is_not_received=None,
wallet_id=None, page=None, page_size=None, resolve=False):
"""
List my transaction outputs.
Usage:
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
[--include_is_received] [--is_received] [--is_not_received]
[--wallet_id=<wallet_id>] [--include_is_received] [--is_received]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
Options:
--type=<type> : (str or list) claim type: stream, channel, support,
purchase, collection, repost, other
--txid=<txid> : (str or list) transaction id of outputs
--unspent : (bool) hide spent outputs, show only unspent ones
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--include_is_received : (bool) calculate the is_received property and
include in output, this happens automatically if you
use the --is_received or --is_not_received filters
--is_received : (bool) only return txos sent from others to this account
--is_not_received : (bool) only return txos created by this account
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--resolve : (bool) resolves each claim to provide additional metadata
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
claims = account.get_txos
claim_count = account.get_txo_count
else:
claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts)
claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts)
constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received}
if is_received is True:
constraints['is_received'] = True
elif is_not_received is True:
constraints['is_received'] = False
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
database.constrain_single_or_list(constraints, 'claim_name', name)
return paginate_rows(claims, claim_count, page, page_size, **constraints)
UTXO_DOC = """
Unspent transaction management.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_utxo_list(self, *args, **kwargs):
def jsonrpc_utxo_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
"""
List unspent transaction outputs
@@ -4174,9 +4123,15 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = ['other', 'purchase']
kwargs['unspent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
if account_id:
account = wallet.get_account_or_error(account_id)
utxos = account.get_utxos
utxo_count = account.get_utxo_count
else:
utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts)
utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(utxos, utxo_count, page, page_size)
@requires(WALLET_COMPONENT)
async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None):
@@ -4629,7 +4584,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
comment_create (<comment> | --comment=<comment>)
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
(--channel_id=<channel_id> | --channel_name=<channel_name>)
[--channel_id=<channel_id>] [--channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
@@ -4657,23 +4612,23 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_body = {
'comment': comment.strip(),
'claim_id': claim_id,
'parent_id': parent_id,
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
}
comment_client.sign_comment(comment_body, channel)
channel = await self.get_channel_or_none(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
if channel:
comment_body.update({
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
if 'signature' in response:
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
return response
@requires(WALLET_COMPONENT)
@@ -25,8 +25,6 @@ def encode_txo_doc():
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_received': "true if txo was sent from external account to this account",
'is_spent': "true if txo is spent",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'purchase'",
'name': "when type is 'claim' or 'support', this is the claim name",
@@ -170,10 +168,6 @@ class JSONResponseEncoder(JSONEncoder):
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_received is not None:
output['is_received'] = txo.is_received
if txo.is_spent is not None:
output['is_spent'] = txo.is_spent
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
+34 -57
View File
@@ -6,42 +6,25 @@ import pathlib
import re
import shlex
import shutil
import platform
import lbry.utils
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
DISABLED = platform.system() == "Windows"
class VideoFileAnalyzer:
def _replace_or_pop_env(self, variable):
if variable + '_ORIG' in self._env_copy:
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
else:
self._env_copy.pop(variable, None)
def __init__(self, conf: TranscodeConfig):
self._conf = conf
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
self._env_copy = dict(os.environ)
if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment:
self._replace_or_pop_env('LD_LIBRARY_PATH')
async def _execute(self, command, arguments):
if DISABLED:
return "Disabled on Windows", -1
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
)
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
@@ -49,10 +32,10 @@ class VideoFileAnalyzer:
try:
version, code = await self._execute(name, "-version")
except Exception as e:
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
code = -1
version = str(e)
version = ""
if code != 0 or not version.startswith(name):
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
return version
@@ -62,7 +45,7 @@ class VideoFileAnalyzer:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
@@ -112,21 +95,24 @@ class VideoFileAnalyzer:
return ""
def _verify_bitrate(self, scan_data: json, file_path):
bit_rate_max = float(self._conf.video_bitrate_maximum)
if bit_rate_max <= 0:
@staticmethod
def _verify_bitrate(scan_data: json):
if "bit_rate" not in scan_data["format"]:
return ""
if "bit_rate" in scan_data["format"]:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
bit_rate = float(scan_data["format"]["bit_rate"])
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
pixels = -1.0
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
pieces = stream["r_frame_rate"].split('/', 1)
frame_rate = float(pieces[0]) if len(pieces) == 1 \
else float(pieces[0]) / float(pieces[1])
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
if pixels > 0.0 and pixels / bit_rate < 3.0:
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
return ""
@@ -190,9 +176,6 @@ class VideoFileAnalyzer:
# https://developers.google.com/media/vp9/settings/vod/
return int(-0.011 * height + 40)
def _get_video_scaler(self):
return self._conf.video_scaler
async def _get_video_encoder(self, scan_data):
# use what the user said if it's there:
# if it's not there, use h264 if we can because it's way faster than the others
@@ -272,12 +255,12 @@ class VideoFileAnalyzer:
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogv"
return "ogg"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogv"
return "ogg"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
@@ -289,35 +272,31 @@ class VideoFileAnalyzer:
scan_data = json.loads(result)
except Exception as e:
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
raise ValueError(f'Absent or unreadable video file: {file_path}')
if validate:
raise Exception(f'Invalid video file: {file_path}')
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
return
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
raise ValueError(f'Assuming image file at: {file_path}')
if "format" not in scan_data:
if validate:
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return
return scan_data
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
async def verify_or_repair(self, validate, repair, file_path):
if not validate and not repair:
return file_path
await self._verify_ffmpeg_installed()
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path
raise
scan_data = await self._get_scan_data(validate, file_path)
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
bitrate_msg = self._verify_bitrate(scan_data)
video_msg = self._verify_video_encoding(scan_data)
audio_msg = self._verify_audio_encoding(scan_data)
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
@@ -343,8 +322,6 @@ class VideoFileAnalyzer:
if video_msg or bitrate_msg:
video_encoder = await self._get_video_encoder(scan_data)
transcode_command.append(video_encoder)
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
transcode_command.append(self._get_video_scaler())
else:
transcode_command.append("copy")
-1
View File
@@ -148,7 +148,6 @@ types_map = {
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'),
'.ogg': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
+1 -3
View File
@@ -38,9 +38,7 @@ class Censor:
for claim_hash, lookup in (
(row['claim_hash'], self.streams),
(row['claim_hash'], self.channels),
(row['channel_hash'], self.channels),
(row['reposted_claim_hash'], self.streams),
(row['reposted_claim_hash'], self.channels)):
(row['channel_hash'], self.channels)):
censoring_channel_hash = lookup.get(claim_hash)
if censoring_channel_hash:
was_censored = True
+2 -15
View File
@@ -548,15 +548,8 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
)
async def support_abandon(self, *args, confirm=True, **kwargs):
if 'blocking' not in kwargs:
kwargs['blocking'] = False
return await self.confirm_and_render(
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
)
async def resolve(self, uri):
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
return await self.out(self.daemon.jsonrpc_resolve(uri))
async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
@@ -564,9 +557,6 @@ class CommandTestCase(IntegrationTestCase):
async def file_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
async def txo_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
@@ -576,12 +566,9 @@ class CommandTestCase(IntegrationTestCase):
async def channel_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
async def transaction_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
@staticmethod
def get_claim_id(tx):
return tx['outputs'][0]['claim_id']
def assertItemCount(self, result, count): # pylint: disable=invalid-name
self.assertEqual(count, result['total_items'])
self.assertEqual(result['total_items'], count)
-6
View File
@@ -4,7 +4,6 @@ import datetime
import random
import socket
import string
import sys
import json
import typing
import asyncio
@@ -277,8 +276,3 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
return response['data']['ip']
except Exception:
return
def is_running_from_bundle():
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
-6
View File
@@ -467,12 +467,6 @@ class Account:
'max_receiving_gap': receiving_gap,
}
def get_txos(self, **constraints):
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
def get_txo_count(self, **constraints):
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
def get_utxos(self, **constraints):
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
+4 -9
View File
@@ -6,7 +6,6 @@ COIN = 100*CENT
TIMEOUT = 30.0
TXO_TYPES = {
"other": 0,
"stream": 1,
"channel": 2,
"support": 3,
@@ -15,13 +14,9 @@ TXO_TYPES = {
"repost": 6,
}
CLAIM_TYPE_NAMES = [
'stream',
'channel',
'collection',
'repost',
]
CLAIM_TYPES = [
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
TXO_TYPES['stream'],
TXO_TYPES['channel'],
TXO_TYPES['collection'],
TXO_TYPES['repost'],
]
+28 -65
View File
@@ -107,9 +107,7 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
if not key:
sql.append(constraint)
continue
if key.startswith('$$'):
col, key = col[2:], key[1:]
elif key.startswith('$'):
if key.startswith('$'):
values[key] = constraint
continue
if key.endswith('__not'):
@@ -223,19 +221,6 @@ def rows_to_dict(rows, fields):
return []
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
if value is not None:
if isinstance(value, list):
value = [convert(v) for v in value]
if len(value) == 1:
constraints[column] = value[0]
elif len(value) > 1:
constraints[f"{column}__in"] = value
else:
constraints[column] = convert(value)
return constraints
class SQLiteMixin:
SCHEMA_VERSION: Optional[str] = None
@@ -365,7 +350,6 @@ class Database(SQLiteMixin):
create index if not exists txo_txid_idx on txo (txid);
create index if not exists txo_address_idx on txo (address);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_claim_name_idx on txo (claim_name);
create index if not exists txo_txo_type_idx on txo (txo_type);
"""
@@ -490,21 +474,20 @@ class Database(SQLiteMixin):
async def select_transactions(self, cols, accounts=None, **constraints):
if not {'txid', 'txid__in'}.intersection(constraints):
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
where, values = constraints_to_sql({
'$$account_address.account__in': [a.public_key.address for a in accounts]
constraints.update({
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
})
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
where = f" WHERE account_address.account IN ({account_values})"
constraints['txid__in'] = f"""
SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where}
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
UNION
SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where}
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
"""
constraints.update(values)
return await self.db.execute_fetchall(
*query(f"SELECT {cols} FROM tx", **constraints)
)
TXO_NOT_MINE = Output(None, None, is_my_account=False)
async def get_transactions(self, wallet=None, **constraints):
tx_rows = await self.select_transactions(
'txid, raw, height, position, is_verified',
@@ -555,7 +538,7 @@ class Database(SQLiteMixin):
if _txo:
txo.update_annotations(_txo)
else:
txo.update_annotations(self.TXO_NOT_MINE)
txo.update_annotations(None)
for tx in txs:
txos = tx.outputs
@@ -577,33 +560,13 @@ class Database(SQLiteMixin):
if txs:
return txs[0]
async def select_txos(self, cols, wallet=None, include_is_received=False, **constraints):
if include_is_received:
assert wallet is not None, 'cannot use is_recieved filter without wallet argument'
account_in_wallet, values = constraints_to_sql({
'$$account__in#is_received': [a.public_key.address for a in wallet.accounts]
})
cols += f""",
NOT EXISTS(
SELECT 1 FROM txi JOIN account_address USING (address)
WHERE txi.txid=txo.txid AND {account_in_wallet}
) as is_received
"""
constraints.update(values)
async def select_txos(self, cols, **constraints):
sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)"
if 'accounts' in constraints:
sql += " JOIN account_address USING (address)"
return await self.db.execute_fetchall(*query(sql, **constraints))
@staticmethod
def constrain_unspent(constraints):
constraints['is_reserved'] = False
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, **constraints):
include_is_received = include_is_received or 'is_received' in constraints
if unspent:
self.constrain_unspent(constraints)
async def get_txos(self, wallet=None, no_tx=False, **constraints):
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
if 'order_by' not in constraints:
constraints['order_by'] = [
@@ -614,9 +577,9 @@ class Database(SQLiteMixin):
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
select group_concat(account||"|"||chain) from account_address
where account_address.address=txo.address
), exists(select 1 from txi where txi.txoid=txo.txoid)
)
""",
wallet=wallet, include_is_received=include_is_received, **constraints
**constraints
)
txos = []
txs = {}
@@ -636,9 +599,6 @@ class Database(SQLiteMixin):
txo = txs[row[0]].outputs[row[5]]
row_accounts = dict(a.split('|') for a in row[8].split(','))
account_match = set(row_accounts) & my_accounts
txo.is_spent = bool(row[9])
if include_is_received:
txo.is_received = bool(row[10])
if account_match:
txo.is_my_account = True
txo.is_change = row_accounts[account_match.pop()] == '1'
@@ -674,28 +634,33 @@ class Database(SQLiteMixin):
return txos
async def get_txo_count(self, unspent=False, **constraints):
constraints['include_is_received'] = 'is_received' in constraints
async def get_txo_count(self, **constraints):
constraints.pop('resolve', None)
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
if unspent:
self.constrain_unspent(constraints)
count = await self.select_txos('count(*)', **constraints)
return count[0][0]
@staticmethod
def constrain_utxo(constraints):
constraints['is_reserved'] = False
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
def get_utxos(self, **constraints):
return self.get_txos(unspent=True, **constraints)
self.constrain_utxo(constraints)
return self.get_txos(**constraints)
def get_utxo_count(self, **constraints):
return self.get_txo_count(unspent=True, **constraints)
self.constrain_utxo(constraints)
return self.get_txo_count(**constraints)
async def get_balance(self, wallet=None, accounts=None, **constraints):
assert wallet or accounts, \
"'wallet' or 'accounts' constraints required to calculate balance"
constraints['accounts'] = accounts or wallet.accounts
self.constrain_unspent(constraints)
self.constrain_utxo(constraints)
balance = await self.select_txos('SUM(amount)', **constraints)
return balance[0][0] or 0
@@ -778,13 +743,11 @@ class Database(SQLiteMixin):
@staticmethod
def constrain_claims(constraints):
if {'txo_type', 'txo_type__in'}.intersection(constraints):
return
claim_types = constraints.pop('claim_type', None)
if claim_types:
constrain_single_or_list(
constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x]
)
if isinstance(claim_types, str) and claim_types:
claim_types = [claim_types]
if isinstance(claim_types, list) and claim_types:
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
else:
constraints['txo_type__in'] = CLAIM_TYPES
+8 -26
View File
@@ -262,15 +262,6 @@ class Ledger(metaclass=LedgerRegistry):
self.constraint_spending_utxos(constraints)
return self.db.get_utxo_count(**constraints)
async def get_txos(self, resolve=False, **constraints):
txos = await self.db.get_txos(**constraints)
if resolve:
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
return txos
def get_txo_count(self, **constraints):
return self.db.get_txo_count(**constraints)
def get_transactions(self, **constraints):
return self.db.get_transactions(**constraints)
@@ -744,13 +735,10 @@ class Ledger(metaclass=LedgerRegistry):
async def _resolve_for_local_results(self, accounts, txos):
results = []
response = await self.resolve(
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
)
response = await self.resolve(accounts, [txo.permanent_url for txo in txos])
for txo in txos:
resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None
resolved = response[txo.permanent_url]
if isinstance(resolved, Output):
resolved.update_annotations(txo)
results.append(resolved)
else:
if isinstance(resolved, dict) and 'error' in resolved:
@@ -855,8 +843,7 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
for txo in tx.my_update_outputs:
if is_my_inputs: # updating my own claim
@@ -875,8 +862,7 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
else: # someone sent us their claim
item['update_info'].append({
@@ -885,8 +871,7 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
for txo in tx.my_support_outputs:
item['support_info'].append({
@@ -896,8 +881,7 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': not is_my_inputs,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
if is_my_inputs:
for txo in tx.other_support_outputs:
@@ -908,8 +892,7 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
for txo in tx.my_abandon_outputs:
item['abandon_info'].append({
@@ -926,8 +909,7 @@ class Ledger(metaclass=LedgerRegistry):
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.purchased_claim_id,
'nout': txo.position,
'is_spent': txo.is_spent,
'nout': txo.position
})
history.append(item)
return history
-4
View File
@@ -162,7 +162,6 @@ class Network:
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
@@ -190,20 +189,17 @@ class Network:
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
async def start(self):
+1 -5
View File
@@ -1,7 +1,6 @@
import time
import asyncio
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import lbry
from lbry.schema.claim import Claim
@@ -146,7 +145,6 @@ class BlockProcessor:
self.blocks_event = asyncio.Event()
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
self.logger = class_logger(__name__, self.__class__.__name__)
self.executor = ThreadPoolExecutor(1)
# Meta
self.next_cache_check = 0
@@ -167,7 +165,6 @@ class BlockProcessor:
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -176,7 +173,7 @@ class BlockProcessor:
# consistent and not being updated elsewhere.
async def run_in_thread_locked():
async with self.state_lock:
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
return await asyncio.shield(run_in_thread_locked())
async def check_and_advance_blocks(self, raw_blocks):
@@ -660,7 +657,6 @@ class BlockProcessor:
self.logger.info('flushing to DB for a clean shutdown...')
await self.flush(True)
self.db.close()
self.executor.shutdown(wait=True)
def force_chain_reorg(self, count):
"""Force a reorg of the given number of blocks.
+1 -17
View File
@@ -3,7 +3,6 @@ import itertools
import json
import time
from functools import wraps
from pylru import lrucache
import aiohttp
@@ -43,8 +42,6 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(1000000)
self._block_cache = lrucache(100000)
async def close(self):
if self.connector:
@@ -222,25 +219,12 @@ class Daemon:
async def block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
if first + count < (self.cached_height() or 0) - 200:
return await self._cached_block_hex_hashes(first, count)
params_iterable = ((h, ) for h in range(first, first + count))
return await self._send_vector('getblockhash', params_iterable)
async def _cached_block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
cached = self._block_hash_cache.get((first, count))
if cached:
return cached
params_iterable = ((h, ) for h in range(first, first + count))
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
return self._block_hash_cache[(first, count)]
async def deserialised_block(self, hex_hash):
"""Return the deserialised block with the given hex hash."""
if not self._block_cache.get(hex_hash):
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
return self._block_cache[hex_hash]
return await self._send_single('getblock', (hex_hash, True))
async def raw_blocks(self, hex_hashes):
"""Return the raw binary blocks with the given hex hashes."""
+10 -32
View File
@@ -291,7 +291,6 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
not_channel_ids_binary = {
unhexlify(ncid)[::-1] for ncid in not_channel_ids
}
constraints['claim.claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
@@ -317,13 +316,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
constraints['claim.txo_hash'] = tx_hash + struct.pack('<I', nout)
if 'claim_type' in constraints:
claim_types = constraints.pop('claim_type')
if isinstance(claim_types, str):
claim_types = [claim_types]
if claim_types:
constraints['claim.claim_type__in'] = {
CLAIM_TYPES[claim_type] for claim_type in claim_types
}
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
@@ -490,14 +483,6 @@ def resolve_url(raw_url):
return channel
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
CASE WHEN claim.claim_type = {CLAIM_TYPES['repost']}
THEN claim.reposted_claim_hash
ELSE claim.claim_hash
END
"""
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
@@ -517,8 +502,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
@@ -531,7 +515,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
@@ -545,16 +529,14 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
@@ -573,17 +555,15 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$all_{attr}{i}' for i in range(len(all_items))
)
if for_count:
constraints[f'#_all_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
)
"""
else:
constraints[f'#_all_{attr}'] = f"""
{len(all_items)}=(
SELECT count(*) FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
@@ -596,16 +576,14 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
SELECT 1 FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
+4 -4
View File
@@ -136,9 +136,9 @@ class SQLDB:
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
-- claim_type filter + order by
create unique index if not exists claim_type_release_idx on claim (release_time, claim_type, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (trending_group, trending_mixed, claim_type, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (effective_amount, claim_type, claim_hash);
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_group, trending_mixed, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
-- stream_type filter + order by
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
@@ -866,7 +866,7 @@ class LBRYLevelDB(LevelDB):
super().__init__(*args, **kwargs)
path = os.path.join(self.env.db_dir, 'claims.db')
trending = []
for algorithm_name in self.env.trending_algorithms:
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
if algorithm_name in TRENDING_ALGORITHMS:
trending.append(TRENDING_ALGORITHMS[algorithm_name])
self.sql = SQLDB(
-1
View File
@@ -37,7 +37,6 @@ class Env:
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
+6 -8
View File
@@ -20,7 +20,7 @@ from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.wallet.server import util
@@ -63,7 +63,6 @@ class LevelDB:
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
@@ -140,7 +139,6 @@ class LevelDB:
def close(self):
self.utxo_db.close()
self.history.close_db()
self.executor.shutdown(wait=True)
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -408,7 +406,7 @@ class LevelDB:
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
return await asyncio.get_event_loop().run_in_executor(None, read_headers)
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -447,7 +445,7 @@ class LevelDB:
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
history = await asyncio.get_event_loop().run_in_executor(None, read_history)
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
@@ -613,7 +611,7 @@ class LevelDB:
return utxos
while True:
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
utxos = await asyncio.get_event_loop().run_in_executor(None, read_utxos)
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
@@ -666,5 +664,5 @@ class LevelDB:
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
hashX_pairs = await asyncio.get_event_loop().run_in_executor(None, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(None, lookup_utxos, hashX_pairs)
+10 -15
View File
@@ -6,13 +6,12 @@
# and warranty status of this software.
"""Mempool handling."""
import os
import asyncio
import itertools
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
import attr
@@ -106,15 +105,14 @@ class MemPool:
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
self.logger.info('beginning processing of daemon mempool. '
'This can take some time...')
start = time.perf_counter()
start = time.time()
await synchronized_event.wait()
elapsed = time.perf_counter() - start
elapsed = time.time() - start
self.logger.info(f'synced in {elapsed:.2f}s')
while True:
self.logger.info(f'{len(self.txs):,d} txs '
@@ -123,15 +121,12 @@ class MemPool:
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
try:
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
finally:
self.executor.shutdown(wait=True)
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(None, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
@@ -294,7 +289,7 @@ class MemPool:
return txs
# Thread this potentially slow operation so as not to block
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
tx_map = await asyncio.get_event_loop().run_in_executor(None, deserialize_txs)
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
-1
View File
@@ -863,7 +863,6 @@ class LBRYElectrumX(SessionBase):
'donation_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
async def server_features_async(self):
+4 -9
View File
@@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator:
class Output(InputOutput):
__slots__ = (
'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account',
'amount', 'script', 'is_change', 'is_my_account',
'channel', 'private_key', 'meta',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims',
@@ -215,16 +215,13 @@ class Output(InputOutput):
def __init__(self, amount: int, script: OutputScript,
tx_ref: TXRef = None, position: int = None,
is_change: Optional[bool] = None, is_spent: Optional[bool] = None,
is_received: Optional[bool] = None, is_my_account: Optional[bool] = None,
is_change: Optional[bool] = None, is_my_account: Optional[bool] = None,
channel: Optional['Output'] = None, private_key: Optional[str] = None
) -> None:
super().__init__(tx_ref, position)
self.amount = amount
self.script = script
self.is_change = is_change
self.is_spent = is_spent
self.is_received = is_received
self.is_my_account = is_my_account
self.channel = channel
self.private_key = private_key
@@ -237,12 +234,10 @@ class Output(InputOutput):
def update_annotations(self, annotated):
if annotated is None:
self.is_change = None
self.is_spent = None
self.is_my_account = None
self.is_change = False
self.is_my_account = False
else:
self.is_change = annotated.is_change
self.is_spent = annotated.is_spent
self.is_my_account = annotated.is_my_account
self.channel = annotated.channel if annotated else None
self.private_key = annotated.private_key if annotated else None
+2 -2
View File
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
try:
await analyzer.verify_or_repair(True, False, video_file)
print("No concerns. Ship it!")
except (FileNotFoundError, ValueError) as e:
print("Analysis failed.", str(e))
except FileNotFoundError as e:
print(str(e))
except Exception as e:
print(str(e))
transcode = input("Would you like to make a repaired clone now? [y/N] ")
+1 -1
View File
@@ -333,7 +333,7 @@ class Examples(CommandTestCase):
)
reply = await r(
'Use the parent_id param to make replies',
'You can r',
'comment', 'create',
'--comment="I have photographic evidence confirming Sasquatch exists"',
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
@@ -1,3 +1,4 @@
import asyncio
import os.path
import tempfile
import logging
@@ -262,7 +263,6 @@ class ClaimSearchCommand(ClaimTestCase):
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
chan3_id = self.get_claim_id(await self.channel_create('@chan3'))
chan4 = await self.channel_create('@chan4', '0.1')
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
@@ -285,7 +285,6 @@ class ClaimSearchCommand(ClaimTestCase):
await match([claim6, claim5, claim4, claim1], not_channel_ids=[chan1_id], claim_type='stream')
await match([claim6, claim3, claim2, claim1], not_channel_ids=[chan2_id], claim_type='stream')
await match([claim6, claim1], not_channel_ids=[chan1_id, chan2_id], claim_type='stream')
await match([claim6, claim1, chan4], not_channel_ids=[chan1_id, chan2_id])
# not_channel_ids + valid_channel_signature
await match([claim5, claim4, claim3, claim2, claim1],
@@ -398,116 +397,12 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([], text='cloud')
class TransactionCommands(ClaimTestCase):
async def test_transaction_list(self):
channel_id = self.get_claim_id(await self.channel_create())
await self.channel_update(channel_id, bid='0.5')
await self.channel_abandon(claim_id=channel_id)
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
await self.stream_abandon(claim_id=stream_id)
r = await self.transaction_list()
self.assertEqual(7, len(r))
self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id'])
self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id'])
self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id'])
self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id'])
self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id'])
self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id'])
class TransactionOutputCommands(ClaimTestCase):
async def test_txo_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
await self.channel_update(channel_id, bid='0.5')
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
# type filtering
r = await self.txo_list(type='channel')
self.assertEqual(2, len(r))
self.assertEqual('channel', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('channel', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type='stream')
self.assertEqual(2, len(r))
self.assertEqual('stream', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('stream', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type=['stream', 'channel'])
self.assertEqual(4, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(claim_id=[stream_id, channel_id])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, _, channel_name, _ = (c['name'] for c in r)
r = await self.txo_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.txo_list(name=stream_name)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=[stream_name, channel_name])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=['beef'])
self.assertEqual(0, len(r))
r = await self.txo_list()
self.assertEqual(9, len(r))
await self.stream_abandon(claim_id=stream_id)
r = await self.txo_list()
self.assertEqual(10, len(r))
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertTrue(r[0]['is_spent'])
self.assertTrue(r[1]['is_spent'])
async def test_txo_list_received_filtering(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create(claim_address=address2)
r = await self.txo_list(include_is_received=True)
self.assertEqual(2, len(r))
self.assertFalse(r[0]['is_received'])
self.assertTrue(r[1]['is_received'])
rt = await self.txo_list(is_not_received=True)
self.assertEqual(1, len(rt))
self.assertEqual(rt[0], r[0])
rf = await self.txo_list(is_received=True)
self.assertEqual(1, len(rf))
self.assertEqual(rf[0], r[1])
r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id)
self.assertEqual(1, len(r))
self.assertTrue(r[0]['is_received'])
class ClaimCommands(ClaimTestCase):
async def test_claim_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
async def test_claim_list_type_filtering(self):
await self.channel_create()
await self.stream_create()
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
@@ -520,31 +415,6 @@ class ClaimCommands(ClaimTestCase):
self.assertEqual(2, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.claim_list(claim_id=stream_id)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(claim_id=[stream_id, channel_id])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, channel_name = (c['name'] for c in r)
r = await self.claim_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.claim_list(name=stream_name)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=[stream_name, channel_name])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=['beef'])
self.assertEqual(0, len(r))
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
@@ -616,9 +486,6 @@ class ClaimCommands(ClaimTestCase):
self.assertTrue(r[0]['meta']['is_controlling'])
self.assertTrue(r[1]['meta']['is_controlling'])
# check that metadata is transfered
self.assertTrue(r[0]['is_mine'])
class ChannelCommands(CommandTestCase):
@@ -960,11 +827,8 @@ class StreamCommands(ClaimTestCase):
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
async def test_repost(self):
sql = self.conductor.spv_node.server.bp.sql
sql.execute(sql.TAG_INDEXES)
await self.channel_create('@goodies', '1.0')
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies', tags=['foo', 'gaming'])
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies')
claim_id = self.get_claim_id(tx)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
@@ -975,16 +839,6 @@ class StreamCommands(ClaimTestCase):
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
# tags are inherited (non-common / indexed tags)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['foo'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['foo'], claim_type=['stream', 'repost']), 0)
# "common" / indexed tags work too
self.assertIn('gaming', sql.TAG_INDEXES) # if this breaks, next test doesn't make sense
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['gaming'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['gaming'], claim_type=['stream', 'repost']), 2)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['gaming'], claim_type=['stream', 'repost']), 0)
await self.channel_create('@reposting-goodies', '1.0')
await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies')
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2)
@@ -1004,29 +858,25 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(search['reposted_claim']['meta']['reposted'], 2)
self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies')
resolved = await self.out(
self.daemon.jsonrpc_resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
)
resolved = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
async def test_filtering_channels_for_removing_content(self):
await self.channel_create('@some_channel', '0.1')
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
await self.channel_create('@some_channel', '1.0')
await self.stream_create('good_content', '1.1', channel_name='@some_channel', tags=['good'])
bad_content_id = self.get_claim_id(
await self.stream_create('bad_content', '0.1', channel_name='@some_channel', tags=['bad'])
await self.stream_create('bad_content', '1.1', channel_name='@some_channel', tags=['bad'])
)
filtering_channel_id = self.get_claim_id(
await self.channel_create('@filtering', '0.1')
await self.channel_create('@filtering', '1.0')
)
self.conductor.spv_node.server.db.sql.filtering_channel_hashes.add(
unhexlify(filtering_channel_id)[::-1]
)
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_streams))
await self.stream_repost(bad_content_id, 'filter1', '0.1', channel_name='@filtering')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_streams))
await self.stream_repost(bad_content_id, 'filter1', '1.1', channel_name='@filtering')
# search for filtered content directly
# search for blocked content directly
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content'))
blocked = result['blocked']
self.assertEqual([], result['items'])
@@ -1035,84 +885,40 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# search inside channel containing filtered content
# search channel containing blocked content
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
filtered = result['blocked']
blocked = result['blocked']
self.assertEqual(1, len(result['items']))
self.assertEqual(1, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(1, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# content was filtered by not_tag before censoring
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
self.assertEqual(0, len(result['items']))
self.assertEqual({"channels": [], "total": 0}, result['blocked'])
# filtered content can still be resolved
result = await self.resolve('lbry://@some_channel/bad_content')
self.assertEqual(bad_content_id, result['claim_id'])
blocking_channel_id = self.get_claim_id(
await self.channel_create('@blocking', '0.1')
await self.channel_create('@blocking', '1.0')
)
self.conductor.spv_node.server.db.sql.blocking_channel_hashes.add(
unhexlify(blocking_channel_id)[::-1]
)
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_streams))
await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_streams))
# filtered content can still be resolved
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
self.assertEqual(bad_content_id, result['lbry://@some_channel/bad_content']['claim_id'])
await self.stream_repost(bad_content_id, 'block1', '1.1', channel_name='@blocking')
# blocked content is not resolveable
error = (await self.resolve('lbry://@some_channel/bad_content'))['error']
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
error = result['lbry://@some_channel/bad_content']['error']
self.assertEqual(error['name'], 'BLOCKED')
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
# a filtered/blocked channel impacts all content inside it
bad_channel_id = self.get_claim_id(
await self.channel_create('@bad_channel', '0.1', tags=['bad-stuff'])
)
worse_content_id = self.get_claim_id(
await self.stream_create('worse_content', '0.1', channel_name='@bad_channel', tags=['bad-stuff'])
)
# check search before filtering channel
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
self.assertEqual(2, result['total_items'])
self.assertEqual('worse_content', result['items'][0]['name'])
self.assertEqual('@bad_channel', result['items'][1]['name'])
# filter channel out
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_channels))
await self.stream_repost(bad_channel_id, 'filter2', '0.1', channel_name='@filtering')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_channels))
# same claim search as previous now returns 0 results
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
filtered = result['blocked']
self.assertEqual(0, len(result['items']))
self.assertEqual(3, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(3, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# filtered channel should still resolve
result = await self.resolve('lbry://@bad_channel')
self.assertEqual(bad_channel_id, result['claim_id'])
result = await self.resolve('lbry://@bad_channel/worse_content')
self.assertEqual(worse_content_id, result['claim_id'])
# block channel
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_channels))
await self.stream_repost(bad_channel_id, 'block2', '0.1', channel_name='@blocking')
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_channels))
# channel, claim in channel or claim individually no longer resolve
self.assertEqual((await self.resolve('lbry://@bad_channel'))['error']['name'], 'BLOCKED')
self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED')
self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED')
async def test_publish_updates_file_list(self):
tx = await self.stream_create(title='created')
txo = tx['outputs'][0]
@@ -1492,12 +1298,11 @@ class StreamCommands(ClaimTestCase):
tx = await self.stream_create(bid='2.5') # creates new claim
claim_id = self.get_claim_id(tx)
txs = await self.transaction_list()
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
self.assertEqual(len(txs[0]['claim_info']), 1)
self.assertEqual(txs[0]['confirmations'], 1)
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
await self.assertBalance(self.account, '7.479893')
@@ -1507,23 +1312,19 @@ class StreamCommands(ClaimTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
await self.stream_update(claim_id, bid='1.0') # updates previous claim
txs = await self.transaction_list()
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
self.assertEqual(len(txs[0]['update_info']), 1)
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['update_info'][0]['is_spent'])
self.assertTrue(txs[1]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.0002165')
await self.assertBalance(self.account, '8.9796765')
await self.stream_abandon(claim_id)
txs = await self.transaction_list()
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[1]['update_info'][0]['is_spent'])
self.assertTrue(txs[2]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
await self.assertBalance(self.account, '9.9795695')
@@ -1573,14 +1374,15 @@ class StreamCommands(ClaimTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
r = await self.resolve('lbry://@abc/foo')
self.assertEqual(
r['claim_id'],
r['lbry://@abc/foo']['claim_id'],
self.get_claim_id(tx3)
)
# publishing again clears channel
tx4 = await self.publish('foo', languages='uk-UA', tags=['Anime', 'anime '])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
claim = await self.resolve('lbry://foo')
r = await self.resolve('lbry://foo')
claim = r['lbry://foo']
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
self.assertNotIn('signing_channel', claim)
self.assertEqual(claim['value']['languages'], ['uk-UA'])
@@ -1622,22 +1424,22 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '3.9998585')
# verify that the incoming tip is marked correctly as is_tip=True in account1
txs = await self.transaction_list(account_id=self.account.id)
txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items']
self.assertEqual(len(txs[0]['support_info']), 1)
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[0]['support_info'][0]['is_tip'])
self.assertFalse(txs[0]['support_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '1.0')
self.assertEqual(txs[0]['fee'], '0.0')
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id)
txs2 = (await self.out(
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
))['items']
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '-1.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
@@ -1653,24 +1455,14 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '1.999717')
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = await self.transaction_list(wallet_id='wallet2')
txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items']
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertFalse(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '0.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
# abandoning the tip increases balance and shows tip as spent
await self.support_abandon(claim_id)
await self.assertBalance(self.account, '4.979662')
txs = await self.transaction_list(account_id=self.account.id)
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(len(txs[1]['support_info']), 1)
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
class CollectionCommands(CommandTestCase):
+2 -6
View File
@@ -30,9 +30,7 @@ class NetworkTests(IntegrationTestCase):
'payment_address': '',
'donation_address': '',
'daily_fee': '0',
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
@@ -56,9 +54,7 @@ class NetworkTests(IntegrationTestCase):
'payment_address': payment_address,
'donation_address': donation_address,
'daily_fee': '42',
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
@@ -148,7 +148,7 @@ class PurchaseCommandTests(CommandTestCase):
url = result[0]['canonical_url']
resolve = await self.resolve(url)
self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id'])
self.assertEqual(result[0]['claim_id'], resolve[url]['purchase_receipt']['claim_id'])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
await self.daemon.jsonrpc_get('lbry://a')
@@ -12,7 +12,7 @@ from lbry.crypto.base58 import Base58
class BaseResolveTestCase(CommandTestCase):
async def assertResolvesToClaimId(self, name, claim_id):
other = await self.resolve(name)
other = (await self.resolve(name))[name]
if claim_id is None:
self.assertIn('error', other)
self.assertEqual(other['error']['name'], 'NOT_FOUND')
@@ -29,20 +29,23 @@ class ResolveCommand(BaseResolveTestCase):
# resolving a channel @abc
response = await self.resolve('lbry://@abc')
self.assertEqual(response['name'], '@abc')
self.assertEqual(response['value_type'], 'channel')
self.assertEqual(response['meta']['claims_in_channel'], 0)
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertEqual(response['lbry://@abc']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['value_type'], 'channel')
self.assertEqual(response['lbry://@abc']['meta']['claims_in_channel'], 0)
await self.stream_create('foo', '0.01', channel_id=channel_id)
await self.stream_create('foo2', '0.01', channel_id=channel_id)
# resolving a channel @abc with some claims in it
response['confirmations'] += 2
response['meta']['claims_in_channel'] = 2
response['lbry://@abc']['confirmations'] += 2
response['lbry://@abc']['meta']['claims_in_channel'] = 2
self.assertEqual(response, await self.resolve('lbry://@abc'))
# resolving claim foo within channel @abc
claim = await self.resolve('lbry://@abc/foo')
response = await self.resolve('lbry://@abc/foo')
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
claim = response['lbry://@abc/foo']
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
@@ -57,12 +60,13 @@ class ResolveCommand(BaseResolveTestCase):
)
# resolving claim foo by itself
self.assertEqual(claim, await self.resolve('lbry://foo'))
self.assertEqual(claim, (await self.resolve('lbry://foo'))['lbry://foo'])
# resolving from the given permanent url
self.assertEqual(claim, await self.resolve(claim['permanent_url']))
permanent_url = response['lbry://@abc/foo']['permanent_url']
self.assertEqual(claim, (await self.resolve(permanent_url))[permanent_url])
# resolving multiple at once
response = await self.out(self.daemon.jsonrpc_resolve(['lbry://foo', 'lbry://foo2']))
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
claim = response['lbry://foo2']
self.assertEqual(claim['name'], 'foo2')
@@ -77,7 +81,7 @@ class ResolveCommand(BaseResolveTestCase):
# resolve handles invalid data
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
await self.generate(1)
response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish"))
response = await self.resolve("lbry://gibberish")
self.assertSetEqual({'lbry://gibberish'}, set(response))
claim = response['lbry://gibberish']
self.assertEqual(claim['name'], 'gibberish')
@@ -85,10 +89,10 @@ class ResolveCommand(BaseResolveTestCase):
# resolve retries
await self.conductor.spv_node.stop()
resolve_task = asyncio.create_task(self.resolve('foo'))
resolving_future = asyncio.ensure_future(self.resolve('foo'))
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.on_ready.first
self.assertIsNotNone((await resolve_task)['claim_id'])
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
async def test_winning_by_effective_amount(self):
# first one remains winner unless something else changes
@@ -178,18 +182,20 @@ class ResolveCommand(BaseResolveTestCase):
# Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
self.assertEqual(
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}},
await self.resolve('lbry://@abc/on-channel-claim')
)
response = await self.resolve('lbry://on-channel-claim')
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {
'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}
}
})
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = await self.resolve(direct_uri)
response = (await self.resolve(direct_uri))[direct_uri]
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
await self.stream_abandon(claim_id=orphan_claim_id)
@@ -199,7 +205,7 @@ class ResolveCommand(BaseResolveTestCase):
valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id'])
# resolves normally
response = await self.resolve(uri)
self.assertTrue(response['is_channel_signature_valid'])
self.assertTrue(response[uri]['is_channel_signature_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.stream_create(
@@ -208,8 +214,8 @@ class ResolveCommand(BaseResolveTestCase):
# it still resolves! but to the older claim
response = await self.resolve(uri)
self.assertTrue(response['is_channel_signature_valid'])
self.assertEqual(response['txid'], valid_claim['txid'])
self.assertTrue(response[uri]['is_channel_signature_valid'])
self.assertEqual(response[uri]['txid'], valid_claim['txid'])
claims = await self.claim_search(name='on-channel-claim')
self.assertEqual(2, len(claims))
self.assertEqual(
@@ -229,8 +235,8 @@ class ResolveCommand(BaseResolveTestCase):
r1 = await self.resolve(f'lbry://{one}')
r2 = await self.resolve(f'lbry://{two}')
self.assertEqual(winner_id, r1['claim_id'])
self.assertEqual(winner_id, r2['claim_id'])
self.assertEqual(winner_id, r1[f'lbry://{one}']['claim_id'])
self.assertEqual(winner_id, r2[f'lbry://{two}']['claim_id'])
async def test_resolve_old_claim(self):
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
@@ -243,7 +249,7 @@ class ResolveCommand(BaseResolveTestCase):
await self.confirm_tx(tx.id)
response = await self.resolve('@olds/example')
self.assertTrue(response['is_channel_signature_valid'])
self.assertTrue(response['@olds/example']['is_channel_signature_valid'])
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
tx = await Transaction.claim_create(
@@ -254,14 +260,16 @@ class ResolveCommand(BaseResolveTestCase):
await self.confirm_tx(tx.id)
response = await self.resolve('bad_example')
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual(
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "@olds/bad_example".',
}},
await self.resolve('@olds/bad_example')
)
self.assertFalse(response['bad_example']['is_channel_signature_valid'])
response = await self.resolve('@olds/bad_example')
self.assertEqual(response, {
'@olds/bad_example': {
'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "@olds/bad_example".',
}
}
})
class ResolveAfterReorg(BaseResolveTestCase):
@@ -281,36 +289,36 @@ class ResolveAfterReorg(BaseResolveTestCase):
channel_id = self.get_claim_id(
await self.channel_create(channel_name, '0.01')
)
self.assertNotIn('error', await self.resolve(channel_name))
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
await self.reorg(206)
self.assertNotIn('error', await self.resolve(channel_name))
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
stream_name = 'foo'
stream_id = self.get_claim_id(
await self.stream_create(stream_name, '0.01', channel_id=channel_id)
)
self.assertNotIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.support_create(stream_id, '0.01')
self.assertNotIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.stream_abandon(stream_id)
self.assertNotIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.channel_abandon(channel_id)
self.assertIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertIn('error', await self.resolve(channel_name))
self.assertIn('error', await self.resolve(stream_name))
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
def generate_signed_legacy(address: bytes, output: Output):
+11 -9
View File
@@ -40,7 +40,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And is the channel resolvable and empty?
response = await self.resolve('lbry://@spam')
self.assertEqual(response['value_type'], 'channel')
self.assertEqual(response['lbry://@spam']['value_type'], 'channel')
# "What goes well with spam?" ponders Chris...
# "A hovercraft with eels!" he exclaims.
@@ -64,7 +64,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# Also checks that his new story can be found on the blockchain before
# giving the link to all his friends.
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(response['value_type'], 'stream')
self.assertEqual(response['lbry://@spam/hovercraft']['value_type'], 'stream')
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
await self.generate(5)
@@ -84,12 +84,13 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(abandon['txid'])
# And now checks that the claim doesn't resolve anymore.
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(
response['lbry://@spam/hovercraft'],
{'error': {
'name': 'NOT_FOUND',
'text': 'Could not find claim at "lbry://@spam/hovercraft".'
}},
await self.resolve('lbry://@spam/hovercraft')
}}
)
# After abandoning he just waits for his LBCs to be returned to his account
@@ -139,8 +140,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And check if his support showed up
resolve_result = await self.resolve(uri)
# It obviously did! Because, blockchain baby \O/
self.assertEqual(resolve_result['amount'], '1.0')
self.assertEqual(resolve_result['meta']['effective_amount'], '1.2')
self.assertEqual(resolve_result[uri]['amount'], '1.0')
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.2')
await self.generate(5)
# Now he also wanted to support the original creator of the Award Winning Novel
@@ -153,7 +154,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And again checks if it went to the just right place
resolve_result = await self.resolve(uri)
# Which it obviously did. Because....?????
self.assertEqual(resolve_result['meta']['effective_amount'], '1.5')
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.5')
await self.generate(5)
# Seeing the ravishing success of his novel Chris adds support to his claim too
@@ -187,10 +188,11 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(abandon['txid'])
# He them checks that the claim doesn't resolve anymore.
response = await self.resolve(uri)
self.assertEqual(
response[uri],
{'error': {
'name': 'NOT_FOUND',
'text': f'Could not find claim at "{uri}".'
}},
await self.resolve(uri)
}}
)
@@ -278,9 +278,7 @@ class CommentCommands(CommandTestCase):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
empty_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
self.assertEqual(0, len(empty_list['items']))
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
@@ -298,41 +296,66 @@ class CommentCommands(CommandTestCase):
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(2, len(comments))
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
comment='Anonymous comment'
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
async def test02_unsigned_comment_list(self):
stream = (await self.stream_create())['outputs'][0]
comments = []
num_items = 28
for i in range(num_items):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(comment_list['total_items'], num_items)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 0)
async def test03_signed_comments_list(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
)
comments.append(comment)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
for field in list_fields:
self.assertIn(field, comment_list)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
comment_temp = comments.pop()
self.assertEqual(comment['comment'], comment_temp['comment'])
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
@@ -343,14 +366,12 @@ class CommentCommands(CommandTestCase):
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], abandoned)
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
@@ -415,12 +436,12 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test06_comment_list(self):
async def test06_comment_list_test(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=claim_id,
@@ -430,48 +451,44 @@ class CommentCommands(CommandTestCase):
owner_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth['claim_id'],
channel_id=moth_id,
)
other_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(normal_list)
anon_comment = await self.daemon.jsonrpc_comment_create(
claim_id=claim_id,
comment='Anonymous comment'
)
self.assertEqual(normal_list['total_items'], 3)
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
for field in list_fields:
self.assertIn(field, normal_list)
self.assertEqual(normal_list['total_items'], 4)
self.assertTrue(normal_list['has_hidden_comments'])
for i, cmnt in enumerate(all_comments):
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(hidden)
)
self.assertTrue(hidden['has_hidden_comments'])
for field in list_fields:
self.assertIn(field, hidden)
self.assertEqual(hidden['total_items'], 1)
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(visible)
)
for field in list_fields:
self.assertIn(field, visible)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(valid_list)
)
for field in list_fields:
self.assertIn(field, valid_list)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(len(valid_list['items']), len(normal_list['items']))
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
async def test07_edit_comments(self):
luda = (await self.channel_create('@Ludacris'))['outputs'][0]
@@ -520,3 +537,18 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
# editing an anonymous comment
anon_comment = await self.daemon.jsonrpc_comment_create(
comment='fast and furiouuuuuus',
claim_id=claim_id
)
anon_cid = anon_comment.get('comment_id')
self.assertIsNotNone(anon_cid)
with self.assertRaises(ValueError):
await self.daemon.jsonrpc_comment_update(
comment='drift drift drift',
comment_id=anon_cid
)
+2 -7
View File
@@ -93,11 +93,6 @@ class TranscodeValidation(ClaimTestCase):
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_max_bit_rate(self):
self.conf.video_bitrate_maximum = 100
with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
async def test_video_format(self):
file_name = self.make_name("bad_video_format_1")
if not file_name.exists():
@@ -135,7 +130,7 @@ class TranscodeValidation(ClaimTestCase):
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "ogv")
self.assertEqual(extension, "ogg")
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
extension = self.analyzer._get_best_container_extension(scan_data, "")
@@ -148,7 +143,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("webm", extension)
extension = self.analyzer._get_best_container_extension("", "libtheora")
self.assertEqual("ogv", extension)
self.assertEqual("ogg", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/"