Compare commits

...
30 Commits
Author SHA1 Message Date
Jack Robison 7a707f7321 fix ProactorEventLoop 2020-03-05 23:49:49 -05:00
Lex Berezhny c4905d02b9 Merge pull request #2793 from lbryio/dont_validate_non_video
don't validate and repair files that aren't videos; add maximum video bit rate setting
2020-03-04 09:16:37 -05:00
Brannon King ee39880fb5 fix items from review 2020-03-03 21:38:30 -07:00
Brannon King 926b3e56b9 take a hint, lint 2020-03-03 21:38:30 -07:00
Brannon King e060df5367 hide ValueError 2020-03-03 21:38:30 -07:00
Brannon King a3294d4a0d make bit_rate check support maximum 2020-03-03 21:38:30 -07:00
Brannon King 19ce0ab246 ogg -> ogv, ignore files that aren't video 2020-03-03 21:38:30 -07:00
Lex Berezhny 75a1cc0d33 Merge pull request #2840 from lbryio/status_returns_server_features
`status` command returns features available of currently connected wallet server, including trending algorithm
2020-03-03 20:47:04 -05:00
Lex Berezhny 05228529b0 fix tests 2020-03-03 20:26:07 -05:00
Lex Berezhny 3cbeadfbc3 lbrynet status command returns features available of currenty connected wallet server, including thetrending algorithm used 2020-03-03 20:04:57 -05:00
Lex Berezhny 3d4938c0e2 Merge pull request #2839 from lbryio/file_analysis_join_path
use `os.path.join()` instead of string addition when searching for file analysis binaries
2020-03-03 10:48:06 -05:00
Lex Berezhny 1b850b8a2b dont prepend path when doing which ffmpeg 2020-03-02 21:11:11 -05:00
Lex Berezhny ebd33f1869 use os.path.join instead of string addition when searching for file analysis binaries 2020-03-02 20:34:54 -05:00
Jack Robison ac846b4df3 v0.62.0 2020-03-01 20:29:32 -05:00
Jack Robison 0d427c9b90 Merge pull request #2836 from lbryio/wallet-server-lru-caches
Add LRU caches for blocks and block hashes in the wallet server
2020-03-01 15:19:28 -05:00
Jack Robison dc6194f862 perf counter 2020-03-01 15:17:21 -05:00
Jack Robison de50214e1f block lru cache 2020-03-01 15:17:21 -05:00
Jack Robison 5ec4a88c35 block hash lru cache 2020-03-01 15:17:05 -05:00
Jack Robison 31c141e757 Merge pull request #2835 from lbryio/wallet-server-threadpools
Improve wallet server performance with separate thread pools for leveldb, the block processor, and mempool
2020-03-01 15:15:20 -05:00
Jack Robison 9d5760d899 fix test 2020-03-01 14:41:14 -05:00
Jack Robison 7945e1ea3c mempool threadpool 2020-03-01 12:40:13 -05:00
Jack Robison d94c40e371 leveldb threadpool 2020-03-01 12:40:13 -05:00
Jack Robison 31f22122e8 block processor threadpool 2020-03-01 12:40:13 -05:00
Lex Berezhny 506582aa2b Merge pull request #2834 from lbryio/list_resolve_includes_local_data
when using `--resolve` with local claim `*_list` commands, update resolved result with local metadata (such as `is_mine`)
2020-02-29 15:48:49 -05:00
Lex Berezhny a02b251c9b when using --resolve with local claim list commands, update resolved result with local metadata (such as is_mine and is_mine) 2020-02-29 09:42:11 -05:00
Lex Berezhny 34cbb6fa79 Merge pull request #2831 from lbryio/claim_type_index_improvement
update `claim_type_*` db indexes to be more performant
2020-02-27 14:27:24 -05:00
Lex Berezhny 33679b56fd update claim_type_* indexes to be more peformant 2020-02-27 12:07:38 -05:00
Lex Berezhny 8897c4d560 Merge pull request #2816 from osilkin98/removes-anonymous
removes ability to comment anonymously
2020-02-27 10:37:47 -05:00
Oleg Silkin f0c18ccbe7 Removes testing for anonymous comments, fixes up existing asserts 2020-02-26 19:59:38 -05:00
Oleg Silkin e560d83c51 drops support for commenting without a channel 2020-02-26 19:58:03 -05:00
22 changed files with 240 additions and 162 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.61.0"
__version__ = "0.62.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+8 -3
View File
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 18 -vf "format=yuv420p"')
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 192k')
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
class CLIConfig(TranscodeConfig):
-4
View File
@@ -3,7 +3,6 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@@ -262,9 +261,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
+1
View File
@@ -121,6 +121,7 @@ class WalletComponent(Component):
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
'servers': [
{
'host': session.server[0],
+15 -14
View File
@@ -3142,8 +3142,9 @@ class Daemon(metaclass=JSONRPCServerType):
f"Use --allow-duplicate-name flag to override."
)
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
file_path = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
tx = await Transaction.claim_create(
@@ -4584,7 +4585,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
comment_create (<comment> | --comment=<comment>)
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
[--channel_id=<channel_id>] [--channel_name=<channel_name>]
(--channel_id=<channel_id> | --channel_name=<channel_name>)
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
@@ -4612,23 +4613,23 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_body = {
'comment': comment.strip(),
'claim_id': claim_id,
'parent_id': parent_id,
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
}
channel = await self.get_channel_or_none(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
if channel:
comment_body.update({
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(comment_body, channel)
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
if 'signature' in response:
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
return response
@requires(WALLET_COMPONENT)
+103 -38
View File
@@ -1,3 +1,4 @@
import platform
import asyncio
import json
import logging
@@ -6,7 +7,7 @@ import pathlib
import re
import shlex
import shutil
from concurrent.futures.thread import ThreadPoolExecutor
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
@@ -20,11 +21,14 @@ class VideoFileAnalyzer:
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
self._loop = None
async def _execute(self, command, arguments):
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, loop=self._loop
)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
@@ -40,14 +44,43 @@ class VideoFileAnalyzer:
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
return version
async def _verify_ffmpeg_installed(self):
def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def _verify():
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
def _inner():
if not self._loop:
if platform.system() == 'Windows':
self._loop = asyncio.ProactorEventLoop()
else:
self._loop = asyncio.new_event_loop()
try:
return self._loop.run_until_complete(
asyncio.ensure_future(_verify(), loop=self._loop)
)
finally:
if self._loop:
self._loop.stop()
self._loop.close()
self._loop = None
async def wrapper():
pool = ThreadPoolExecutor(1)
try:
return await asyncio.get_event_loop().run_in_executor(
pool, _inner
)
finally:
pool.shutdown(wait=True)
return asyncio.ensure_future(wrapper())
async def status(self, reset=False, recheck=False):
if reset:
@@ -95,24 +128,21 @@ class VideoFileAnalyzer:
return ""
@staticmethod
def _verify_bitrate(scan_data: json):
if "bit_rate" not in scan_data["format"]:
def _verify_bitrate(self, scan_data: json, file_path):
bit_rate_max = float(self._conf.video_bitrate_maximum)
if bit_rate_max <= 0:
return ""
bit_rate = float(scan_data["format"]["bit_rate"])
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
pixels = -1.0
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
pieces = stream["r_frame_rate"].split('/', 1)
frame_rate = float(pieces[0]) if len(pieces) == 1 \
else float(pieces[0]) / float(pieces[1])
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
if "bit_rate" in scan_data["format"]:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
if pixels > 0.0 and pixels / bit_rate < 3.0:
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
return ""
@@ -176,6 +206,9 @@ class VideoFileAnalyzer:
# https://developers.google.com/media/vp9/settings/vod/
return int(-0.011 * height + 40)
def _get_video_scaler(self):
return self._conf.video_scaler
async def _get_video_encoder(self, scan_data):
# use what the user said if it's there:
# if it's not there, use h264 if we can because it's way faster than the others
@@ -255,12 +288,12 @@ class VideoFileAnalyzer:
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogg"
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogg"
return "ogv"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
@@ -272,31 +305,35 @@ class VideoFileAnalyzer:
scan_data = json.loads(result)
except Exception as e:
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
if validate:
raise Exception(f'Invalid video file: {file_path}')
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
return
raise ValueError(f'Absent or unreadable video file: {file_path}')
if "format" not in scan_data:
if validate:
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
raise ValueError(f'Assuming image file at: {file_path}')
return scan_data
async def verify_or_repair(self, validate, repair, file_path):
async def _verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair:
return file_path
await self._verify_ffmpeg_installed()
scan_data = await self._get_scan_data(validate, file_path)
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
video_msg = self._verify_video_encoding(scan_data)
audio_msg = self._verify_audio_encoding(scan_data)
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
@@ -322,6 +359,8 @@ class VideoFileAnalyzer:
if video_msg or bitrate_msg:
video_encoder = await self._get_video_encoder(scan_data)
transcode_command.append(video_encoder)
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
transcode_command.append(self._get_video_scaler())
else:
transcode_command.append("copy")
@@ -355,3 +394,29 @@ class VideoFileAnalyzer:
return file_path
return str(output)
def _blocking_verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if platform.system() == 'Windows':
self._loop = asyncio.ProactorEventLoop()
else:
self._loop = asyncio.new_event_loop()
try:
return self._loop.run_until_complete(
asyncio.ensure_future(
self._verify_or_repair(validate, repair, file_path, ignore_non_video), loop=self._loop
)
)
finally:
if self._loop:
self._loop.stop()
self._loop.close()
self._loop = None
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
pool = ThreadPoolExecutor(1)
try:
return await asyncio.get_event_loop().run_in_executor(
pool, self._blocking_verify_or_repair, validate, repair, file_path, ignore_non_video
)
finally:
pool.shutdown(wait=True)
+1
View File
@@ -148,6 +148,7 @@ types_map = {
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'),
'.ogg': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
+1
View File
@@ -739,6 +739,7 @@ class Ledger(metaclass=LedgerRegistry):
for txo in txos:
resolved = response[txo.permanent_url]
if isinstance(resolved, Output):
resolved.update_annotations(txo)
results.append(resolved)
else:
if isinstance(resolved, dict) and 'error' in resolved:
+4
View File
@@ -162,6 +162,7 @@ class Network:
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
@@ -189,17 +190,20 @@ class Network:
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
async def start(self):
+5 -1
View File
@@ -1,6 +1,7 @@
import time
import asyncio
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import lbry
from lbry.schema.claim import Claim
@@ -145,6 +146,7 @@ class BlockProcessor:
self.blocks_event = asyncio.Event()
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
self.logger = class_logger(__name__, self.__class__.__name__)
self.executor = ThreadPoolExecutor(1)
# Meta
self.next_cache_check = 0
@@ -165,6 +167,7 @@ class BlockProcessor:
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -173,7 +176,7 @@ class BlockProcessor:
# consistent and not being updated elsewhere.
async def run_in_thread_locked():
async with self.state_lock:
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
return await asyncio.shield(run_in_thread_locked())
async def check_and_advance_blocks(self, raw_blocks):
@@ -657,6 +660,7 @@ class BlockProcessor:
self.logger.info('flushing to DB for a clean shutdown...')
await self.flush(True)
self.db.close()
self.executor.shutdown(wait=True)
def force_chain_reorg(self, count):
"""Force a reorg of the given number of blocks.
+17 -1
View File
@@ -3,6 +3,7 @@ import itertools
import json
import time
from functools import wraps
from pylru import lrucache
import aiohttp
@@ -42,6 +43,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(1000000)
self._block_cache = lrucache(100000)
async def close(self):
if self.connector:
@@ -219,12 +222,25 @@ class Daemon:
async def block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
if first + count < (self.cached_height() or 0) - 200:
return await self._cached_block_hex_hashes(first, count)
params_iterable = ((h, ) for h in range(first, first + count))
return await self._send_vector('getblockhash', params_iterable)
async def _cached_block_hex_hashes(self, first, count):
"""Return the hex hashes of count block starting at height first."""
cached = self._block_hash_cache.get((first, count))
if cached:
return cached
params_iterable = ((h, ) for h in range(first, first + count))
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
return self._block_hash_cache[(first, count)]
async def deserialised_block(self, hex_hash):
"""Return the deserialised block with the given hex hash."""
return await self._send_single('getblock', (hex_hash, True))
if not self._block_cache.get(hex_hash):
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
return self._block_cache[hex_hash]
async def raw_blocks(self, hex_hashes):
"""Return the raw binary blocks with the given hex hashes."""
+4 -4
View File
@@ -136,9 +136,9 @@ class SQLDB:
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
-- claim_type filter + order by
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_group, trending_mixed, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
create unique index if not exists claim_type_release_idx on claim (release_time, claim_type, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (trending_group, trending_mixed, claim_type, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (effective_amount, claim_type, claim_hash);
-- stream_type filter + order by
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
@@ -866,7 +866,7 @@ class LBRYLevelDB(LevelDB):
super().__init__(*args, **kwargs)
path = os.path.join(self.env.db_dir, 'claims.db')
trending = []
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
for algorithm_name in self.env.trending_algorithms:
if algorithm_name in TRENDING_ALGORITHMS:
trending.append(TRENDING_ALGORITHMS[algorithm_name])
self.sql = SQLDB(
+1
View File
@@ -37,6 +37,7 @@ class Env:
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
+8 -6
View File
@@ -20,7 +20,7 @@ from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.wallet.server import util
@@ -63,6 +63,7 @@ class LevelDB:
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
@@ -139,6 +140,7 @@ class LevelDB:
def close(self):
self.utxo_db.close()
self.history.close_db()
self.executor.shutdown(wait=True)
async def open_for_compacting(self):
await self._open_dbs(True, True)
@@ -406,7 +408,7 @@ class LevelDB:
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(None, read_headers)
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -445,7 +447,7 @@ class LevelDB:
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
while True:
history = await asyncio.get_event_loop().run_in_executor(None, read_history)
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if all(hash is not None for hash, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
@@ -611,7 +613,7 @@ class LevelDB:
return utxos
while True:
utxos = await asyncio.get_event_loop().run_in_executor(None, read_utxos)
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
if all(utxo.tx_hash is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
@@ -664,5 +666,5 @@ class LevelDB:
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await asyncio.get_event_loop().run_in_executor(None, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(None, lookup_utxos, hashX_pairs)
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
+15 -10
View File
@@ -6,12 +6,13 @@
# and warranty status of this software.
"""Mempool handling."""
import os
import asyncio
import itertools
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
import attr
@@ -105,14 +106,15 @@ class MemPool:
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
self.logger.info('beginning processing of daemon mempool. '
'This can take some time...')
start = time.time()
start = time.perf_counter()
await synchronized_event.wait()
elapsed = time.time() - start
elapsed = time.perf_counter() - start
self.logger.info(f'synced in {elapsed:.2f}s')
while True:
self.logger.info(f'{len(self.txs):,d} txs '
@@ -121,12 +123,15 @@ class MemPool:
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(None, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
try:
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
finally:
self.executor.shutdown(wait=True)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
@@ -289,7 +294,7 @@ class MemPool:
return txs
# Thread this potentially slow operation so as not to block
tx_map = await asyncio.get_event_loop().run_in_executor(None, deserialize_txs)
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
+1
View File
@@ -863,6 +863,7 @@ class LBRYElectrumX(SessionBase):
'donation_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
async def server_features_async(self):
+2 -2
View File
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
try:
await analyzer.verify_or_repair(True, False, video_file)
print("No concerns. Ship it!")
except FileNotFoundError as e:
print(str(e))
except (FileNotFoundError, ValueError) as e:
print("Analysis failed.", str(e))
except Exception as e:
print(str(e))
transcode = input("Would you like to make a repaired clone now? [y/N] ")
+1 -1
View File
@@ -333,7 +333,7 @@ class Examples(CommandTestCase):
)
reply = await r(
'You can r',
'Use the parent_id param to make replies',
'comment', 'create',
'--comment="I have photographic evidence confirming Sasquatch exists"',
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
@@ -488,6 +488,9 @@ class ClaimCommands(ClaimTestCase):
self.assertTrue(r[0]['meta']['is_controlling'])
self.assertTrue(r[1]['meta']['is_controlling'])
# check that metadata is transfered
self.assertTrue(r[0]['is_mine'])
class ChannelCommands(CommandTestCase):
+6 -2
View File
@@ -30,7 +30,9 @@ class NetworkTests(IntegrationTestCase):
'payment_address': '',
'donation_address': '',
'daily_fee': '0',
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
@@ -54,7 +56,9 @@ class NetworkTests(IntegrationTestCase):
'payment_address': payment_address,
'donation_address': donation_address,
'daily_fee': '42',
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
@@ -278,7 +278,9 @@ class CommentCommands(CommandTestCase):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
empty_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
self.assertEqual(0, len(empty_list['items']))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
@@ -296,66 +298,41 @@ class CommentCommands(CommandTestCase):
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(2, len(comments))
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
comment='Anonymous comment'
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
async def test02_unsigned_comment_list(self):
stream = (await self.stream_create())['outputs'][0]
comments = []
num_items = 28
for i in range(num_items):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(comment_list['total_items'], num_items)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 0)
async def test03_signed_comments_list(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
self.assertIn('comment_id', comment)
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
comment_temp = comments.pop()
self.assertEqual(comment['comment'], comment_temp['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
@@ -366,12 +343,14 @@ class CommentCommands(CommandTestCase):
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], abandoned)
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
@@ -436,12 +415,12 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test06_comment_list_test(self):
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=claim_id,
@@ -451,44 +430,48 @@ class CommentCommands(CommandTestCase):
owner_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth_id,
channel_id=moth['claim_id'],
)
other_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
anon_comment = await self.daemon.jsonrpc_comment_create(
claim_id=claim_id,
comment='Anonymous comment'
)
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
for field in list_fields:
self.assertIn(field, normal_list)
self.assertEqual(normal_list['total_items'], 4)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(normal_list)
)
self.assertEqual(normal_list['total_items'], 3)
self.assertTrue(normal_list['has_hidden_comments'])
for i, cmnt in enumerate(all_comments):
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(hidden)
)
self.assertTrue(hidden['has_hidden_comments'])
for field in list_fields:
self.assertIn(field, hidden)
self.assertEqual(hidden['total_items'], 1)
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
for field in list_fields:
self.assertIn(field, visible)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(visible)
)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
for field in list_fields:
self.assertIn(field, valid_list)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(valid_list)
)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
self.assertEqual(len(valid_list['items']), len(normal_list['items']))
async def test07_edit_comments(self):
luda = (await self.channel_create('@Ludacris'))['outputs'][0]
@@ -537,18 +520,3 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
# editing an anonymous comment
anon_comment = await self.daemon.jsonrpc_comment_create(
comment='fast and furiouuuuuus',
claim_id=claim_id
)
anon_cid = anon_comment.get('comment_id')
self.assertIsNotNone(anon_cid)
with self.assertRaises(ValueError):
await self.daemon.jsonrpc_comment_update(
comment='drift drift drift',
comment_id=anon_cid
)
+2 -2
View File
@@ -130,7 +130,7 @@ class TranscodeValidation(ClaimTestCase):
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "ogg")
self.assertEqual(extension, "ogv")
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
extension = self.analyzer._get_best_container_extension(scan_data, "")
@@ -143,7 +143,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("webm", extension)
extension = self.analyzer._get_best_container_extension("", "libtheora")
self.assertEqual("ogg", extension)
self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/"