Compare commits

...
13 Commits
Author SHA1 Message Date
Jack Robison 7a707f7321 fix ProactorEventLoop 2020-03-05 23:49:49 -05:00
Lex Berezhny c4905d02b9 Merge pull request #2793 from lbryio/dont_validate_non_video
don't validate and repair files that aren't videos; add maximum video bit rate setting
2020-03-04 09:16:37 -05:00
Brannon King ee39880fb5 fix items from review 2020-03-03 21:38:30 -07:00
Brannon King 926b3e56b9 take a hint, lint 2020-03-03 21:38:30 -07:00
Brannon King e060df5367 hide ValueError 2020-03-03 21:38:30 -07:00
Brannon King a3294d4a0d make bit_rate check support maximum 2020-03-03 21:38:30 -07:00
Brannon King 19ce0ab246 ogg -> ogv, ignore files that aren't video 2020-03-03 21:38:30 -07:00
Lex Berezhny 75a1cc0d33 Merge pull request #2840 from lbryio/status_returns_server_features
`status` command returns features available of currently connected wallet server, including trending algorithm
2020-03-03 20:47:04 -05:00
Lex Berezhny 05228529b0 fix tests 2020-03-03 20:26:07 -05:00
Lex Berezhny 3cbeadfbc3 lbrynet status command returns features available of currenty connected wallet server, including thetrending algorithm used 2020-03-03 20:04:57 -05:00
Lex Berezhny 3d4938c0e2 Merge pull request #2839 from lbryio/file_analysis_join_path
use `os.path.join()` instead of string addition when searching for file analysis binaries
2020-03-03 10:48:06 -05:00
Lex Berezhny 1b850b8a2b dont prepend path when doing which ffmpeg 2020-03-02 21:11:11 -05:00
Lex Berezhny ebd33f1869 use os.path.join instead of string addition when searching for file analysis binaries 2020-03-02 20:34:54 -05:00
13 changed files with 133 additions and 54 deletions

No files matched your search

+8 -3
View File
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 18 -vf "format=yuv420p"')
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 192k')
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
class CLIConfig(TranscodeConfig):
-4
View File
@@ -3,7 +3,6 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@@ -262,9 +261,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
+1
View File
@@ -121,6 +121,7 @@ class WalletComponent(Component):
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
'servers': [
{
'host': session.server[0],
+3 -2
View File
@@ -3142,8 +3142,9 @@ class Daemon(metaclass=JSONRPCServerType):
f"Use --allow-duplicate-name flag to override."
)
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
file_path = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
tx = await Transaction.claim_create(
+103 -38
View File
@@ -1,3 +1,4 @@
import platform
import asyncio
import json
import logging
@@ -6,7 +7,7 @@ import pathlib
import re
import shlex
import shutil
from concurrent.futures.thread import ThreadPoolExecutor
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
@@ -20,11 +21,14 @@ class VideoFileAnalyzer:
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
self._loop = None
async def _execute(self, command, arguments):
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, loop=self._loop
)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
@@ -40,14 +44,43 @@ class VideoFileAnalyzer:
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
return version
async def _verify_ffmpeg_installed(self):
def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def _verify():
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
def _inner():
if not self._loop:
if platform.system() == 'Windows':
self._loop = asyncio.ProactorEventLoop()
else:
self._loop = asyncio.new_event_loop()
try:
return self._loop.run_until_complete(
asyncio.ensure_future(_verify(), loop=self._loop)
)
finally:
if self._loop:
self._loop.stop()
self._loop.close()
self._loop = None
async def wrapper():
pool = ThreadPoolExecutor(1)
try:
return await asyncio.get_event_loop().run_in_executor(
pool, _inner
)
finally:
pool.shutdown(wait=True)
return asyncio.ensure_future(wrapper())
async def status(self, reset=False, recheck=False):
if reset:
@@ -95,24 +128,21 @@ class VideoFileAnalyzer:
return ""
@staticmethod
def _verify_bitrate(scan_data: json):
if "bit_rate" not in scan_data["format"]:
def _verify_bitrate(self, scan_data: json, file_path):
bit_rate_max = float(self._conf.video_bitrate_maximum)
if bit_rate_max <= 0:
return ""
bit_rate = float(scan_data["format"]["bit_rate"])
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
pixels = -1.0
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
pieces = stream["r_frame_rate"].split('/', 1)
frame_rate = float(pieces[0]) if len(pieces) == 1 \
else float(pieces[0]) / float(pieces[1])
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
if "bit_rate" in scan_data["format"]:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
if pixels > 0.0 and pixels / bit_rate < 3.0:
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
return ""
@@ -176,6 +206,9 @@ class VideoFileAnalyzer:
# https://developers.google.com/media/vp9/settings/vod/
return int(-0.011 * height + 40)
def _get_video_scaler(self):
return self._conf.video_scaler
async def _get_video_encoder(self, scan_data):
# use what the user said if it's there:
# if it's not there, use h264 if we can because it's way faster than the others
@@ -255,12 +288,12 @@ class VideoFileAnalyzer:
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogg"
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogg"
return "ogv"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
@@ -272,31 +305,35 @@ class VideoFileAnalyzer:
scan_data = json.loads(result)
except Exception as e:
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
if validate:
raise Exception(f'Invalid video file: {file_path}')
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
return
raise ValueError(f'Absent or unreadable video file: {file_path}')
if "format" not in scan_data:
if validate:
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
raise ValueError(f'Assuming image file at: {file_path}')
return scan_data
async def verify_or_repair(self, validate, repair, file_path):
async def _verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair:
return file_path
await self._verify_ffmpeg_installed()
scan_data = await self._get_scan_data(validate, file_path)
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
video_msg = self._verify_video_encoding(scan_data)
audio_msg = self._verify_audio_encoding(scan_data)
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
@@ -322,6 +359,8 @@ class VideoFileAnalyzer:
if video_msg or bitrate_msg:
video_encoder = await self._get_video_encoder(scan_data)
transcode_command.append(video_encoder)
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
transcode_command.append(self._get_video_scaler())
else:
transcode_command.append("copy")
@@ -355,3 +394,29 @@ class VideoFileAnalyzer:
return file_path
return str(output)
def _blocking_verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if platform.system() == 'Windows':
self._loop = asyncio.ProactorEventLoop()
else:
self._loop = asyncio.new_event_loop()
try:
return self._loop.run_until_complete(
asyncio.ensure_future(
self._verify_or_repair(validate, repair, file_path, ignore_non_video), loop=self._loop
)
)
finally:
if self._loop:
self._loop.stop()
self._loop.close()
self._loop = None
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
pool = ThreadPoolExecutor(1)
try:
return await asyncio.get_event_loop().run_in_executor(
pool, self._blocking_verify_or_repair, validate, repair, file_path, ignore_non_video
)
finally:
pool.shutdown(wait=True)
+1
View File
@@ -148,6 +148,7 @@ types_map = {
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'),
'.ogg': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
+4
View File
@@ -162,6 +162,7 @@ class Network:
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
@@ -189,17 +190,20 @@ class Network:
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
async def start(self):
+1 -1
View File
@@ -866,7 +866,7 @@ class LBRYLevelDB(LevelDB):
super().__init__(*args, **kwargs)
path = os.path.join(self.env.db_dir, 'claims.db')
trending = []
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
for algorithm_name in self.env.trending_algorithms:
if algorithm_name in TRENDING_ALGORITHMS:
trending.append(TRENDING_ALGORITHMS[algorithm_name])
self.sql = SQLDB(
+1
View File
@@ -37,6 +37,7 @@ class Env:
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
+1
View File
@@ -863,6 +863,7 @@ class LBRYElectrumX(SessionBase):
'donation_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
async def server_features_async(self):
+2 -2
View File
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
try:
await analyzer.verify_or_repair(True, False, video_file)
print("No concerns. Ship it!")
except FileNotFoundError as e:
print(str(e))
except (FileNotFoundError, ValueError) as e:
print("Analysis failed.", str(e))
except Exception as e:
print(str(e))
transcode = input("Would you like to make a repaired clone now? [y/N] ")
+6 -2
View File
@@ -30,7 +30,9 @@ class NetworkTests(IntegrationTestCase):
'payment_address': '',
'donation_address': '',
'daily_fee': '0',
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
@@ -54,7 +56,9 @@ class NetworkTests(IntegrationTestCase):
'payment_address': payment_address,
'donation_address': donation_address,
'daily_fee': '42',
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
+2 -2
View File
@@ -130,7 +130,7 @@ class TranscodeValidation(ClaimTestCase):
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
extension = self.analyzer._get_best_container_extension(scan_data, "")
self.assertEqual(extension, "ogg")
self.assertEqual(extension, "ogv")
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
extension = self.analyzer._get_best_container_extension(scan_data, "")
@@ -143,7 +143,7 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("webm", extension)
extension = self.analyzer._get_best_container_extension("", "libtheora")
self.assertEqual("ogg", extension)
self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/"