Compare commits
13
Commits
v0.62.0
...
fix-proactor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7a707f7321 | ||
|
|
c4905d02b9 | ||
|
|
ee39880fb5 | ||
|
|
926b3e56b9 | ||
|
|
e060df5367 | ||
|
|
a3294d4a0d | ||
|
|
19ce0ab246 | ||
|
|
75a1cc0d33 | ||
|
|
05228529b0 | ||
|
|
3cbeadfbc3 | ||
|
|
3d4938c0e2 | ||
|
|
1b850b8a2b | ||
|
|
ebd33f1869 |
No files matched your search
+8
-3
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
|
||||
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 18 -vf "format=yuv420p"')
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 192k')
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
|
||||
@@ -3,7 +3,6 @@ import sys
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import platform
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -262,9 +261,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
if sys.version_info < (3, 8) and platform.system() == "Windows":
|
||||
# TODO: remove after we move to requiring Python 3.8
|
||||
asyncio.set_event_loop(asyncio.ProactorEventLoop())
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
|
||||
@@ -121,6 +121,7 @@ class WalletComponent(Component):
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
|
||||
@@ -3142,8 +3142,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(
|
||||
validate_file, optimize_file, file_path, ignore_non_video=True
|
||||
)
|
||||
claim = Claim()
|
||||
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
|
||||
tx = await Transaction.claim_create(
|
||||
|
||||
+103
-38
@@ -1,3 +1,4 @@
|
||||
import platform
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
@@ -6,7 +7,7 @@ import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from lbry.conf import TranscodeConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -20,11 +21,14 @@ class VideoFileAnalyzer:
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._checked_ffmpeg = False
|
||||
self._loop = None
|
||||
|
||||
async def _execute(self, command, arguments):
|
||||
args = shlex.split(arguments)
|
||||
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
os.path.join(self._conf.ffmpeg_folder, command), *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, loop=self._loop
|
||||
)
|
||||
stdout, stderr = await process.communicate() # returns when the streams are closed
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
|
||||
@@ -40,14 +44,43 @@ class VideoFileAnalyzer:
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
|
||||
return version
|
||||
|
||||
async def _verify_ffmpeg_installed(self):
|
||||
def _verify_ffmpeg_installed(self):
|
||||
if self._ffmpeg_installed:
|
||||
return
|
||||
await self._verify_executable("ffprobe")
|
||||
version = await self._verify_executable("ffmpeg")
|
||||
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
async def _verify():
|
||||
await self._verify_executable("ffprobe")
|
||||
version = await self._verify_executable("ffmpeg")
|
||||
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
def _inner():
|
||||
if not self._loop:
|
||||
if platform.system() == 'Windows':
|
||||
self._loop = asyncio.ProactorEventLoop()
|
||||
else:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return self._loop.run_until_complete(
|
||||
asyncio.ensure_future(_verify(), loop=self._loop)
|
||||
)
|
||||
finally:
|
||||
if self._loop:
|
||||
self._loop.stop()
|
||||
self._loop.close()
|
||||
self._loop = None
|
||||
|
||||
async def wrapper():
|
||||
pool = ThreadPoolExecutor(1)
|
||||
try:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
pool, _inner
|
||||
)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
|
||||
return asyncio.ensure_future(wrapper())
|
||||
|
||||
async def status(self, reset=False, recheck=False):
|
||||
if reset:
|
||||
@@ -95,24 +128,21 @@ class VideoFileAnalyzer:
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _verify_bitrate(scan_data: json):
|
||||
if "bit_rate" not in scan_data["format"]:
|
||||
def _verify_bitrate(self, scan_data: json, file_path):
|
||||
bit_rate_max = float(self._conf.video_bitrate_maximum)
|
||||
if bit_rate_max <= 0:
|
||||
return ""
|
||||
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
|
||||
pixels = -1.0
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
pieces = stream["r_frame_rate"].split('/', 1)
|
||||
frame_rate = float(pieces[0]) if len(pieces) == 1 \
|
||||
else float(pieces[0]) / float(pieces[1])
|
||||
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
|
||||
if "bit_rate" in scan_data["format"]:
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
else:
|
||||
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
|
||||
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||
|
||||
if pixels > 0.0 and pixels / bit_rate < 3.0:
|
||||
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
|
||||
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
|
||||
if bit_rate > bit_rate_max:
|
||||
return "The bit rate is above the configured maximum. Actual: " \
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -176,6 +206,9 @@ class VideoFileAnalyzer:
|
||||
# https://developers.google.com/media/vp9/settings/vod/
|
||||
return int(-0.011 * height + 40)
|
||||
|
||||
def _get_video_scaler(self):
|
||||
return self._conf.video_scaler
|
||||
|
||||
async def _get_video_encoder(self, scan_data):
|
||||
# use what the user said if it's there:
|
||||
# if it's not there, use h264 if we can because it's way faster than the others
|
||||
@@ -255,12 +288,12 @@ class VideoFileAnalyzer:
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
|
||||
if "theora" in video_encoder:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
@@ -272,31 +305,35 @@ class VideoFileAnalyzer:
|
||||
scan_data = json.loads(result)
|
||||
except Exception as e:
|
||||
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
||||
if validate:
|
||||
raise Exception(f'Invalid video file: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
|
||||
return
|
||||
raise ValueError(f'Absent or unreadable video file: {file_path}')
|
||||
|
||||
if "format" not in scan_data:
|
||||
if validate:
|
||||
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
|
||||
return
|
||||
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
|
||||
|
||||
if float(scan_data["format"]["duration"]) < 0.1:
|
||||
log.debug("Media file appears to be an image: %s", file_path)
|
||||
raise ValueError(f'Assuming image file at: {file_path}')
|
||||
|
||||
return scan_data
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path):
|
||||
async def _verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if not validate and not repair:
|
||||
return file_path
|
||||
|
||||
await self._verify_ffmpeg_installed()
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
try:
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
except ValueError:
|
||||
if ignore_non_video:
|
||||
return file_path
|
||||
raise
|
||||
|
||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||
log.debug("Analyzing %s:", file_path)
|
||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||
container_msg = self._verify_container(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||
video_msg = self._verify_video_encoding(scan_data)
|
||||
audio_msg = self._verify_audio_encoding(scan_data)
|
||||
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
||||
@@ -322,6 +359,8 @@ class VideoFileAnalyzer:
|
||||
if video_msg or bitrate_msg:
|
||||
video_encoder = await self._get_video_encoder(scan_data)
|
||||
transcode_command.append(video_encoder)
|
||||
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
|
||||
transcode_command.append(self._get_video_scaler())
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
@@ -355,3 +394,29 @@ class VideoFileAnalyzer:
|
||||
return file_path
|
||||
|
||||
return str(output)
|
||||
|
||||
def _blocking_verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if platform.system() == 'Windows':
|
||||
self._loop = asyncio.ProactorEventLoop()
|
||||
else:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return self._loop.run_until_complete(
|
||||
asyncio.ensure_future(
|
||||
self._verify_or_repair(validate, repair, file_path, ignore_non_video), loop=self._loop
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if self._loop:
|
||||
self._loop.stop()
|
||||
self._loop.close()
|
||||
self._loop = None
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
pool = ThreadPoolExecutor(1)
|
||||
try:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
pool, self._blocking_verify_or_repair, validate, repair, file_path, ignore_non_video
|
||||
)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
@@ -148,6 +148,7 @@ types_map = {
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.ogg': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
|
||||
@@ -162,6 +162,7 @@ class Network:
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
@@ -189,17 +190,20 @@ class Network:
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
|
||||
@@ -866,7 +866,7 @@ class LBRYLevelDB(LevelDB):
|
||||
super().__init__(*args, **kwargs)
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
trending = []
|
||||
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
|
||||
for algorithm_name in self.env.trending_algorithms:
|
||||
if algorithm_name in TRENDING_ALGORITHMS:
|
||||
trending.append(TRENDING_ALGORITHMS[algorithm_name])
|
||||
self.sql = SQLDB(
|
||||
|
||||
@@ -37,6 +37,7 @@ class Env:
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
|
||||
@@ -863,6 +863,7 @@ class LBRYElectrumX(SessionBase):
|
||||
'donation_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
}
|
||||
|
||||
async def server_features_async(self):
|
||||
|
||||
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
|
||||
try:
|
||||
await analyzer.verify_or_repair(True, False, video_file)
|
||||
print("No concerns. Ship it!")
|
||||
except FileNotFoundError as e:
|
||||
print(str(e))
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print("Analysis failed.", str(e))
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
transcode = input("Would you like to make a repaired clone now? [y/N] ")
|
||||
|
||||
@@ -30,7 +30,9 @@ class NetworkTests(IntegrationTestCase):
|
||||
'payment_address': '',
|
||||
'donation_address': '',
|
||||
'daily_fee': '0',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
@@ -54,7 +56,9 @@ class NetworkTests(IntegrationTestCase):
|
||||
'payment_address': payment_address,
|
||||
'donation_address': donation_address,
|
||||
'daily_fee': '42',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
@@ -130,7 +130,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
self.assertEqual(extension, "ogg")
|
||||
self.assertEqual(extension, "ogv")
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
@@ -143,7 +143,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
self.assertEqual("webm", extension)
|
||||
|
||||
extension = self.analyzer._get_best_container_extension("", "libtheora")
|
||||
self.assertEqual("ogg", extension)
|
||||
self.assertEqual("ogv", extension)
|
||||
|
||||
async def test_no_ffmpeg(self):
|
||||
self.conf.ffmpeg_folder = "I don't really exist/"
|
||||
|
||||
Reference in new issue
Block a user