Compare commits
57
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24d749ed2b | ||
|
|
bdd843196d | ||
|
|
dd456347c6 | ||
|
|
e31323fa9d | ||
|
|
ed8572a961 | ||
|
|
9b9e55d07d | ||
|
|
eb9e913788 | ||
|
|
539655292a | ||
|
|
5a7312cbb8 | ||
|
|
a5a4b224e9 | ||
|
|
7f88fcd1b5 | ||
|
|
7e2381816d | ||
|
|
3f2346eb01 | ||
|
|
260a713b08 | ||
|
|
f6b4fee814 | ||
|
|
e9623fae13 | ||
|
|
a1090679a5 | ||
|
|
ad66358d18 | ||
|
|
34c6e09e6f | ||
|
|
507db5f79a | ||
|
|
e6663603a6 | ||
|
|
899852d8ba | ||
|
|
41175a814b | ||
|
|
78faf4e691 | ||
|
|
7e1795b7b1 | ||
|
|
837dc1b078 | ||
|
|
8d0f31e466 | ||
|
|
404f0dcfe0 | ||
|
|
c9b868a238 | ||
|
|
c9092cd1c7 | ||
|
|
86cc65d894 | ||
|
|
8ef2647fa9 | ||
|
|
3ff9e99416 | ||
|
|
1731046011 | ||
|
|
c3d96184b6 | ||
|
|
c255c606a7 | ||
|
|
cc69faa1fd | ||
|
|
0f53cd86c8 | ||
|
|
212c8f188d | ||
|
|
dec248adec | ||
|
|
98e6a066f4 | ||
|
|
c90db54a3d | ||
|
|
927d156933 | ||
|
|
c71f91073f | ||
|
|
ab28387692 | ||
|
|
c4905d02b9 | ||
|
|
ee39880fb5 | ||
|
|
926b3e56b9 | ||
|
|
e060df5367 | ||
|
|
a3294d4a0d | ||
|
|
19ce0ab246 | ||
|
|
75a1cc0d33 | ||
|
|
05228529b0 | ||
|
|
3cbeadfbc3 | ||
|
|
3d4938c0e2 | ||
|
|
1b850b8a2b | ||
|
|
ebd33f1869 |
No files matched your search
+2
-1
@@ -41,7 +41,7 @@ test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork
|
||||
- tox -e datanetwork --recreate
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
@@ -94,6 +94,7 @@ build:linux:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
.PHONY: install tools lint test idea
|
||||
|
||||
install:
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.62.0"
|
||||
__version__ = "0.64.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+8
-3
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
|
||||
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 18 -vf "format=yuv420p"')
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 192k')
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
|
||||
@@ -3,7 +3,6 @@ import sys
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import platform
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -262,9 +261,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
if sys.version_info < (3, 8) and platform.system() == "Windows":
|
||||
# TODO: remove after we move to requiring Python 3.8
|
||||
asyncio.set_event_loop(asyncio.ProactorEventLoop())
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
|
||||
@@ -17,11 +17,17 @@ from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,10 +39,11 @@ WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -121,6 +128,7 @@ class WalletComponent(Component):
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
@@ -318,23 +326,23 @@ class HashAnnouncerComponent(Component):
|
||||
}
|
||||
|
||||
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.stream_manager:
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -343,16 +351,52 @@ class StreamManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
|
||||
+148
-98
@@ -21,9 +21,10 @@ from prometheus_client import generate_latest as prom_generate_latest
|
||||
from google.protobuf.message import DecodeError
|
||||
from lbry.wallet import (
|
||||
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
|
||||
Transaction, Output, Input, Account
|
||||
Transaction, Output, Input, Account, database
|
||||
)
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES
|
||||
|
||||
from lbry import utils
|
||||
from lbry.conf import Config, Setting, NOT_SET
|
||||
@@ -37,7 +38,7 @@ from lbry.error import (
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
from lbry.extras.daemon.components import STREAM_MANAGER_COMPONENT
|
||||
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT
|
||||
from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT
|
||||
from lbry.extras.daemon.componentmanager import RequiredCondition
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
@@ -54,8 +55,8 @@ if typing.TYPE_CHECKING:
|
||||
from lbry.extras.daemon.components import UPnPComponent
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.wallet import WalletManager, Ledger
|
||||
from lbry.file.file_manager import FileManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -340,8 +341,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
|
||||
@property
|
||||
def stream_manager(self) -> typing.Optional['StreamManager']:
|
||||
return self.component_manager.get_component(STREAM_MANAGER_COMPONENT)
|
||||
def file_manager(self) -> typing.Optional['FileManager']:
|
||||
return self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
|
||||
@property
|
||||
def exchange_rate_manager(self) -> typing.Optional['ExchangeRateManager']:
|
||||
@@ -567,8 +568,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
else:
|
||||
name, claim_id = name_and_claim_id.split("/")
|
||||
uri = f"lbry://{name}#{claim_id}"
|
||||
if not self.stream_manager.started.is_set():
|
||||
await self.stream_manager.started.wait()
|
||||
if not self.file_manager.started.is_set():
|
||||
await self.file_manager.started.wait()
|
||||
stream = await self.jsonrpc_get(uri)
|
||||
if isinstance(stream, dict):
|
||||
raise web.HTTPServerError(text=stream['error'])
|
||||
@@ -592,11 +593,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
async def _handle_stream_range_request(self, request: web.Request):
|
||||
sd_hash = request.path.split("/stream/")[1]
|
||||
if not self.stream_manager.started.is_set():
|
||||
await self.stream_manager.started.wait()
|
||||
if sd_hash not in self.stream_manager.streams:
|
||||
if not self.file_manager.started.is_set():
|
||||
await self.file_manager.started.wait()
|
||||
if sd_hash not in self.file_manager.streams:
|
||||
return web.HTTPNotFound()
|
||||
return await self.stream_manager.stream_partial_content(request, sd_hash)
|
||||
return await self.file_manager.stream_partial_content(request, sd_hash)
|
||||
|
||||
async def _process_rpc_call(self, data):
|
||||
args = data.get('params', {})
|
||||
@@ -809,7 +810,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'exchange_rate_manager': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'stream_manager': (bool),
|
||||
'file_manager': (bool),
|
||||
'libtorrent_component': (bool),
|
||||
'upnp': (bool),
|
||||
'wallet': (bool),
|
||||
},
|
||||
@@ -836,6 +838,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
],
|
||||
},
|
||||
'libtorrent_component': {
|
||||
'running': (bool) libtorrent was detected and started successfully,
|
||||
},
|
||||
'dht': {
|
||||
'node_id': (str) lbry dht node id - hex encoded,
|
||||
'peers_in_routing_table': (int) the number of peers in the routing table,
|
||||
@@ -857,7 +862,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'stream_manager': {
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the stream manager,
|
||||
},
|
||||
'upnp': {
|
||||
@@ -1009,7 +1014,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return results
|
||||
|
||||
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT,
|
||||
STREAM_MANAGER_COMPONENT)
|
||||
FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_get(
|
||||
self, uri, file_name=None, download_directory=None, timeout=None, save_file=None, wallet_id=None):
|
||||
"""
|
||||
@@ -1035,7 +1040,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if download_directory and not os.path.isdir(download_directory):
|
||||
return {"error": f"specified download directory \"{download_directory}\" does not exist"}
|
||||
try:
|
||||
stream = await self.stream_manager.download_stream_from_uri(
|
||||
stream = await self.file_manager.download_from_uri(
|
||||
uri, self.exchange_rate_manager, timeout, file_name, download_directory,
|
||||
save_file=save_file, wallet=wallet
|
||||
)
|
||||
@@ -1875,7 +1880,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
File management.
|
||||
"""
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_list(
|
||||
self, sort=None, reverse=False, comparison=None,
|
||||
wallet_id=None, page=None, page_size=None, **kwargs):
|
||||
@@ -1920,7 +1925,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
sort = sort or 'rowid'
|
||||
comparison = comparison or 'eq'
|
||||
paginated = paginate_list(
|
||||
self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size
|
||||
self.file_manager.get_filtered(sort, reverse, comparison, **kwargs), page, page_size
|
||||
)
|
||||
if paginated['items']:
|
||||
receipts = {
|
||||
@@ -1934,7 +1939,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
stream.purchase_receipt = receipts.get(stream.claim_id)
|
||||
return paginated
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_set_status(self, status, **kwargs):
|
||||
"""
|
||||
Start or stop downloading a file
|
||||
@@ -1958,12 +1963,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if status not in ['start', 'stop']:
|
||||
raise Exception('Status must be "start" or "stop".')
|
||||
|
||||
streams = self.stream_manager.get_filtered_streams(**kwargs)
|
||||
streams = self.file_manager.get_filtered(**kwargs)
|
||||
if not streams:
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running:
|
||||
await stream.save_file(node=self.stream_manager.node)
|
||||
await stream.save_file()
|
||||
msg = "Resumed download"
|
||||
elif status == 'stop' and stream.running:
|
||||
await stream.stop()
|
||||
@@ -1975,7 +1980,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
return msg
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_delete(self, delete_from_download_dir=False, delete_all=False, **kwargs):
|
||||
"""
|
||||
Delete a LBRY file
|
||||
@@ -2007,7 +2012,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(bool) true if deletion was successful
|
||||
"""
|
||||
|
||||
streams = self.stream_manager.get_filtered_streams(**kwargs)
|
||||
streams = self.file_manager.get_filtered(**kwargs)
|
||||
|
||||
if len(streams) > 1:
|
||||
if not delete_all:
|
||||
@@ -2024,12 +2029,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
else:
|
||||
for stream in streams:
|
||||
message = f"Deleted file {stream.file_name}"
|
||||
await self.stream_manager.delete_stream(stream, delete_file=delete_from_download_dir)
|
||||
await self.file_manager.delete(stream, delete_file=delete_from_download_dir)
|
||||
log.info(message)
|
||||
result = True
|
||||
return result
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_save(self, file_name=None, download_directory=None, **kwargs):
|
||||
"""
|
||||
Start saving a file to disk.
|
||||
@@ -2056,7 +2061,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {File}
|
||||
"""
|
||||
|
||||
streams = self.stream_manager.get_filtered_streams(**kwargs)
|
||||
streams = self.file_manager.get_filtered(**kwargs)
|
||||
|
||||
if len(streams) > 1:
|
||||
log.warning("There are %i matching files, use narrower filters to select one", len(streams))
|
||||
@@ -2167,19 +2172,20 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_claim_list(
|
||||
self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
|
||||
def jsonrpc_claim_list(self, claim_type=None, **kwargs):
|
||||
"""
|
||||
List my stream and channel claims.
|
||||
|
||||
Usage:
|
||||
claim_list [--claim_type=<claim_type>...]
|
||||
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
[--resolve]
|
||||
|
||||
Options:
|
||||
--claim_type=<claim_type> : (str) claim type: channel, stream, repost, collection
|
||||
--claim_type=<claim_type> : (str or list) claim type: channel, stream, repost, collection
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -2188,15 +2194,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
claims = account.get_claims
|
||||
claim_count = account.get_claim_count
|
||||
else:
|
||||
claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts)
|
||||
claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
|
||||
kwargs['type'] = claim_type or CLAIM_TYPE_NAMES
|
||||
kwargs['unspent'] = True
|
||||
return self.jsonrpc_txo_list(**kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_claim_search(self, **kwargs):
|
||||
@@ -2699,15 +2699,18 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_channel_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
|
||||
def jsonrpc_channel_list(self, *args, **kwargs):
|
||||
"""
|
||||
List my channel claims.
|
||||
|
||||
Usage:
|
||||
channel_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...]
|
||||
[--page=<page>] [--page_size=<page_size>] [--resolve]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) channel name
|
||||
--claim_id=<claim_id> : (str or list) channel id
|
||||
--account_id=<account_id> : (str) id of the account to use
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -2716,15 +2719,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
channels = account.get_channels
|
||||
channel_count = account.get_channel_count
|
||||
else:
|
||||
channels = partial(self.ledger.get_channels, wallet=wallet, accounts=wallet.accounts)
|
||||
channel_count = partial(self.ledger.get_channel_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(channels, channel_count, page, page_size, resolve=resolve)
|
||||
kwargs['type'] = 'channel'
|
||||
kwargs['unspent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_channel_export(self, channel_id=None, channel_name=None, account_id=None, wallet_id=None):
|
||||
@@ -2824,7 +2821,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Create, update, abandon, list and inspect your stream claims.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_publish(self, name, **kwargs):
|
||||
"""
|
||||
Create or replace a stream claim at a given name (use 'stream create/update' for more control).
|
||||
@@ -2946,7 +2943,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"to update a specific stream claim."
|
||||
)
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_stream_repost(self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None,
|
||||
channel_name=None, channel_account_id=None, account_id=None, wallet_id=None,
|
||||
claim_address=None, funding_account_ids=None, preview=False, blocking=False):
|
||||
@@ -3018,7 +3015,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_stream_create(
|
||||
self, name, bid, file_path, allow_duplicate_name=False,
|
||||
channel_id=None, channel_name=None, channel_account_id=None,
|
||||
@@ -3142,8 +3139,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(
|
||||
validate_file, optimize_file, file_path, ignore_non_video=True
|
||||
)
|
||||
claim = Claim()
|
||||
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
|
||||
tx = await Transaction.claim_create(
|
||||
@@ -3153,7 +3151,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
file_stream = None
|
||||
if not preview:
|
||||
file_stream = await self.stream_manager.create_stream(file_path)
|
||||
file_stream = await self.file_manager.create_stream(file_path)
|
||||
claim.stream.source.sd_hash = file_stream.sd_hash
|
||||
new_txo.script.generate()
|
||||
|
||||
@@ -3173,7 +3171,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_stream_update(
|
||||
self, claim_id, bid=None, file_path=None,
|
||||
channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False,
|
||||
@@ -3352,11 +3350,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
stream_hash = None
|
||||
if not preview:
|
||||
old_stream = self.stream_manager.streams.get(old_txo.claim.stream.source.sd_hash, None)
|
||||
old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)
|
||||
old_stream = old_stream[0] if old_stream else None
|
||||
if file_path is not None:
|
||||
if old_stream:
|
||||
await self.stream_manager.delete_stream(old_stream, delete_file=False)
|
||||
file_stream = await self.stream_manager.create_stream(file_path)
|
||||
await self.file_manager.delete(old_stream, delete_file=False)
|
||||
file_stream = await self.file_manager.create_stream(file_path)
|
||||
new_txo.claim.stream.source.sd_hash = file_stream.sd_hash
|
||||
new_txo.script.generate()
|
||||
stream_hash = file_stream.stream_hash
|
||||
@@ -3440,15 +3439,18 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_stream_list(self, account_id=None, wallet_id=None, page=None, page_size=None, resolve=False):
|
||||
def jsonrpc_stream_list(self, *args, **kwargs):
|
||||
"""
|
||||
List my stream claims.
|
||||
|
||||
Usage:
|
||||
stream_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--page=<page>] [--page_size=<page_size>] [--resolve]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...]
|
||||
[--page=<page>] [--page_size=<page_size>] [--resolve]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) stream name
|
||||
--claim_id=<claim_id> : (str or list) stream id
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -3457,15 +3459,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
streams = account.get_streams
|
||||
stream_count = account.get_stream_count
|
||||
else:
|
||||
streams = partial(self.ledger.get_streams, wallet=wallet, accounts=wallet.accounts)
|
||||
stream_count = partial(self.ledger.get_stream_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(streams, stream_count, page, page_size, resolve=resolve)
|
||||
kwargs['type'] = 'stream'
|
||||
kwargs['unspent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
|
||||
DHT_COMPONENT, DATABASE_COMPONENT)
|
||||
@@ -3911,15 +3907,19 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_support_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
|
||||
def jsonrpc_support_list(self, *args, tips=None, **kwargs):
|
||||
"""
|
||||
List supports and tips in my control.
|
||||
|
||||
Usage:
|
||||
support_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--name=<name>...] [--claim_id=<claim_id>...] [--tips]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str or list) claim name
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--tips : (bool) only show tips (is_received=true)
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -3927,15 +3927,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
supports = account.get_supports
|
||||
support_count = account.get_support_count
|
||||
else:
|
||||
supports = partial(self.ledger.get_supports, wallet=wallet, accounts=wallet.accounts)
|
||||
support_count = partial(self.ledger.get_support_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(supports, support_count, page, page_size)
|
||||
kwargs['type'] = 'support'
|
||||
kwargs['unspent'] = True
|
||||
kwargs['include_is_received'] = True
|
||||
if tips is True:
|
||||
kwargs['is_received'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_support_abandon(
|
||||
@@ -4102,12 +4099,71 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
return self.wallet_manager.get_transaction(txid)
|
||||
|
||||
TXO_DOC = """
|
||||
List transaction outputs.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_txo_list(
|
||||
self, account_id=None, type=None, txid=None, # pylint: disable=redefined-builtin
|
||||
claim_id=None, name=None, unspent=False,
|
||||
include_is_received=False, is_received=None, is_not_received=None,
|
||||
wallet_id=None, page=None, page_size=None, resolve=False):
|
||||
"""
|
||||
List my transaction outputs.
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
|
||||
[--include_is_received] [--is_received] [--is_not_received]
|
||||
[--wallet_id=<wallet_id>] [--include_is_received] [--is_received]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
[--resolve]
|
||||
|
||||
Options:
|
||||
--type=<type> : (str or list) claim type: stream, channel, support,
|
||||
purchase, collection, repost, other
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--unspent : (bool) hide spent outputs, show only unspent ones
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--include_is_received : (bool) calculate the is_received property and
|
||||
include in output, this happens automatically if you
|
||||
use the --is_received or --is_not_received filters
|
||||
--is_received : (bool) only return txos sent from others to this account
|
||||
--is_not_received : (bool) only return txos created by this account
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
--resolve : (bool) resolves each claim to provide additional metadata
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
claims = account.get_txos
|
||||
claim_count = account.get_txo_count
|
||||
else:
|
||||
claims = partial(self.ledger.get_txos, wallet=wallet, accounts=wallet.accounts)
|
||||
claim_count = partial(self.ledger.get_txo_count, wallet=wallet, accounts=wallet.accounts)
|
||||
constraints = {'resolve': resolve, 'unspent': unspent, 'include_is_received': include_is_received}
|
||||
if is_received is True:
|
||||
constraints['is_received'] = True
|
||||
elif is_not_received is True:
|
||||
constraints['is_received'] = False
|
||||
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
|
||||
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
|
||||
database.constrain_single_or_list(constraints, 'claim_name', name)
|
||||
return paginate_rows(claims, claim_count, page, page_size, **constraints)
|
||||
|
||||
UTXO_DOC = """
|
||||
Unspent transaction management.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_utxo_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
|
||||
def jsonrpc_utxo_list(self, *args, **kwargs):
|
||||
"""
|
||||
List unspent transaction outputs
|
||||
|
||||
@@ -4123,15 +4179,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
utxos = account.get_utxos
|
||||
utxo_count = account.get_utxo_count
|
||||
else:
|
||||
utxos = partial(self.ledger.get_utxos, wallet=wallet, accounts=wallet.accounts)
|
||||
utxo_count = partial(self.ledger.get_utxo_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(utxos, utxo_count, page, page_size)
|
||||
kwargs['type'] = ['other', 'purchase']
|
||||
kwargs['unspent'] = True
|
||||
return self.jsonrpc_txo_list(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_utxo_release(self, account_id=None, wallet_id=None):
|
||||
@@ -4204,9 +4254,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
if not blob_hash or not is_valid_blobhash(blob_hash):
|
||||
return f"Invalid blob hash to delete '{blob_hash}'"
|
||||
streams = self.stream_manager.get_filtered_streams(sd_hash=blob_hash)
|
||||
streams = self.file_manager.get_filtered(sd_hash=blob_hash)
|
||||
if streams:
|
||||
await self.stream_manager.delete_stream(streams[0])
|
||||
await self.file_manager.delete(streams[0])
|
||||
else:
|
||||
await self.blob_manager.delete_blobs([blob_hash])
|
||||
return "Deleted %s" % blob_hash
|
||||
@@ -4379,7 +4429,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
raise NotImplementedError()
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_reflect(self, **kwargs):
|
||||
"""
|
||||
Reflect all the blobs in a file matching the filter criteria
|
||||
@@ -4408,8 +4458,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
else:
|
||||
server, port = random.choice(self.conf.reflector_servers)
|
||||
reflected = await asyncio.gather(*[
|
||||
self.stream_manager.reflect_stream(stream, server, port)
|
||||
for stream in self.stream_manager.get_filtered_streams(**kwargs)
|
||||
self.file_manager['stream'].reflect_stream(stream, server, port)
|
||||
for stream in self.file_manager.get_filtered_streams(**kwargs)
|
||||
])
|
||||
total = []
|
||||
for reflected_for_stream in reflected:
|
||||
@@ -4874,10 +4924,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
results = await self.ledger.resolve(accounts, urls)
|
||||
if self.conf.save_resolved_claims and results:
|
||||
try:
|
||||
claims = self.stream_manager._convert_to_old_resolve_output(self.wallet_manager, results)
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in claims.values() if 'error' not in value
|
||||
])
|
||||
await self.storage.save_claim_from_output(
|
||||
self.ledger,
|
||||
*(result for result in results.values() if isinstance(result, Output))
|
||||
)
|
||||
except DecodeError:
|
||||
pass
|
||||
return results
|
||||
|
||||
@@ -7,6 +7,7 @@ from json import JSONEncoder
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
@@ -25,6 +26,8 @@ def encode_txo_doc():
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_received': "true if txo was sent from external account to this account",
|
||||
'is_spent': "true if txo is spent",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
@@ -124,6 +127,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, TorrentSource):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
@@ -168,6 +173,10 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_received is not None:
|
||||
output['is_received'] = txo.is_received
|
||||
if txo.is_spent is not None:
|
||||
output['is_spent'] = txo.is_spent
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
@@ -259,26 +268,32 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'streaming_url': managed_stream.stream_url if is_stream else f'file://{managed_stream.full_path}',
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'stream_hash': managed_stream.stream_hash if is_stream else None,
|
||||
'stream_name': managed_stream.descriptor.stream_name if is_stream else None,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name if is_stream else None,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None,
|
||||
'mime_type': managed_stream.mime_type if is_stream else None,
|
||||
'key': managed_stream.descriptor.key if is_stream else None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'written_bytes': managed_stream.written_bytes if is_stream else managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed if is_stream else None,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None,
|
||||
'blobs_remaining': managed_stream.blobs_remaining if is_stream else None,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
@@ -295,7 +310,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected if is_stream else False
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Optional
|
||||
from lbry.wallet import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.dht.constants import DATA_EXPIRATION
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
@@ -727,6 +727,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||
return self.save_claims([{
|
||||
"claim_id": output.claim_id,
|
||||
"name": output.claim_name,
|
||||
"amount": dewies_to_lbc(output.amount),
|
||||
"address": output.get_address(ledger),
|
||||
"txid": output.tx_ref.id,
|
||||
"nout": output.position,
|
||||
"value": output.claim,
|
||||
"height": output.tx_ref.height,
|
||||
"claim_sequence": -1,
|
||||
} for output in outputs])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
@@ -740,7 +753,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||
assert stream_hash or bt_infohash
|
||||
# get the claim id and serialized metadata
|
||||
claim_info = transaction.execute(
|
||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||
@@ -788,6 +802,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||
def _save_torrent(transaction):
|
||||
transaction.execute(
|
||||
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||
).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||
).fetchall()
|
||||
await self.db.run(_save_torrent)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if bt_infohash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[bt_infohash]()
|
||||
|
||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||
claim = None
|
||||
@@ -799,6 +826,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||
return claims[bt_infohash].as_dict() if claims else None
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,286 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||
self.started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self.source_managers['stream']._sources
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||
if 'stream' in self.source_managers:
|
||||
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self):
|
||||
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||
for manager in self.source_managers.values():
|
||||
await manager.started.wait()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
for manager in self.source_managers.values():
|
||||
# fixme: pop or not?
|
||||
manager.stop()
|
||||
self.started.clear()
|
||||
|
||||
@cache_concurrent
|
||||
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||
|
||||
wallet = wallet or self.wallet_manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
claim = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(wallet.accounts, [uri]),
|
||||
resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||
if not resolved_result or uri not in resolved_result:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
txo = resolved_result[uri]
|
||||
if isinstance(txo, dict):
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||
claim = txo.claim
|
||||
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||
|
||||
####################
|
||||
# update or replace
|
||||
####################
|
||||
|
||||
if claim.stream.source.bt_infohash:
|
||||
source_manager = self.source_managers['torrent']
|
||||
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||
else:
|
||||
source_manager = self.source_managers['stream']
|
||||
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
to_replace, updated_stream = None, None
|
||||
if existing and existing[0].claim_id != txo.claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
if claim.stream.source.bt_infohash:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||
existing[0].set_claim(claim_info, claim)
|
||||
else:
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await source_manager._update_content_claim(existing[0])
|
||||
updated_stream = existing[0]
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||
save_file = False
|
||||
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||
if not existing_for_claim_id[0].output_file_exists and (
|
||||
save_file or file_name or download_directory):
|
||||
await existing_for_claim_id[0].save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
to_replace = existing_for_claim_id[0]
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
####################
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(timeout, save_file)
|
||||
|
||||
####################
|
||||
# success case: delete to_replace if applicable, broadcast fee payment
|
||||
####################
|
||||
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await source_manager.delete(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await self.wallet_manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
source_manager.add(stream)
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
else:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||
stream.set_claim(claim_info, claim)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await self.wallet_manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||
# TODO: analytics for torrents
|
||||
pass
|
||||
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||
|
||||
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||
for manager in self.source_managers.values():
|
||||
await manager.delete(source, delete_file)
|
||||
@@ -0,0 +1,161 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.utils import generate_id
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedDownloadSource:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.storage = storage
|
||||
self.config = config
|
||||
self.identifier = identifier
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||
# key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# raise NotImplementedError()
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop_tasks(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
# if not claim_info:
|
||||
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
# self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name
|
||||
|
||||
@property
|
||||
def added_on(self) -> Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
# @property
|
||||
# def stream_url(self):
|
||||
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
@@ -0,0 +1,115 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
class SourceManager:
|
||||
filter_fields = {
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name'
|
||||
}
|
||||
|
||||
source_class = ManagedDownloadSource
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
def add(self, source: ManagedDownloadSource):
|
||||
self._sources[source.identifier] = source
|
||||
|
||||
def remove(self, source: ManagedDownloadSource):
|
||||
if source.identifier not in self._sources:
|
||||
return
|
||||
self._sources.pop(source.identifier)
|
||||
source.stop_tasks()
|
||||
|
||||
async def initialize_from_database(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def start(self):
|
||||
await self.initialize_from_database()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
while self._sources:
|
||||
_, source = self._sources.popitem()
|
||||
source.stop_tasks()
|
||||
self.started.clear()
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
self.remove(source)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in self.filter_fields:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by:
|
||||
if search not in self.filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
sources = []
|
||||
for stream in self._sources.values():
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
sources.append(stream)
|
||||
break
|
||||
else:
|
||||
sources = list(self._sources.values())
|
||||
if sort_by:
|
||||
sources.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
sources.reverse()
|
||||
return sources
|
||||
+57
-34
@@ -6,25 +6,42 @@ import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import platform
|
||||
|
||||
import lbry.utils
|
||||
from lbry.conf import TranscodeConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
DISABLED = platform.system() == "Windows"
|
||||
|
||||
|
||||
class VideoFileAnalyzer:
|
||||
|
||||
def _replace_or_pop_env(self, variable):
|
||||
if variable + '_ORIG' in self._env_copy:
|
||||
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
|
||||
else:
|
||||
self._env_copy.pop(variable, None)
|
||||
|
||||
def __init__(self, conf: TranscodeConfig):
|
||||
self._conf = conf
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._checked_ffmpeg = False
|
||||
self._env_copy = dict(os.environ)
|
||||
if lbry.utils.is_running_from_bundle():
|
||||
# handle the situation where PyInstaller overrides our runtime environment:
|
||||
self._replace_or_pop_env('LD_LIBRARY_PATH')
|
||||
|
||||
async def _execute(self, command, arguments):
|
||||
if DISABLED:
|
||||
return "Disabled on Windows", -1
|
||||
args = shlex.split(arguments)
|
||||
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
os.path.join(self._conf.ffmpeg_folder, command), *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
|
||||
)
|
||||
stdout, stderr = await process.communicate() # returns when the streams are closed
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
|
||||
@@ -32,10 +49,10 @@ class VideoFileAnalyzer:
|
||||
try:
|
||||
version, code = await self._execute(name, "-version")
|
||||
except Exception as e:
|
||||
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
|
||||
code = -1
|
||||
version = ""
|
||||
version = str(e)
|
||||
if code != 0 or not version.startswith(name):
|
||||
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
|
||||
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
|
||||
return version
|
||||
@@ -45,7 +62,7 @@ class VideoFileAnalyzer:
|
||||
return
|
||||
await self._verify_executable("ffprobe")
|
||||
version = await self._verify_executable("ffmpeg")
|
||||
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
|
||||
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
@@ -95,24 +112,21 @@ class VideoFileAnalyzer:
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _verify_bitrate(scan_data: json):
|
||||
if "bit_rate" not in scan_data["format"]:
|
||||
def _verify_bitrate(self, scan_data: json, file_path):
|
||||
bit_rate_max = float(self._conf.video_bitrate_maximum)
|
||||
if bit_rate_max <= 0:
|
||||
return ""
|
||||
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
|
||||
pixels = -1.0
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
pieces = stream["r_frame_rate"].split('/', 1)
|
||||
frame_rate = float(pieces[0]) if len(pieces) == 1 \
|
||||
else float(pieces[0]) / float(pieces[1])
|
||||
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
|
||||
if "bit_rate" in scan_data["format"]:
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
else:
|
||||
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
|
||||
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||
|
||||
if pixels > 0.0 and pixels / bit_rate < 3.0:
|
||||
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
|
||||
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
|
||||
if bit_rate > bit_rate_max:
|
||||
return "The bit rate is above the configured maximum. Actual: " \
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -176,6 +190,9 @@ class VideoFileAnalyzer:
|
||||
# https://developers.google.com/media/vp9/settings/vod/
|
||||
return int(-0.011 * height + 40)
|
||||
|
||||
def _get_video_scaler(self):
|
||||
return self._conf.video_scaler
|
||||
|
||||
async def _get_video_encoder(self, scan_data):
|
||||
# use what the user said if it's there:
|
||||
# if it's not there, use h264 if we can because it's way faster than the others
|
||||
@@ -255,12 +272,12 @@ class VideoFileAnalyzer:
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
|
||||
if "theora" in video_encoder:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
@@ -272,31 +289,35 @@ class VideoFileAnalyzer:
|
||||
scan_data = json.loads(result)
|
||||
except Exception as e:
|
||||
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
||||
if validate:
|
||||
raise Exception(f'Invalid video file: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
|
||||
return
|
||||
raise ValueError(f'Absent or unreadable video file: {file_path}')
|
||||
|
||||
if "format" not in scan_data:
|
||||
if validate:
|
||||
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
|
||||
return
|
||||
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
|
||||
|
||||
if float(scan_data["format"]["duration"]) < 0.1:
|
||||
log.debug("Media file appears to be an image: %s", file_path)
|
||||
raise ValueError(f'Assuming image file at: {file_path}')
|
||||
|
||||
return scan_data
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path):
|
||||
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if not validate and not repair:
|
||||
return file_path
|
||||
|
||||
await self._verify_ffmpeg_installed()
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
try:
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
except ValueError:
|
||||
if ignore_non_video:
|
||||
return file_path
|
||||
raise
|
||||
|
||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||
log.debug("Analyzing %s:", file_path)
|
||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||
container_msg = self._verify_container(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||
video_msg = self._verify_video_encoding(scan_data)
|
||||
audio_msg = self._verify_audio_encoding(scan_data)
|
||||
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
||||
@@ -322,6 +343,8 @@ class VideoFileAnalyzer:
|
||||
if video_msg or bitrate_msg:
|
||||
video_encoder = await self._get_video_encoder(scan_data)
|
||||
transcode_command.append(video_encoder)
|
||||
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
|
||||
transcode_command.append(self._get_video_scaler())
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
|
||||
@@ -148,6 +148,7 @@ types_map = {
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.ogg': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
|
||||
@@ -92,8 +92,8 @@ class StreamDownloader:
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||
# set up peer accumulation
|
||||
if node:
|
||||
self.node = node
|
||||
self.node = node or self.node # fixme: this shouldnt be set here!
|
||||
if self.node:
|
||||
if self.accumulate_task and not self.accumulate_task.done():
|
||||
self.accumulate_task.cancel()
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
|
||||
+51
-165
@@ -3,9 +3,8 @@ import asyncio
|
||||
import time
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
@@ -13,6 +12,7 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
@@ -40,76 +40,32 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class ManagedStream:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
'blob_manager',
|
||||
'sd_hash',
|
||||
'download_directory',
|
||||
'_file_name',
|
||||
'_added_on',
|
||||
'_status',
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'content_fee',
|
||||
'purchase_receipt',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
'streaming',
|
||||
'_running',
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt'
|
||||
]
|
||||
|
||||
class ManagedStream(ManagedDownloadSource):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None,
|
||||
added_on: typing.Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
|
||||
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
|
||||
claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
descriptor: Optional[StreamDescriptor] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
|
||||
download_id, rowid, content_fee, analytics_manager, added_on)
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.file_output_task: Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.streaming = asyncio.Event(loop=self.loop)
|
||||
self._running = asyncio.Event(loop=self.loop)
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return self.identifier
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
@@ -124,17 +80,9 @@ class ManagedStream:
|
||||
return self.descriptor.stream_hash
|
||||
|
||||
@property
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||
|
||||
@property
|
||||
def added_on(self) -> typing.Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
@@ -152,55 +100,6 @@ class ManagedStream:
|
||||
self._status = status
|
||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def blobs_completed(self) -> int:
|
||||
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
||||
@@ -214,39 +113,30 @@ class ManagedStream:
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
@property
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
"""
|
||||
Generate a stream from a file and save it to the db
|
||||
"""
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
|
||||
# file_path: str, key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# """
|
||||
# Generate a stream from a file and save it to the db
|
||||
# """
|
||||
# descriptor = await StreamDescriptor.create_stream(
|
||||
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
# blob_completed_callback=blob_manager.blob_completed
|
||||
# )
|
||||
# await blob_manager.storage.store_stream(
|
||||
# blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
# )
|
||||
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
# os.path.dirname(file_path), 0)
|
||||
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
||||
async def start(self, timeout: Optional[float] = None,
|
||||
save_now: bool = False):
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if self._running.is_set():
|
||||
@@ -254,7 +144,7 @@ class ManagedStream:
|
||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||
self._running.set()
|
||||
try:
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
@@ -264,6 +154,11 @@ class ManagedStream:
|
||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||
if save_now:
|
||||
if not self._file_name:
|
||||
self._file_name = await get_next_available_file_name(
|
||||
self.loop, self.download_directory,
|
||||
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
|
||||
)
|
||||
file_name, download_dir = self._file_name, self.download_directory
|
||||
else:
|
||||
file_name, download_dir = None, None
|
||||
@@ -283,7 +178,7 @@ class ManagedStream:
|
||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||
|
||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
||||
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
|
||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||
raise IndexError(start_blob_num)
|
||||
@@ -295,7 +190,7 @@ class ManagedStream:
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
async def stream_file(self, request: Request, node: Optional['Node'] = None) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||
@@ -386,9 +281,8 @@ class ManagedStream:
|
||||
self.saving.clear()
|
||||
self.finished_write_attempt.set()
|
||||
|
||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
||||
node: typing.Optional['Node'] = None):
|
||||
await self.start(node)
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.start()
|
||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||
self.file_output_task.cancel()
|
||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||
@@ -460,15 +354,7 @@ class ManagedStream:
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
||||
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
if not claim_info:
|
||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
+77
-322
@@ -6,83 +6,65 @@ import random
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.error import InvalidStreamDescriptorError
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet import Output
|
||||
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.wallet.wallet import WalletManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
FILTER_FIELDS = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
def path_or_none(path) -> Optional[str]:
|
||||
if not path:
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(path).decode()
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class StreamManager:
|
||||
class StreamManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedStream]
|
||||
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
||||
self.resume_saving_task: Optional[asyncio.Task] = None
|
||||
self.re_reflect_task: Optional[asyncio.Task] = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self._sources
|
||||
|
||||
def add(self, source: ManagedStream):
|
||||
super().add(source)
|
||||
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
@@ -113,10 +95,10 @@ class StreamManager:
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: bool):
|
||||
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: Optional[bool]):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -129,10 +111,9 @@ class StreamManager:
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
self.add(stream)
|
||||
|
||||
async def load_and_resume_streams_from_database(self):
|
||||
async def initialize_from_database(self):
|
||||
to_recover = []
|
||||
to_start = []
|
||||
|
||||
@@ -146,7 +127,6 @@ class StreamManager:
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Recover %i files", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
@@ -157,7 +137,7 @@ class StreamManager:
|
||||
download_directory = path_or_none(file_info['download_directory'])
|
||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
||||
add_stream_tasks.append(self.loop.create_task(self._load_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
@@ -165,25 +145,22 @@ class StreamManager:
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
log.info("Started stream manager with %i files", len(self._sources))
|
||||
if not self.node:
|
||||
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
if to_resume_saving:
|
||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
||||
|
||||
async def resume(self, to_resume_saving):
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
await asyncio.gather(
|
||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
)
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
|
||||
*(self._sources[sd_hash].save_file(file_name, download_directory)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
))
|
||||
|
||||
async def reflect_streams(self):
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||
batch = []
|
||||
while sd_hashes:
|
||||
stream = self.streams[sd_hashes.pop()]
|
||||
@@ -199,18 +176,15 @@ class StreamManager:
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_and_resume_streams_from_database()
|
||||
await super().start()
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||
self.resume_saving_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
_, stream = self.streams.popitem()
|
||||
stream.stop_tasks()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
@@ -233,261 +207,42 @@ class StreamManager:
|
||||
)
|
||||
return task
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=self.blob_manager.blob_completed
|
||||
)
|
||||
await self.storage.store_stream(
|
||||
self.blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await self.storage.save_published_file(
|
||||
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
|
||||
rowid=row_id, descriptor=descriptor
|
||||
)
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
self.reflect_stream(stream)
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[stream.sd_hash].cancel()
|
||||
stream.stop_tasks()
|
||||
if stream.sd_hash in self.streams:
|
||||
del self.streams[stream.sd_hash]
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
if not isinstance(source, ManagedStream):
|
||||
return
|
||||
if source.identifier in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[source.identifier].cancel()
|
||||
source.stop_tasks()
|
||||
if source.identifier in self.streams:
|
||||
del self.streams[source.identifier]
|
||||
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and stream.output_file_exists:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
streams = list(self.streams.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
|
||||
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_old_resolve_output(wallet_manager, resolves):
|
||||
result = {}
|
||||
for url, txo in resolves.items():
|
||||
if isinstance(txo, Output):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = wallet_manager.ledger.headers.height
|
||||
result[url] = {
|
||||
'name': txo.claim_name,
|
||||
'value': txo.claim,
|
||||
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
|
||||
'claim_id': txo.claim_id,
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'effective_amount': txo.meta.get('effective_amount', 0),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'claim_sequence': -1,
|
||||
'address': txo.get_address(wallet_manager.ledger),
|
||||
'valid_at_height': txo.meta.get('activation_height', None),
|
||||
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
|
||||
'supports': []
|
||||
}
|
||||
else:
|
||||
result[url] = txo
|
||||
return result
|
||||
|
||||
@cache_concurrent
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None,
|
||||
file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None,
|
||||
resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedStream:
|
||||
manager = self.wallet_manager
|
||||
wallet = wallet or manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
txo: Optional[Output] = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
manager.ledger.resolve(wallet.accounts, [uri]),
|
||||
resolve_timeout
|
||||
)
|
||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in resolved_result.values() if 'error' not in value
|
||||
])
|
||||
resolved = resolved_result.get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
txo = response[uri]
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory, node=self.node
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(self.node, timeout)
|
||||
stream.set_claim(resolved, claim)
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
await self.storage.delete_stream(source.descriptor)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
||||
return await self._sources[sd_hash].stream_file(request, self.node)
|
||||
@@ -548,6 +548,13 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def support_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def resolve(self, uri):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
|
||||
|
||||
@@ -557,6 +564,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def file_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
||||
|
||||
async def txo_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
|
||||
|
||||
async def claim_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
||||
|
||||
@@ -566,6 +576,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def channel_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
|
||||
|
||||
async def transaction_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
|
||||
|
||||
@staticmethod
|
||||
def get_claim_id(tx):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,290 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from hashlib import sha1
|
||||
from tempfile import mkdtemp
|
||||
from typing import Optional
|
||||
|
||||
import libtorrent
|
||||
|
||||
|
||||
NOTIFICATION_MASKS = [
|
||||
"error",
|
||||
"peer",
|
||||
"port_mapping",
|
||||
"storage",
|
||||
"tracker",
|
||||
"debug",
|
||||
"status",
|
||||
"progress",
|
||||
"ip_block",
|
||||
"dht",
|
||||
"stats",
|
||||
"session_log",
|
||||
"torrent_log",
|
||||
"peer_log",
|
||||
"incoming_request",
|
||||
"dht_log",
|
||||
"dht_operation",
|
||||
"port_mapping_log",
|
||||
"picker_log",
|
||||
"file_progress",
|
||||
"piece_progress",
|
||||
"upload",
|
||||
"block_progress"
|
||||
]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||
)
|
||||
|
||||
|
||||
def get_notification_type(notification) -> str:
|
||||
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||
if (1 << i) & notification:
|
||||
return notification_type
|
||||
raise ValueError("unrecognized notification type")
|
||||
|
||||
|
||||
class TorrentHandle:
|
||||
def __init__(self, loop, executor, handle):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._handle: libtorrent.torrent_handle = handle
|
||||
self.started = asyncio.Event(loop=loop)
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
self.metadata_completed = asyncio.Event(loop=loop)
|
||||
self.size = 0
|
||||
self.total_wanted_done = 0
|
||||
self.name = ''
|
||||
self.tasks = []
|
||||
self.torrent_file: Optional[libtorrent.file_storage] = None
|
||||
self._base_path = None
|
||||
self._handle.set_sequential_download(1)
|
||||
|
||||
@property
|
||||
def largest_file(self) -> Optional[str]:
|
||||
if not self.torrent_file:
|
||||
return None
|
||||
index = self.largest_file_index
|
||||
return os.path.join(self._base_path, self.torrent_file.at(index).path)
|
||||
|
||||
@property
|
||||
def largest_file_index(self):
|
||||
largest_size, index = 0, 0
|
||||
for file_num in range(self.torrent_file.num_files()):
|
||||
if self.torrent_file.file_size(file_num) > largest_size:
|
||||
largest_size = self.torrent_file.file_size(file_num)
|
||||
index = file_num
|
||||
return index
|
||||
|
||||
def stop_tasks(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
|
||||
def _show_status(self):
|
||||
# fixme: cleanup
|
||||
if not self._handle.is_valid():
|
||||
return
|
||||
status = self._handle.status()
|
||||
if status.has_metadata:
|
||||
self.size = status.total_wanted
|
||||
self.total_wanted_done = status.total_wanted_done
|
||||
self.name = status.name
|
||||
if not self.metadata_completed.is_set():
|
||||
self.metadata_completed.set()
|
||||
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
|
||||
self.torrent_file = self._handle.get_torrent_info().files()
|
||||
self._base_path = status.save_path
|
||||
first_piece = self.torrent_file.at(self.largest_file_index).offset
|
||||
if not self.started.is_set():
|
||||
if self._handle.have_piece(first_piece):
|
||||
self.started.set()
|
||||
else:
|
||||
# prioritize it
|
||||
self._handle.set_piece_deadline(first_piece, 100)
|
||||
if not status.is_seeding:
|
||||
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.num_seeds, status.state, status.save_path)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
log.info("Torrent finished: %s", self.name)
|
||||
|
||||
async def status_loop(self):
|
||||
while True:
|
||||
self._show_status()
|
||||
if self.finished.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
|
||||
class TorrentSession:
|
||||
def __init__(self, loop, executor):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._session: Optional[libtorrent.session] = None
|
||||
self._handles = {}
|
||||
self.tasks = []
|
||||
self.wait_start = True
|
||||
|
||||
async def add_fake_torrent(self):
|
||||
tmpdir = mkdtemp()
|
||||
info, btih = _create_fake_torrent(tmpdir)
|
||||
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
|
||||
handle = self._session.add_torrent({
|
||||
'ti': info, 'save_path': tmpdir, 'flags': flags
|
||||
})
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
return btih
|
||||
|
||||
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||
settings = {
|
||||
'listen_interfaces': f"{interface}:{port}",
|
||||
'enable_outgoing_utp': True,
|
||||
'enable_incoming_utp': True,
|
||||
'enable_outgoing_tcp': False,
|
||||
'enable_incoming_tcp': False
|
||||
}
|
||||
self._session = await self._loop.run_in_executor(
|
||||
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||
)
|
||||
self.tasks.append(self._loop.create_task(self.process_alerts()))
|
||||
|
||||
def stop(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
self._session.save_state()
|
||||
self._session.pause()
|
||||
self._session.stop_dht()
|
||||
self._session.stop_lsd()
|
||||
self._session.stop_natpmp()
|
||||
self._session.stop_upnp()
|
||||
self._session = None
|
||||
|
||||
def _pop_alerts(self):
|
||||
for alert in self._session.pop_alerts():
|
||||
log.info("torrent alert: %s", alert)
|
||||
|
||||
async def process_alerts(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._pop_alerts
|
||||
)
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._session.resume
|
||||
)
|
||||
|
||||
def _add_torrent(self, btih: str, download_directory: Optional[str]):
|
||||
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
|
||||
if download_directory:
|
||||
params['save_path'] = download_directory
|
||||
handle = self._session.add_torrent(params)
|
||||
handle.force_dht_announce()
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
|
||||
def full_path(self, btih):
|
||||
return self._handles[btih].largest_file
|
||||
|
||||
async def add_torrent(self, btih, download_path):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._add_torrent, btih, download_path
|
||||
)
|
||||
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
|
||||
await self._handles[btih].metadata_completed.wait()
|
||||
if self.wait_start:
|
||||
# fixme: temporary until we add streaming support, otherwise playback fails!
|
||||
await self._handles[btih].started.wait()
|
||||
|
||||
def remove_torrent(self, btih, remove_files=False):
|
||||
if btih in self._handles:
|
||||
handle = self._handles[btih]
|
||||
handle.stop_tasks()
|
||||
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
|
||||
self._handles.pop(btih)
|
||||
|
||||
async def save_file(self, btih, download_directory):
|
||||
handle = self._handles[btih]
|
||||
await handle.resume()
|
||||
|
||||
def get_size(self, btih):
|
||||
return self._handles[btih].size
|
||||
|
||||
def get_name(self, btih):
|
||||
return self._handles[btih].name
|
||||
|
||||
def get_downloaded(self, btih):
|
||||
return self._handles[btih].total_wanted_done
|
||||
|
||||
def is_completed(self, btih):
|
||||
return self._handles[btih].finished.is_set()
|
||||
|
||||
|
||||
def get_magnet_uri(btih):
|
||||
return f"magnet:?xt=urn:btih:{btih}"
|
||||
|
||||
|
||||
def _create_fake_torrent(tmpdir):
|
||||
# beware, that's just for testing
|
||||
path = os.path.join(tmpdir, 'tmp')
|
||||
with open(path, 'wb') as myfile:
|
||||
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
|
||||
file_storage = libtorrent.file_storage()
|
||||
file_storage.add_file('tmp', size)
|
||||
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
|
||||
libtorrent.set_piece_hashes(t, tmpdir)
|
||||
info = libtorrent.torrent_info(t.generate())
|
||||
btih = sha1(info.metadata()).hexdigest()
|
||||
return info, btih
|
||||
|
||||
|
||||
async def main():
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
|
||||
|
||||
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
|
||||
|
||||
executor = None
|
||||
session = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
session2 = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
await session.bind('localhost', port=4040)
|
||||
await session2.bind('localhost', port=4041)
|
||||
btih = await session.add_fake_torrent()
|
||||
session2._session.add_dht_node(('localhost', 4040))
|
||||
await session2.add_torrent(btih, "/tmp/down")
|
||||
while True:
|
||||
await asyncio.sleep(100)
|
||||
await session.pause()
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TorrentInfo:
|
||||
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
|
||||
|
||||
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
|
||||
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
|
||||
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
|
||||
self.dht_seeds = dht_seeds
|
||||
self.http_seeds = http_seeds
|
||||
self.trackers = trackers
|
||||
self.total_size = total_size
|
||||
|
||||
@classmethod
|
||||
def from_libtorrent_info(cls, torrent_info):
|
||||
return cls(
|
||||
torrent_info.nodes(), tuple(
|
||||
{
|
||||
'url': web_seed['url'],
|
||||
'type': web_seed['type'],
|
||||
'auth': web_seed['auth']
|
||||
} for web_seed in torrent_info.web_seeds()
|
||||
), tuple(
|
||||
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
|
||||
), torrent_info.total_size()
|
||||
)
|
||||
|
||||
|
||||
class Torrent:
|
||||
def __init__(self, loop, handle):
|
||||
self._loop = loop
|
||||
self._handle = handle
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
|
||||
def _threaded_update_status(self):
|
||||
status = self._handle.status()
|
||||
if not status.is_seeding:
|
||||
log.info(
|
||||
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.state
|
||||
)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
|
||||
async def wait_for_finished(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
None, self._threaded_update_status
|
||||
)
|
||||
if self.finished.is_set():
|
||||
log.info("finished downloading torrent!")
|
||||
await self.pause()
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
log.info("pause torrent")
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.resume
|
||||
)
|
||||
@@ -0,0 +1,140 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class TorrentSource(ManagedDownloadSource):
|
||||
STATUS_STOPPED = "stopped"
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'bt_infohash'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
|
||||
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
|
||||
rowid, content_fee, analytics_manager, added_on)
|
||||
self.torrent_session = torrent_session
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
full_path = self.torrent_session.full_path(self.identifier)
|
||||
self.download_directory = os.path.dirname(full_path)
|
||||
return full_path
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
await self.torrent_session.remove_torrent(self.identifier)
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.torrent_session.save_file(self.identifier, download_directory)
|
||||
|
||||
@property
|
||||
def torrent_length(self):
|
||||
return self.torrent_session.get_size(self.identifier)
|
||||
|
||||
@property
|
||||
def written_bytes(self):
|
||||
return self.torrent_session.get_downloaded(self.identifier)
|
||||
|
||||
@property
|
||||
def torrent_name(self):
|
||||
return self.torrent_session.get_name(self.identifier)
|
||||
|
||||
@property
|
||||
def bt_infohash(self):
|
||||
return self.identifier
|
||||
|
||||
def stop_tasks(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
return self.torrent_session.is_completed(self.identifier)
|
||||
|
||||
|
||||
class TorrentManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedDownloadSource]
|
||||
|
||||
filter_fields = set(SourceManager.filter_fields)
|
||||
filter_fields.update({
|
||||
'bt_infohash',
|
||||
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
|
||||
'blobs_in_stream'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.torrent_session: 'TorrentSession' = torrent_session
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
|
||||
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
|
||||
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
|
||||
torrent_session=self.torrent_session
|
||||
)
|
||||
self.add(stream)
|
||||
|
||||
async def initialize_from_database(self):
|
||||
pass
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
log.info("finished stopping the torrent manager")
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
await super().delete(source, delete_file)
|
||||
self.torrent_session.remove_torrent(source.identifier, delete_file)
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
raise NotImplementedError
|
||||
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
# await self.storage.delete_stream(source.descriptor)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
raise NotImplementedError
|
||||
@@ -4,6 +4,7 @@ import datetime
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
@@ -276,3 +277,8 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
|
||||
return response['data']['ip']
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
@@ -467,6 +467,12 @@ class Account:
|
||||
'max_receiving_gap': receiving_gap,
|
||||
}
|
||||
|
||||
def get_txos(self, **constraints):
|
||||
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_txo_count(self, **constraints):
|
||||
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ COIN = 100*CENT
|
||||
TIMEOUT = 30.0
|
||||
|
||||
TXO_TYPES = {
|
||||
"other": 0,
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
@@ -14,9 +15,13 @@ TXO_TYPES = {
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = [
|
||||
'stream',
|
||||
'channel',
|
||||
'collection',
|
||||
'repost',
|
||||
]
|
||||
|
||||
CLAIM_TYPES = [
|
||||
TXO_TYPES['stream'],
|
||||
TXO_TYPES['channel'],
|
||||
TXO_TYPES['collection'],
|
||||
TXO_TYPES['repost'],
|
||||
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||
]
|
||||
+65
-28
@@ -107,7 +107,9 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
if not key:
|
||||
sql.append(constraint)
|
||||
continue
|
||||
if key.startswith('$'):
|
||||
if key.startswith('$$'):
|
||||
col, key = col[2:], key[1:]
|
||||
elif key.startswith('$'):
|
||||
values[key] = constraint
|
||||
continue
|
||||
if key.endswith('__not'):
|
||||
@@ -221,6 +223,19 @@ def rows_to_dict(rows, fields):
|
||||
return []
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
constraints[f"{column}__in"] = value
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
|
||||
|
||||
class SQLiteMixin:
|
||||
|
||||
SCHEMA_VERSION: Optional[str] = None
|
||||
@@ -350,6 +365,7 @@ class Database(SQLiteMixin):
|
||||
create index if not exists txo_txid_idx on txo (txid);
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
create index if not exists txo_claim_id_idx on txo (claim_id);
|
||||
create index if not exists txo_claim_name_idx on txo (claim_name);
|
||||
create index if not exists txo_txo_type_idx on txo (txo_type);
|
||||
"""
|
||||
|
||||
@@ -474,20 +490,21 @@ class Database(SQLiteMixin):
|
||||
async def select_transactions(self, cols, accounts=None, **constraints):
|
||||
if not {'txid', 'txid__in'}.intersection(constraints):
|
||||
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
|
||||
constraints.update({
|
||||
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
|
||||
where, values = constraints_to_sql({
|
||||
'$$account_address.account__in': [a.public_key.address for a in accounts]
|
||||
})
|
||||
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
|
||||
where = f" WHERE account_address.account IN ({account_values})"
|
||||
constraints['txid__in'] = f"""
|
||||
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
|
||||
SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where}
|
||||
UNION
|
||||
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
|
||||
SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where}
|
||||
"""
|
||||
constraints.update(values)
|
||||
return await self.db.execute_fetchall(
|
||||
*query(f"SELECT {cols} FROM tx", **constraints)
|
||||
)
|
||||
|
||||
TXO_NOT_MINE = Output(None, None, is_my_account=False)
|
||||
|
||||
async def get_transactions(self, wallet=None, **constraints):
|
||||
tx_rows = await self.select_transactions(
|
||||
'txid, raw, height, position, is_verified',
|
||||
@@ -538,7 +555,7 @@ class Database(SQLiteMixin):
|
||||
if _txo:
|
||||
txo.update_annotations(_txo)
|
||||
else:
|
||||
txo.update_annotations(None)
|
||||
txo.update_annotations(self.TXO_NOT_MINE)
|
||||
|
||||
for tx in txs:
|
||||
txos = tx.outputs
|
||||
@@ -560,13 +577,33 @@ class Database(SQLiteMixin):
|
||||
if txs:
|
||||
return txs[0]
|
||||
|
||||
async def select_txos(self, cols, **constraints):
|
||||
async def select_txos(self, cols, wallet=None, include_is_received=False, **constraints):
|
||||
if include_is_received:
|
||||
assert wallet is not None, 'cannot use is_recieved filter without wallet argument'
|
||||
account_in_wallet, values = constraints_to_sql({
|
||||
'$$account__in#is_received': [a.public_key.address for a in wallet.accounts]
|
||||
})
|
||||
cols += f""",
|
||||
NOT EXISTS(
|
||||
SELECT 1 FROM txi JOIN account_address USING (address)
|
||||
WHERE txi.txid=txo.txid AND {account_in_wallet}
|
||||
) as is_received
|
||||
"""
|
||||
constraints.update(values)
|
||||
sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)"
|
||||
if 'accounts' in constraints:
|
||||
sql += " JOIN account_address USING (address)"
|
||||
return await self.db.execute_fetchall(*query(sql, **constraints))
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints):
|
||||
@staticmethod
|
||||
def constrain_unspent(constraints):
|
||||
constraints['is_reserved'] = False
|
||||
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, unspent=False, include_is_received=False, **constraints):
|
||||
include_is_received = include_is_received or 'is_received' in constraints
|
||||
if unspent:
|
||||
self.constrain_unspent(constraints)
|
||||
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = [
|
||||
@@ -577,9 +614,9 @@ class Database(SQLiteMixin):
|
||||
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
|
||||
select group_concat(account||"|"||chain) from account_address
|
||||
where account_address.address=txo.address
|
||||
)
|
||||
), exists(select 1 from txi where txi.txoid=txo.txoid)
|
||||
""",
|
||||
**constraints
|
||||
wallet=wallet, include_is_received=include_is_received, **constraints
|
||||
)
|
||||
txos = []
|
||||
txs = {}
|
||||
@@ -599,6 +636,9 @@ class Database(SQLiteMixin):
|
||||
txo = txs[row[0]].outputs[row[5]]
|
||||
row_accounts = dict(a.split('|') for a in row[8].split(','))
|
||||
account_match = set(row_accounts) & my_accounts
|
||||
txo.is_spent = bool(row[9])
|
||||
if include_is_received:
|
||||
txo.is_received = bool(row[10])
|
||||
if account_match:
|
||||
txo.is_my_account = True
|
||||
txo.is_change = row_accounts[account_match.pop()] == '1'
|
||||
@@ -634,33 +674,28 @@ class Database(SQLiteMixin):
|
||||
|
||||
return txos
|
||||
|
||||
async def get_txo_count(self, **constraints):
|
||||
async def get_txo_count(self, unspent=False, **constraints):
|
||||
constraints['include_is_received'] = 'is_received' in constraints
|
||||
constraints.pop('resolve', None)
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
if unspent:
|
||||
self.constrain_unspent(constraints)
|
||||
count = await self.select_txos('count(*)', **constraints)
|
||||
return count[0][0]
|
||||
|
||||
@staticmethod
|
||||
def constrain_utxo(constraints):
|
||||
constraints['is_reserved'] = False
|
||||
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constrain_utxo(constraints)
|
||||
return self.get_txos(**constraints)
|
||||
return self.get_txos(unspent=True, **constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constrain_utxo(constraints)
|
||||
return self.get_txo_count(**constraints)
|
||||
return self.get_txo_count(unspent=True, **constraints)
|
||||
|
||||
async def get_balance(self, wallet=None, accounts=None, **constraints):
|
||||
assert wallet or accounts, \
|
||||
"'wallet' or 'accounts' constraints required to calculate balance"
|
||||
constraints['accounts'] = accounts or wallet.accounts
|
||||
self.constrain_utxo(constraints)
|
||||
self.constrain_unspent(constraints)
|
||||
balance = await self.select_txos('SUM(amount)', **constraints)
|
||||
return balance[0][0] or 0
|
||||
|
||||
@@ -743,11 +778,13 @@ class Database(SQLiteMixin):
|
||||
|
||||
@staticmethod
|
||||
def constrain_claims(constraints):
|
||||
if {'txo_type', 'txo_type__in'}.intersection(constraints):
|
||||
return
|
||||
claim_types = constraints.pop('claim_type', None)
|
||||
if isinstance(claim_types, str) and claim_types:
|
||||
claim_types = [claim_types]
|
||||
if isinstance(claim_types, list) and claim_types:
|
||||
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
|
||||
if claim_types:
|
||||
constrain_single_or_list(
|
||||
constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x]
|
||||
)
|
||||
else:
|
||||
constraints['txo_type__in'] = CLAIM_TYPES
|
||||
|
||||
|
||||
+25
-8
@@ -262,6 +262,15 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self.constraint_spending_utxos(constraints)
|
||||
return self.db.get_utxo_count(**constraints)
|
||||
|
||||
async def get_txos(self, resolve=False, **constraints):
|
||||
txos = await self.db.get_txos(**constraints)
|
||||
if resolve:
|
||||
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
|
||||
return txos
|
||||
|
||||
def get_txo_count(self, **constraints):
|
||||
return self.db.get_txo_count(**constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
return self.db.get_transactions(**constraints)
|
||||
|
||||
@@ -735,9 +744,11 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
|
||||
async def _resolve_for_local_results(self, accounts, txos):
|
||||
results = []
|
||||
response = await self.resolve(accounts, [txo.permanent_url for txo in txos])
|
||||
response = await self.resolve(
|
||||
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
|
||||
)
|
||||
for txo in txos:
|
||||
resolved = response[txo.permanent_url]
|
||||
resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None
|
||||
if isinstance(resolved, Output):
|
||||
resolved.update_annotations(txo)
|
||||
results.append(resolved)
|
||||
@@ -844,7 +855,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
for txo in tx.my_update_outputs:
|
||||
if is_my_inputs: # updating my own claim
|
||||
@@ -863,7 +875,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
else: # someone sent us their claim
|
||||
item['update_info'].append({
|
||||
@@ -872,7 +885,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
for txo in tx.my_support_outputs:
|
||||
item['support_info'].append({
|
||||
@@ -882,7 +896,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': not is_my_inputs,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
if is_my_inputs:
|
||||
for txo in tx.other_support_outputs:
|
||||
@@ -893,7 +908,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': is_my_inputs,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
for txo in tx.my_abandon_outputs:
|
||||
item['abandon_info'].append({
|
||||
@@ -910,7 +926,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.purchased_claim_id,
|
||||
'nout': txo.position
|
||||
'nout': txo.position,
|
||||
'is_spent': txo.is_spent,
|
||||
})
|
||||
history.append(item)
|
||||
return history
|
||||
|
||||
@@ -162,6 +162,7 @@ class Network:
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
@@ -189,17 +190,20 @@ class Network:
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
|
||||
@@ -866,7 +866,7 @@ class LBRYLevelDB(LevelDB):
|
||||
super().__init__(*args, **kwargs)
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
trending = []
|
||||
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
|
||||
for algorithm_name in self.env.trending_algorithms:
|
||||
if algorithm_name in TRENDING_ALGORITHMS:
|
||||
trending.append(TRENDING_ALGORITHMS[algorithm_name])
|
||||
self.sql = SQLDB(
|
||||
|
||||
@@ -37,6 +37,7 @@ class Env:
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
|
||||
@@ -863,6 +863,7 @@ class LBRYElectrumX(SessionBase):
|
||||
'donation_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
}
|
||||
|
||||
async def server_features_async(self):
|
||||
|
||||
@@ -207,7 +207,7 @@ class OutputEffectiveAmountEstimator:
|
||||
class Output(InputOutput):
|
||||
|
||||
__slots__ = (
|
||||
'amount', 'script', 'is_change', 'is_my_account',
|
||||
'amount', 'script', 'is_change', 'is_spent', 'is_received', 'is_my_account',
|
||||
'channel', 'private_key', 'meta',
|
||||
'purchase', 'purchased_claim', 'purchase_receipt',
|
||||
'reposted_claim', 'claims',
|
||||
@@ -215,13 +215,16 @@ class Output(InputOutput):
|
||||
|
||||
def __init__(self, amount: int, script: OutputScript,
|
||||
tx_ref: TXRef = None, position: int = None,
|
||||
is_change: Optional[bool] = None, is_my_account: Optional[bool] = None,
|
||||
is_change: Optional[bool] = None, is_spent: Optional[bool] = None,
|
||||
is_received: Optional[bool] = None, is_my_account: Optional[bool] = None,
|
||||
channel: Optional['Output'] = None, private_key: Optional[str] = None
|
||||
) -> None:
|
||||
super().__init__(tx_ref, position)
|
||||
self.amount = amount
|
||||
self.script = script
|
||||
self.is_change = is_change
|
||||
self.is_spent = is_spent
|
||||
self.is_received = is_received
|
||||
self.is_my_account = is_my_account
|
||||
self.channel = channel
|
||||
self.private_key = private_key
|
||||
@@ -234,10 +237,12 @@ class Output(InputOutput):
|
||||
|
||||
def update_annotations(self, annotated):
|
||||
if annotated is None:
|
||||
self.is_change = False
|
||||
self.is_my_account = False
|
||||
self.is_change = None
|
||||
self.is_spent = None
|
||||
self.is_my_account = None
|
||||
else:
|
||||
self.is_change = annotated.is_change
|
||||
self.is_spent = annotated.is_spent
|
||||
self.is_my_account = annotated.is_my_account
|
||||
self.channel = annotated.channel if annotated else None
|
||||
self.private_key = annotated.private_key if annotated else None
|
||||
|
||||
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
|
||||
try:
|
||||
await analyzer.verify_or_repair(True, False, video_file)
|
||||
print("No concerns. Ship it!")
|
||||
except FileNotFoundError as e:
|
||||
print(str(e))
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print("Analysis failed.", str(e))
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
transcode = input("Would you like to make a repaired clone now? [y/N] ")
|
||||
|
||||
@@ -6,7 +6,7 @@ source =
|
||||
lbry
|
||||
.tox/*/lib/python*/site-packages/lbry
|
||||
|
||||
[cryptography.*,coincurve.*,pbkdf2]
|
||||
[cryptography.*,coincurve.*,pbkdf2, libtorrent]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[pylint]
|
||||
@@ -18,6 +18,7 @@ max-line-length=120
|
||||
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
disable=
|
||||
c-extension-no-member,
|
||||
fixme,
|
||||
broad-except,
|
||||
no-else-return,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
import os.path
|
||||
import tempfile
|
||||
import logging
|
||||
@@ -399,12 +398,116 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([], text='cloud')
|
||||
|
||||
|
||||
class TransactionCommands(ClaimTestCase):
|
||||
|
||||
async def test_transaction_list(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
await self.channel_update(channel_id, bid='0.5')
|
||||
await self.channel_abandon(claim_id=channel_id)
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
await self.stream_update(stream_id, bid='0.5')
|
||||
await self.stream_abandon(claim_id=stream_id)
|
||||
|
||||
r = await self.transaction_list()
|
||||
self.assertEqual(7, len(r))
|
||||
self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id'])
|
||||
self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id'])
|
||||
self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id'])
|
||||
self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id'])
|
||||
self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id'])
|
||||
self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id'])
|
||||
|
||||
|
||||
class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
async def test_txo_list_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
await self.channel_update(channel_id, bid='0.5')
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
await self.stream_update(stream_id, bid='0.5')
|
||||
|
||||
# type filtering
|
||||
r = await self.txo_list(type='channel')
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
self.assertFalse(r[0]['is_spent'])
|
||||
self.assertEqual('channel', r[1]['value_type'])
|
||||
self.assertTrue(r[1]['is_spent'])
|
||||
|
||||
r = await self.txo_list(type='stream')
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
self.assertFalse(r[0]['is_spent'])
|
||||
self.assertEqual('stream', r[1]['value_type'])
|
||||
self.assertTrue(r[1]['is_spent'])
|
||||
|
||||
r = await self.txo_list(type=['stream', 'channel'])
|
||||
self.assertEqual(4, len(r))
|
||||
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
|
||||
|
||||
# claim_id filtering
|
||||
r = await self.txo_list(claim_id=stream_id)
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(claim_id=[stream_id, channel_id])
|
||||
self.assertEqual(4, len(r))
|
||||
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
|
||||
stream_name, _, channel_name, _ = (c['name'] for c in r)
|
||||
|
||||
r = await self.txo_list(claim_id=['beef'])
|
||||
self.assertEqual(0, len(r))
|
||||
|
||||
# claim_name filtering
|
||||
r = await self.txo_list(name=stream_name)
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(name=[stream_name, channel_name])
|
||||
self.assertEqual(4, len(r))
|
||||
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(name=['beef'])
|
||||
self.assertEqual(0, len(r))
|
||||
|
||||
r = await self.txo_list()
|
||||
self.assertEqual(9, len(r))
|
||||
await self.stream_abandon(claim_id=stream_id)
|
||||
r = await self.txo_list()
|
||||
self.assertEqual(10, len(r))
|
||||
r = await self.txo_list(claim_id=stream_id)
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertTrue(r[0]['is_spent'])
|
||||
self.assertTrue(r[1]['is_spent'])
|
||||
|
||||
async def test_txo_list_received_filtering(self):
|
||||
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
|
||||
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
|
||||
await self.channel_create(claim_address=address2)
|
||||
|
||||
r = await self.txo_list(include_is_received=True)
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertFalse(r[0]['is_received'])
|
||||
self.assertTrue(r[1]['is_received'])
|
||||
rt = await self.txo_list(is_not_received=True)
|
||||
self.assertEqual(1, len(rt))
|
||||
self.assertEqual(rt[0], r[0])
|
||||
rf = await self.txo_list(is_received=True)
|
||||
self.assertEqual(1, len(rf))
|
||||
self.assertEqual(rf[0], r[1])
|
||||
|
||||
r = await self.txo_list(include_is_received=True, wallet_id=wallet2.id)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertTrue(r[0]['is_received'])
|
||||
|
||||
|
||||
class ClaimCommands(ClaimTestCase):
|
||||
|
||||
async def test_claim_list_type_filtering(self):
|
||||
await self.channel_create()
|
||||
await self.stream_create()
|
||||
async def test_claim_list_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
|
||||
# type filtering
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
@@ -417,6 +520,31 @@ class ClaimCommands(ClaimTestCase):
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
|
||||
|
||||
# claim_id filtering
|
||||
r = await self.claim_list(claim_id=stream_id)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.claim_list(claim_id=[stream_id, channel_id])
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
|
||||
stream_name, channel_name = (c['name'] for c in r)
|
||||
|
||||
r = await self.claim_list(claim_id=['beef'])
|
||||
self.assertEqual(0, len(r))
|
||||
|
||||
# claim_name filtering
|
||||
r = await self.claim_list(name=stream_name)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.claim_list(name=[stream_name, channel_name])
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.claim_list(name=['beef'])
|
||||
self.assertEqual(0, len(r))
|
||||
|
||||
async def test_claim_stream_channel_list_with_resolve(self):
|
||||
self.assertListEqual([], await self.claim_list(resolve=True))
|
||||
|
||||
@@ -1364,11 +1492,12 @@ class StreamCommands(ClaimTestCase):
|
||||
|
||||
tx = await self.stream_create(bid='2.5') # creates new claim
|
||||
claim_id = self.get_claim_id(tx)
|
||||
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
|
||||
txs = await self.transaction_list()
|
||||
self.assertEqual(len(txs[0]['claim_info']), 1)
|
||||
self.assertEqual(txs[0]['confirmations'], 1)
|
||||
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
|
||||
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim_id)
|
||||
self.assertFalse(txs[0]['claim_info'][0]['is_spent'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
await self.assertBalance(self.account, '7.479893')
|
||||
@@ -1378,19 +1507,23 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
|
||||
|
||||
await self.stream_update(claim_id, bid='1.0') # updates previous claim
|
||||
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
|
||||
txs = await self.transaction_list()
|
||||
self.assertEqual(len(txs[0]['update_info']), 1)
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
|
||||
self.assertFalse(txs[0]['update_info'][0]['is_spent'])
|
||||
self.assertTrue(txs[1]['claim_info'][0]['is_spent'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0002165')
|
||||
await self.assertBalance(self.account, '8.9796765')
|
||||
|
||||
await self.stream_abandon(claim_id)
|
||||
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
|
||||
txs = await self.transaction_list()
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
|
||||
self.assertTrue(txs[1]['update_info'][0]['is_spent'])
|
||||
self.assertTrue(txs[2]['claim_info'][0]['is_spent'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
await self.assertBalance(self.account, '9.9795695')
|
||||
@@ -1489,22 +1622,22 @@ class SupportCommands(CommandTestCase):
|
||||
await self.assertBalance(account2, '3.9998585')
|
||||
|
||||
# verify that the incoming tip is marked correctly as is_tip=True in account1
|
||||
txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items']
|
||||
txs = await self.transaction_list(account_id=self.account.id)
|
||||
self.assertEqual(len(txs[0]['support_info']), 1)
|
||||
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id)
|
||||
self.assertTrue(txs[0]['support_info'][0]['is_tip'])
|
||||
self.assertFalse(txs[0]['support_info'][0]['is_spent'])
|
||||
self.assertEqual(txs[0]['value'], '1.0')
|
||||
self.assertEqual(txs[0]['fee'], '0.0')
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = (await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
|
||||
))['items']
|
||||
txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
|
||||
self.assertTrue(txs2[0]['support_info'][0]['is_tip'])
|
||||
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
|
||||
self.assertEqual(txs2[0]['value'], '-1.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
@@ -1520,14 +1653,24 @@ class SupportCommands(CommandTestCase):
|
||||
await self.assertBalance(account2, '1.999717')
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items']
|
||||
txs2 = await self.transaction_list(wallet_id='wallet2')
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
|
||||
self.assertFalse(txs2[0]['support_info'][0]['is_tip'])
|
||||
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
# abandoning the tip increases balance and shows tip as spent
|
||||
await self.support_abandon(claim_id)
|
||||
await self.assertBalance(self.account, '4.979662')
|
||||
txs = await self.transaction_list(account_id=self.account.id)
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(len(txs[1]['support_info']), 1)
|
||||
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
|
||||
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
|
||||
|
||||
|
||||
class CollectionCommands(CommandTestCase):
|
||||
|
||||
|
||||
@@ -30,7 +30,9 @@ class NetworkTests(IntegrationTestCase):
|
||||
'payment_address': '',
|
||||
'donation_address': '',
|
||||
'daily_fee': '0',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
@@ -54,7 +56,9 @@ class NetworkTests(IntegrationTestCase):
|
||||
'payment_address': payment_address,
|
||||
'donation_address': donation_address,
|
||||
'daily_fee': '42',
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
@@ -2,10 +2,60 @@ import asyncio
|
||||
import os
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.schema import Claim
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.wallet import Transaction
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
async def initialize_torrent(self, tx_to_update=None):
|
||||
if not hasattr(self, 'seeder_session'):
|
||||
self.seeder_session = TorrentSession(self.loop, None)
|
||||
self.addCleanup(self.seeder_session.stop)
|
||||
await self.seeder_session.bind(port=4040)
|
||||
btih = await self.seeder_session.add_fake_torrent()
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
if not tx_to_update:
|
||||
claim = Claim()
|
||||
claim.stream.update(bt_infohash=btih)
|
||||
tx = await Transaction.claim_create(
|
||||
'torrent', claim, 1, address, [self.account], self.account
|
||||
)
|
||||
else:
|
||||
claim = tx_to_update.outputs[0].claim
|
||||
claim.stream.update(bt_infohash=btih)
|
||||
tx = await Transaction.claim_update(
|
||||
tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
|
||||
self.client_session._session.add_dht_node(('localhost', 4040))
|
||||
self.client_session.wait_start = False # fixme: this is super slow on tests
|
||||
return tx, btih
|
||||
|
||||
async def test_download_torrent(self):
|
||||
tx, btih = await self.initialize_torrent()
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
# second call, see its there and move on
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih)
|
||||
self.assertIn(btih, self.client_session._handles)
|
||||
tx, new_btih = await self.initialize_torrent(tx)
|
||||
self.assertNotEqual(btih, new_btih)
|
||||
# claim now points to another torrent, update to it
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih)
|
||||
self.assertIn(new_btih, self.client_session._handles)
|
||||
self.assertNotIn(btih, self.client_session._handles)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
await self.daemon.jsonrpc_file_delete(delete_all=True)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
|
||||
self.assertNotIn(new_btih, self.client_session._handles)
|
||||
|
||||
async def create_streams_in_range(self, *args, **kwargs):
|
||||
self.stream_claim_ids = []
|
||||
@@ -223,11 +273,11 @@ class FileCommands(CommandTestCase):
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with open(original_path, 'wb') as handle:
|
||||
handle.write(b'some other stuff was there instead')
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
|
||||
stream = self.daemon.file_manager.get_filtered(stream_hash=file_info['stream_hash'])[0]
|
||||
file_info = (await self.file_list())[0]
|
||||
self.assertEqual(stream.file_name, file_info['file_name'])
|
||||
# checks if what the API shows is what he have at the very internal level.
|
||||
@@ -250,7 +300,7 @@ class FileCommands(CommandTestCase):
|
||||
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
self.daemon.stream_manager.stop()
|
||||
self.daemon.file_manager.stop()
|
||||
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
|
||||
self.assertFalse(os.path.isfile(path))
|
||||
|
||||
@@ -343,8 +393,8 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
# restart the daemon and make sure the fee is still there
|
||||
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].content_fee.raw, raw_content_fee)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
|
||||
@@ -21,8 +21,8 @@ def get_random_bytes(n: int) -> bytes:
|
||||
|
||||
class RangeRequests(CommandTestCase):
|
||||
async def _restart_stream_manager(self):
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
return
|
||||
|
||||
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
|
||||
|
||||
@@ -6,7 +6,7 @@ from lbry.conf import Config
|
||||
from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
@@ -21,7 +21,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.api = 'localhost:5299'
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
@@ -34,4 +34,4 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
with contextlib.redirect_stdout(actual_output):
|
||||
cli.main(["--api", "localhost:5299", "status"])
|
||||
actual_output = actual_output.getvalue()
|
||||
self.assertIn("connection_status", actual_output)
|
||||
self.assertIn("connection_status", actual_output)
|
||||
@@ -93,6 +93,11 @@ class TranscodeValidation(ClaimTestCase):
|
||||
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
|
||||
pathlib.Path(fixed_file).unlink()
|
||||
|
||||
async def test_max_bit_rate(self):
|
||||
self.conf.video_bitrate_maximum = 100
|
||||
with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"):
|
||||
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
|
||||
|
||||
async def test_video_format(self):
|
||||
file_name = self.make_name("bad_video_format_1")
|
||||
if not file_name.exists():
|
||||
@@ -130,7 +135,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
self.assertEqual(extension, "ogg")
|
||||
self.assertEqual(extension, "ogv")
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
@@ -143,7 +148,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
self.assertEqual("webm", extension)
|
||||
|
||||
extension = self.analyzer._get_best_container_extension("", "libtheora")
|
||||
self.assertEqual("ogg", extension)
|
||||
self.assertEqual("ogv", extension)
|
||||
|
||||
async def test_no_ffmpeg(self):
|
||||
self.conf.ffmpeg_folder = "I don't really exist/"
|
||||
|
||||
@@ -16,6 +16,7 @@ class TestComponentManager(AsyncioTestCase):
|
||||
[
|
||||
components.DatabaseComponent,
|
||||
components.ExchangeRateManagerComponent,
|
||||
components.TorrentComponent,
|
||||
components.UPnPComponent
|
||||
],
|
||||
[
|
||||
@@ -24,9 +25,9 @@ class TestComponentManager(AsyncioTestCase):
|
||||
components.WalletComponent
|
||||
],
|
||||
[
|
||||
components.FileManagerComponent,
|
||||
components.HashAnnouncerComponent,
|
||||
components.PeerProtocolServerComponent,
|
||||
components.StreamManagerComponent,
|
||||
components.WalletServerPaymentsComponent
|
||||
]
|
||||
]
|
||||
@@ -135,8 +136,8 @@ class FakeDelayedBlobManager(FakeComponent):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
class FakeDelayedStreamManager(FakeComponent):
|
||||
component_name = "stream_manager"
|
||||
class FakeDelayedFileManager(FakeComponent):
|
||||
component_name = "file_manager"
|
||||
depends_on = [FakeDelayedBlobManager.component_name]
|
||||
|
||||
async def start(self):
|
||||
@@ -153,7 +154,7 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT],
|
||||
wallet=FakeDelayedWallet,
|
||||
stream_manager=FakeDelayedStreamManager,
|
||||
file_manager=FakeDelayedFileManager,
|
||||
blob_manager=FakeDelayedBlobManager
|
||||
)
|
||||
|
||||
@@ -163,17 +164,17 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
|
||||
await self.advance(0)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
async def test_proper_stopping_of_components(self):
|
||||
asyncio.create_task(self.component_manager.start())
|
||||
@@ -182,18 +183,18 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
asyncio.create_task(self.component_manager.stop())
|
||||
await self.advance(0)
|
||||
self.assertFalse(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
await self.advance(1)
|
||||
self.assertFalse(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
await self.advance(1)
|
||||
self.assertFalse(self.component_manager.get_component('stream_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('wallet').running)
|
||||
@@ -76,7 +76,8 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
await self.stream.save_file(node=mock_node)
|
||||
self.stream.downloader.node = mock_node
|
||||
await self.stream.save_file()
|
||||
await self.stream.finished_write_attempt.wait()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
if stop_when_done:
|
||||
@@ -109,7 +110,6 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334, allow_localhost=True)
|
||||
|
||||
@@ -123,7 +123,8 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
await self.stream.save_file(node=mock_node)
|
||||
self.stream.downloader.node = mock_node
|
||||
await self.stream.save_file()
|
||||
await self.stream.finished_writing.wait()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
with open(self.stream.full_path, 'rb') as f:
|
||||
|
||||
@@ -39,7 +39,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(self.cleartext)
|
||||
|
||||
self.stream = await self.stream_manager.create_stream(file_path)
|
||||
self.stream = await self.stream_manager.create(file_path)
|
||||
|
||||
async def _test_reflect_stream(self, response_chunk_size):
|
||||
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
|
||||
|
||||
@@ -5,6 +5,8 @@ from unittest import mock
|
||||
import asyncio
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from lbry.file.file_manager import FileManager
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from lbry.testcase import get_fake_exchange_rate_manager
|
||||
from lbry.utils import generate_id
|
||||
@@ -107,10 +109,7 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
|
||||
async def mock_resolve(*args):
|
||||
result = {txo.meta['permanent_url']: txo}
|
||||
claims = [
|
||||
StreamManager._convert_to_old_resolve_output(manager, result)[txo.meta['permanent_url']]
|
||||
]
|
||||
await storage.save_claims(claims)
|
||||
await storage.save_claim_from_output(ledger, txo)
|
||||
return result
|
||||
manager.ledger.resolve = mock_resolve
|
||||
|
||||
@@ -135,11 +134,20 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
)
|
||||
self.sd_hash = descriptor.sd_hash
|
||||
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
|
||||
self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client),
|
||||
AnalyticsManager(self.client_config,
|
||||
binascii.hexlify(generate_id()).decode(),
|
||||
binascii.hexlify(generate_id()).decode()))
|
||||
analytics_manager = AnalyticsManager(
|
||||
self.client_config,
|
||||
binascii.hexlify(generate_id()).decode(),
|
||||
binascii.hexlify(generate_id()).decode()
|
||||
)
|
||||
self.stream_manager = StreamManager(
|
||||
self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client),
|
||||
analytics_manager
|
||||
)
|
||||
self.file_manager = FileManager(
|
||||
self.loop, self.client_config, self.mock_wallet, self.client_storage, analytics_manager
|
||||
)
|
||||
self.file_manager.source_managers['stream'] = self.stream_manager
|
||||
self.exchange_rate_manager = get_fake_exchange_rate_manager()
|
||||
|
||||
async def _test_time_to_first_bytes(self, check_post, error=None, after_setup=None):
|
||||
@@ -156,9 +164,9 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.stream_manager.analytics_manager._post = _check_post
|
||||
if error:
|
||||
with self.assertRaises(error):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
|
||||
else:
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertTrue(checked_analytics_event)
|
||||
|
||||
@@ -278,7 +286,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream_hash = stream.stream_hash
|
||||
self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream})
|
||||
self.assertTrue(stream.running)
|
||||
@@ -299,7 +307,8 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
)
|
||||
self.assertEqual(stored_status, "stopped")
|
||||
|
||||
await stream.save_file(node=self.stream_manager.node)
|
||||
stream.downloader.node = self.stream_manager.node
|
||||
await stream.save_file()
|
||||
await stream.finished_writing.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertTrue(stream.finished)
|
||||
@@ -310,7 +319,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
)
|
||||
self.assertEqual(stored_status, "finished")
|
||||
|
||||
await self.stream_manager.delete_stream(stream, True)
|
||||
await self.stream_manager.delete(stream, True)
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
@@ -322,7 +331,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
async def _test_download_error_on_start(self, expected_error, timeout=None):
|
||||
error = None
|
||||
try:
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout)
|
||||
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
@@ -398,7 +407,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
last_blob_hash = json.loads(sdf.read())['blobs'][-2]['blob_hash']
|
||||
self.server_blob_manager.delete_blob(last_blob_hash)
|
||||
self.client_config.blob_download_timeout = 0.1
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await stream.started_writing.wait()
|
||||
self.assertEqual('running', stream.status)
|
||||
self.assertIsNotNone(stream.full_path)
|
||||
@@ -430,7 +439,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await stream.finished_writing.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.stream_manager.stop()
|
||||
|
||||
@@ -11,6 +11,7 @@ commands =
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install lbry-libtorrent
|
||||
orchstr8 download
|
||||
blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs}
|
||||
datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs}
|
||||
|
||||
Reference in new issue
Block a user