Compare commits

..
10 Commits
Author SHA1 Message Date
Lex Berezhny 0aa535ed11 upgrade ubuntu
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (claims) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
ci / tests / integration (takeovers) (push) Waiting to run
ci / tests / integration (transactions) (push) Waiting to run
2022-10-18 17:27:05 -04:00
Lex Berezhny 926d39759d fixing tests 2022-10-18 17:27:05 -04:00
Lex Berezhny 52232101e0 upgrade pyyaml and pyinstaller 2022-10-18 17:27:05 -04:00
Lex Berezhny 6d305b2fed lint 2022-10-18 17:27:05 -04:00
Lex Berezhny e4fb23202a remove loop argument and other changes 2022-10-18 17:27:05 -04:00
Lex Berezhny a3be5f44cf upgrade cryptography 2022-10-18 17:27:05 -04:00
Lex Berezhny 0ac1fec992 pylint disable shuffle() arg check 2022-10-18 17:27:05 -04:00
Lex Berezhny 24c041c26e shuffle() needs custom random, removed loop from Event()/Queue() 2022-10-18 17:27:05 -04:00
Lex Berezhny a2dc1fd16f passing loop to asyncio functions is deprecated 2022-10-18 17:27:05 -04:00
Lex Berezhny 6fca266ae1 try building 3.9 2022-10-18 17:27:05 -04:00
34 changed files with 230 additions and 542 deletions

No files matched your search

+11 -11
View File
@@ -5,12 +5,12 @@ jobs:
lint:
name: lint
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
python-version: '3.9'
- name: extract pip cache
uses: actions/cache@v2
with:
@@ -26,7 +26,7 @@ jobs:
strategy:
matrix:
os:
- ubuntu-20.04
- ubuntu-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
@@ -34,7 +34,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
python-version: '3.9'
- name: set pip cache dir
id: pip-cache
run: echo "::set-output name=dir::$(pip cache dir)"
@@ -72,7 +72,7 @@ jobs:
tests-integration:
name: "tests / integration"
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
strategy:
matrix:
test:
@@ -96,7 +96,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
python-version: '3.9'
- if: matrix.test == 'other'
run: |
sudo apt-get update
@@ -123,7 +123,7 @@ jobs:
coverage:
needs: ["tests-unit", "tests-integration"]
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- name: finalize coverage report submission
env:
@@ -138,7 +138,7 @@ jobs:
strategy:
matrix:
os:
- ubuntu-18.04
- ubuntu-20.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
@@ -146,7 +146,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
python-version: '3.9'
- id: os-name
uses: ASzc/change-string-case-action@v1
with:
@@ -160,7 +160,7 @@ jobs:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install pyinstaller==4.6
- run: pip install pyinstaller==5.3
- run: pip install -e .
- if: startsWith(github.ref, 'refs/tags/v')
run: python docker/set_build.py
@@ -184,7 +184,7 @@ jobs:
name: "release"
if: startsWith(github.ref, 'refs/tags/v')
needs: ["build"]
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/download-artifact@v2
+1 -1
View File
@@ -7,7 +7,7 @@ on:
jobs:
release:
name: "slack notification"
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
id: markdown
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.112.0"
__version__ = "0.111.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
-2
View File
@@ -42,8 +42,6 @@ class BlobAnnouncer:
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
self.announcements_sent_metric.labels(peers=0, error=True).inc()
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
async def _announce(self, batch_size: typing.Optional[int] = 10):
+1 -2
View File
@@ -8,7 +8,6 @@ from prometheus_client import Gauge
from lbry import utils
from lbry.dht import constants
from lbry.dht.error import RemoteException
from lbry.dht.protocol.distance import Distance
if typing.TYPE_CHECKING:
from lbry.dht.peer import KademliaPeer, PeerManager
@@ -396,7 +395,7 @@ class TreeRoutingTable:
try:
await probe(to_replace)
return False
except (asyncio.TimeoutError, RemoteException):
except asyncio.TimeoutError:
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
if to_replace in self.buckets[bucket_index]:
+2 -2
View File
@@ -81,8 +81,8 @@ Code | Name | Message
511 | CorruptBlob | Blobs is corrupted.
520 | BlobFailedEncryption | Failed to encrypt blob.
531 | DownloadCancelled | Download was canceled.
532 | DownloadMetadataTimeout | Failed to download metadata for {download} within timeout.
533 | DownloadDataTimeout | Failed to download data blobs for {download} within timeout.
532 | DownloadSDTimeout | Failed to download sd blob {download} within timeout.
533 | DownloadDataTimeout | Failed to download data blobs for sd hash {download} within timeout.
534 | InvalidStreamDescriptor | {message}
535 | InvalidData | {message}
536 | InvalidBlobHash | {message}
+3 -3
View File
@@ -411,18 +411,18 @@ class DownloadCancelledError(BlobError):
super().__init__("Download was canceled.")
class DownloadMetadataTimeoutError(BlobError):
class DownloadSDTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download metadata for {download} within timeout.")
super().__init__(f"Failed to download sd blob {download} within timeout.")
class DownloadDataTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download data blobs for {download} within timeout.")
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
class InvalidStreamDescriptorError(BlobError):
-2
View File
@@ -560,8 +560,6 @@ class UPnPComponent(Component):
self.upnp = await UPnP.discover(loop=self.component_manager.loop)
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.warning("upnp discovery failed: %s", err)
self.upnp = None
+12 -17
View File
@@ -36,7 +36,7 @@ from lbry.blob.blob_file import is_valid_blobhash, BlobBuffer
from lbry.blob_exchange.downloader import download_blob
from lbry.dht.peer import make_kademlia_peer
from lbry.error import (
DownloadMetadataTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError,
InputValueError
@@ -639,7 +639,7 @@ class Daemon(metaclass=JSONRPCServerType):
stream = await self.jsonrpc_get(uri)
if isinstance(stream, dict):
raise web.HTTPServerError(text=stream['error'])
raise web.HTTPFound(f"/stream/{stream.identifier}")
raise web.HTTPFound(f"/stream/{stream.sd_hash}")
async def handle_stream_range_request(self, request: web.Request):
try:
@@ -658,13 +658,12 @@ class Daemon(metaclass=JSONRPCServerType):
log.debug("finished handling /stream range request")
async def _handle_stream_range_request(self, request: web.Request):
identifier = request.path.split("/stream/")[1]
sd_hash = request.path.split("/stream/")[1]
if not self.file_manager.started.is_set():
await self.file_manager.started.wait()
stream = self.file_manager.get_filtered(identifier=identifier)
if not stream:
if sd_hash not in self.file_manager.streams:
return web.HTTPNotFound()
return await self.file_manager.stream_partial_content(request, identifier)
return await self.file_manager.stream_partial_content(request, sd_hash)
async def _process_rpc_call(self, data):
args = data.get('params', {})
@@ -1140,7 +1139,7 @@ class Daemon(metaclass=JSONRPCServerType):
save_file=save_file, wallet=wallet
)
if not stream:
raise DownloadMetadataTimeoutError(uri)
raise DownloadSDTimeoutError(uri)
except Exception as e:
# TODO: use error from lbry.error
log.warning("Error downloading %s: %s", uri, str(e))
@@ -2944,21 +2943,19 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_sign(
self, channel_name=None, channel_id=None, hexdata=None, salt=None,
channel_account_id=None, wallet_id=None):
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
"""
Signs data using the specified channel signing key.
Usage:
channel_sign [<channel_name> | --channel_name=<channel_name>] [<channel_id> | --channel_id=<channel_id>]
[<hexdata> | --hexdata=<hexdata>] [<salt> | --salt=<salt>]
channel_sign [<channel_name> | --channel_name=<channel_name>]
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
--salt=<salt> : (str) salt to use for signing, default is to use timestamp
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
@@ -2975,13 +2972,11 @@ class Daemon(metaclass=JSONRPCServerType):
signing_channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
if salt is None:
salt = str(int(time.time()))
signature = signing_channel.sign_data(unhexlify(str(hexdata)), salt)
timestamp = str(int(time.time()))
signature = signing_channel.sign_data(unhexlify(str(hexdata)), timestamp)
return {
'signature': signature,
'signing_ts': salt, # DEPRECATED
'salt': salt,
'signing_ts': timestamp
}
@requires(WALLET_COMPONENT)
@@ -80,8 +80,6 @@ class MarketFeed:
self.rate = ExchangeRate(self.market, rate, int(time.time()))
self.last_check = time.time()
return self.rate
except asyncio.CancelledError:
raise
except asyncio.TimeoutError:
log.warning("Timed out fetching exchange rate from %s.", self.name)
except json.JSONDecodeError as e:
+12 -4
View File
@@ -285,7 +285,7 @@ class JSONResponseEncoder(JSONEncoder):
else:
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
result = {
'streaming_url': managed_stream.stream_url,
'streaming_url': None,
'completed': managed_stream.completed,
'file_name': None,
'download_directory': None,
@@ -293,10 +293,10 @@ class JSONResponseEncoder(JSONEncoder):
'points_paid': 0.0,
'stopped': not managed_stream.running,
'stream_hash': None,
'stream_name': managed_stream.stream_name,
'suggested_file_name': managed_stream.suggested_file_name,
'stream_name': None,
'suggested_file_name': None,
'sd_hash': None,
'mime_type': managed_stream.mime_type,
'mime_type': None,
'key': None,
'total_bytes_lower_bound': total_bytes_lower_bound,
'total_bytes': total_bytes,
@@ -326,8 +326,12 @@ class JSONResponseEncoder(JSONEncoder):
}
if is_stream:
result.update({
'streaming_url': managed_stream.stream_url,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.stream_name,
'suggested_file_name': managed_stream.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
@@ -336,6 +340,10 @@ class JSONResponseEncoder(JSONEncoder):
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
})
else:
result.update({
'streaming_url': f'file://{managed_stream.full_path}',
})
if output_exists:
result.update({
'file_name': managed_stream.file_name,
+8 -24
View File
@@ -5,7 +5,6 @@ import typing
import asyncio
import binascii
import time
from operator import itemgetter
from typing import Optional
from lbry.wallet import SQLiteMixin
from lbry.conf import Config
@@ -212,7 +211,7 @@ def delete_torrent(transaction: sqlite3.Connection, bt_infohash: str):
transaction.execute("delete from torrent where bt_infohash=?", (bt_infohash,)).fetchall()
def store_file(transaction: sqlite3.Connection, identifier_value: str, file_name: typing.Optional[str],
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
content_fee: typing.Optional[Transaction], added_on: typing.Optional[int] = None) -> int:
if not file_name and not download_directory:
@@ -220,18 +219,15 @@ def store_file(transaction: sqlite3.Connection, identifier_value: str, file_name
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
is_torrent = len(identifier_value) == 40
time_added = added_on or int(time.time())
transaction.execute(
f"insert or replace into file values ({'NULL, ?' if is_torrent else '?, NULL'}, ?, ?, ?, ?, ?, ?, ?)",
(identifier_value, encoded_file_name, encoded_download_dir, data_payment_rate, status,
"insert or replace into file values (?, NULL, ?, ?, ?, ?, ?, ?, ?)",
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
None if not content_fee else binascii.hexlify(content_fee.raw).decode(), time_added)
).fetchall()
return transaction.execute(
f"select rowid from file where {'bt_infohash' if is_torrent else 'stream_hash'}=?",
(identifier_value, )).fetchone()[0]
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
class SQLiteStorage(SQLiteMixin):
@@ -636,13 +632,6 @@ class SQLiteStorage(SQLiteMixin):
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
async def get_all_torrent_files(self) -> typing.List[typing.Dict]:
def _get_all_torrent_files(transaction):
cursor = transaction.execute(
"select file.ROWID as rowid, * from file join torrent on file.bt_infohash=torrent.bt_infohash")
return map(lambda row: dict(zip(list(map(itemgetter(0), cursor.description)), row)), cursor.fetchall())
return list(await self.db.run(_get_all_torrent_files))
def change_file_status(self, stream_hash: str, new_status: str):
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
@@ -883,20 +872,15 @@ class SQLiteStorage(SQLiteMixin):
if stream_hash in self.content_claim_callbacks:
await self.content_claim_callbacks[stream_hash]()
async def add_torrent(self, bt_infohash, length, name):
def _save_torrent(transaction, bt_infohash, length, name):
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
def _save_torrent(transaction):
transaction.execute(
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
).fetchall()
return await self.db.run(_save_torrent, bt_infohash, length, name)
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
def _save_torrent_claim(transaction):
transaction.execute(
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
).fetchall()
await self.add_torrent(bt_infohash, length, name)
await self.db.run(_save_torrent_claim)
await self.db.run(_save_torrent)
# update corresponding ManagedEncryptedFileDownloader object
if bt_infohash in self.content_claim_callbacks:
await self.content_claim_callbacks[bt_infohash]()
@@ -914,7 +898,7 @@ class SQLiteStorage(SQLiteMixin):
async def get_content_claim_for_torrent(self, bt_infohash):
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
return claims[bt_infohash] if claims else None
return claims[bt_infohash].as_dict() if claims else None
# # # # # # # # # reflector functions # # # # # # # # #
+19 -30
View File
@@ -3,7 +3,7 @@ import logging
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.error import ResolveError, DownloadMetadataTimeoutError, InsufficientFundsError
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
from lbry.error import InvalidStreamURLError
from lbry.stream.managed_stream import ManagedStream
@@ -13,12 +13,11 @@ from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
from lbry.extras.daemon.storage import StoredContentClaim
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager
from lbry.wallet import WalletManager, Output
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
@@ -99,8 +98,6 @@ class FileManager:
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
if 'error' in resolved_result:
@@ -139,7 +136,7 @@ class FileManager:
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
existing[0].set_claim(claim_info.as_dict() if claim_info else None, claim)
existing[0].set_claim(claim_info, claim)
else:
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
@@ -195,24 +192,21 @@ class FileManager:
####################
# make downloader and wait for start
####################
# temporary with fields we know so downloader can start. Missing fields are populated later.
stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
amount=txo.amount, height=txo.tx_ref.height,
serialized=claim.to_bytes().hex())
if not claim.stream.source.bt_infohash:
# fixme: this shouldnt be here
stream = ManagedStream(
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager, claim=stored_claim
analytics_manager=self.analytics_manager
)
stream.downloader.node = source_manager.node
else:
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
file_name=file_name, download_directory=download_directory or self.config.download_dir,
status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
status=ManagedStream.STATUS_RUNNING,
analytics_manager=self.analytics_manager,
torrent_session=source_manager.torrent_session
)
log.info("starting download for %s", uri)
@@ -242,15 +236,15 @@ class FileManager:
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
stream.set_claim(claim_info.as_dict() if claim_info else None, claim)
stream.set_claim(claim_info, claim)
if save_file:
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download))
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.identifier)
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadMetadataTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
except (Exception, asyncio.CancelledError) as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
@@ -290,24 +284,19 @@ class FileManager:
)
)
async def stream_partial_content(self, request: Request, identifier: str):
for source_manager in self.source_managers.values():
if source_manager.get_filtered(identifier=identifier):
return await source_manager.stream_partial_content(request, identifier)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
"""
Get a list of filtered and sorted ManagedDownloadSource objects from all available source managers
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
result = last_error = None
for manager in self.source_managers.values():
try:
result = (result or []) + manager.get_filtered(*args, **kwargs)
except ValueError as error:
last_error = error
if result is not None:
return result
raise last_error
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
async def delete(self, source: ManagedDownloadSource, delete_file=False):
for manager in self.source_managers.values():
+4 -13
View File
@@ -1,6 +1,5 @@
import os
import asyncio
import time
import typing
import logging
import binascii
@@ -44,7 +43,7 @@ class ManagedDownloadSource:
self.rowid = rowid
self.content_fee = content_fee
self.purchase_receipt = None
self._added_on = added_on or int(time.time())
self._added_on = added_on
self.analytics_manager = analytics_manager
self.downloader = None
@@ -92,14 +91,6 @@ class ManagedDownloadSource:
def added_on(self) -> Optional[int]:
return self._added_on
@property
def suggested_file_name(self):
return self._file_name
@property
def stream_name(self):
return self.suggested_file_name
@property
def status(self) -> str:
return self._status
@@ -108,9 +99,9 @@ class ManagedDownloadSource:
def completed(self):
raise NotImplementedError()
@property
def stream_url(self):
return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.identifier}"
# @property
# def stream_url(self):
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
@property
def finished(self) -> bool:
-2
View File
@@ -23,7 +23,6 @@ COMPARISON_OPERATORS = {
class SourceManager:
filter_fields = {
'identifier',
'rowid',
'status',
'file_name',
@@ -84,7 +83,6 @@ class SourceManager:
raise NotImplementedError()
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
await self.storage.delete_torrent(source.identifier)
self.remove(source)
if delete_file and source.output_file_exists:
os.remove(source.full_path)
-2
View File
@@ -22,8 +22,6 @@ class BackgroundDownloader:
await downloader.download_stream_blob(blob_info)
except ValueError:
return
except asyncio.CancelledError:
raise
except Exception:
log.error("Unexpected download error on background downloader")
finally:
+2 -2
View File
@@ -4,7 +4,7 @@ import logging
import binascii
from lbry.dht.node import get_kademlia_peers_from_hosts
from lbry.error import DownloadMetadataTimeoutError
from lbry.error import DownloadSDTimeoutError
from lbry.utils import lru_cache_concurrent
from lbry.stream.descriptor import StreamDescriptor
from lbry.blob_exchange.downloader import BlobDownloader
@@ -77,7 +77,7 @@ class StreamDownloader:
log.info("downloaded sd blob %s", self.sd_hash)
self.time_to_descriptor = self.loop.time() - now
except asyncio.TimeoutError:
raise DownloadMetadataTimeoutError(self.sd_hash)
raise DownloadSDTimeoutError(self.sd_hash)
# parse the descriptor
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
+8 -4
View File
@@ -5,7 +5,7 @@ import typing
import logging
from typing import Optional
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbry.error import DownloadMetadataTimeoutError
from lbry.error import DownloadSDTimeoutError
from lbry.schema.mime_types import guess_media_type
from lbry.stream.downloader import StreamDownloader
from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
@@ -104,6 +104,10 @@ class ManagedStream(ManagedDownloadSource):
def completed(self):
return self.written_bytes >= self.descriptor.lower_bound_decrypted_length()
@property
def stream_url(self):
return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}"
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
@@ -160,7 +164,7 @@ class ManagedStream(ManagedDownloadSource):
await asyncio.wait_for(self.downloader.start(), timeout)
except asyncio.TimeoutError:
self._running.clear()
raise DownloadMetadataTimeoutError(self.identifier)
raise DownloadSDTimeoutError(self.sd_hash)
if self.delayed_stop_task and not self.delayed_stop_task.done():
self.delayed_stop_task.cancel()
@@ -275,7 +279,7 @@ class ManagedStream(ManagedDownloadSource):
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
self.sd_hash[:6], self.full_path)
await self.blob_manager.storage.set_saved_file(self.stream_hash)
except Exception as err:
except (Exception, asyncio.CancelledError) as err:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
@@ -362,7 +366,7 @@ class ManagedStream(ManagedDownloadSource):
return sent
except ConnectionError:
return sent
except (OSError, Exception) as err:
except (OSError, Exception, asyncio.CancelledError) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
elif isinstance(err, OSError):
+1 -3
View File
@@ -32,7 +32,7 @@ def path_or_none(encoded_path) -> Optional[str]:
class StreamManager(SourceManager):
_sources: typing.Dict[str, ManagedStream]
filter_fields = set(SourceManager.filter_fields)
filter_fields = SourceManager.filter_fields
filter_fields.update({
'sd_hash',
'stream_hash',
@@ -164,8 +164,6 @@ class StreamManager(SourceManager):
async def reflect_streams(self):
try:
return await self._reflect_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
-1
View File
@@ -394,7 +394,6 @@ class CommandTestCase(IntegrationTestCase):
logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
logging.getLogger('lbry.torrent').setLevel(self.VERBOSITY)
logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
await super().asyncSetUp()
+68 -133
View File
@@ -3,8 +3,9 @@ import binascii
import os
import logging
import random
from hashlib import sha1
from tempfile import mkdtemp
from typing import Optional, Tuple, Dict
from typing import Optional
import libtorrent
@@ -13,8 +14,6 @@ log = logging.getLogger(__name__)
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
libtorrent.add_torrent_params_flags_t.flag_auto_managed
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
| libtorrent.add_torrent_params_flags_t.flag_sequential_download
| libtorrent.add_torrent_params_flags_t.flag_paused
)
@@ -23,102 +22,66 @@ class TorrentHandle:
self._loop = loop
self._executor = executor
self._handle: libtorrent.torrent_handle = handle
self.finished = asyncio.Event(loop=loop)
self.metadata_completed = asyncio.Event(loop=loop)
self.size = handle.status().total_wanted
self.started = asyncio.Event()
self.finished = asyncio.Event()
self.metadata_completed = asyncio.Event()
self.size = 0
self.total_wanted_done = 0
self.name = ''
self.tasks = []
self._torrent_info: libtorrent.torrent_info = handle.torrent_file()
self.torrent_file: Optional[libtorrent.file_storage] = None
self._base_path = None
self._handle.set_sequential_download(1)
@property
def torrent_file(self) -> Optional[libtorrent.file_storage]:
return self._torrent_info.files()
def full_path_at(self, file_num) -> Optional[str]:
if self.torrent_file is None:
def largest_file(self) -> Optional[str]:
if not self.torrent_file:
return None
return os.path.join(self.save_path, self.torrent_file.file_path(file_num))
def size_at(self, file_num) -> Optional[int]:
if self.torrent_file is not None:
return self.torrent_file.file_size(file_num)
index = self.largest_file_index
return os.path.join(self._base_path, self.torrent_file.at(index).path)
@property
def save_path(self) -> Optional[str]:
if not self._base_path:
self._base_path = self._handle.status().save_path
return self._base_path
def index_from_name(self, file_name):
def largest_file_index(self):
largest_size, index = 0, 0
for file_num in range(self.torrent_file.num_files()):
if '.pad' in self.torrent_file.file_path(file_num):
continue # ignore padding files
if file_name == os.path.basename(self.full_path_at(file_num)):
return file_num
if self.torrent_file.file_size(file_num) > largest_size:
largest_size = self.torrent_file.file_size(file_num)
index = file_num
return index
def stop_tasks(self):
self._handle.save_resume_data()
while self.tasks:
self.tasks.pop().cancel()
def byte_range_to_piece_range(
self, file_index, start_offset, end_offset) -> Tuple[libtorrent.peer_request, libtorrent.peer_request]:
start_piece = self._torrent_info.map_file(file_index, start_offset, 0)
end_piece = self._torrent_info.map_file(file_index, end_offset, 0)
return start_piece, end_piece
async def stream_range_as_completed(self, file_name, start, end):
file_index = self.index_from_name(file_name)
if file_index is None:
raise ValueError(f"Attempt to stream from invalid file. Expected name: {file_name}")
first_piece, final_piece = self.byte_range_to_piece_range(file_index, start, end)
start_piece_offset = first_piece.start
piece_size = self._torrent_info.piece_length()
log.info("Streaming torrent from piece %d to %d (bytes: %d -> %d, piece size: %d): %s",
first_piece.piece, final_piece.piece, start, end, piece_size, self.name)
self.prioritize(file_index, start, end)
for piece_index in range(first_piece.piece, final_piece.piece + 1):
while not self._handle.have_piece(piece_index):
log.info("Waiting for piece %d: %s", piece_index, self.name)
self._handle.set_piece_deadline(piece_index, 0)
await asyncio.sleep(0.2)
log.info("Streaming piece offset %d / %d for torrent %s", piece_index, final_piece.piece, self.name)
yield piece_size - start_piece_offset
def _show_status(self):
# fixme: cleanup
if not self._handle.is_valid():
return
status = self._handle.status()
self._base_path = status.save_path
if status.has_metadata:
self.size = status.total_wanted
self.total_wanted_done = status.total_wanted_done
self.name = status.name
if not self.metadata_completed.is_set():
self.metadata_completed.set()
self._torrent_info = self._handle.torrent_file()
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.num_seeds, status.state, status.save_path)
if (status.is_finished or status.is_seeding) and not self.finished.is_set():
self.torrent_file = self._handle.get_torrent_info().files()
self._base_path = status.save_path
first_piece = self.torrent_file.at(self.largest_file_index).offset
if not self.started.is_set():
if self._handle.have_piece(first_piece):
self.started.set()
else:
# prioritize it
self._handle.set_piece_deadline(first_piece, 100)
if not status.is_seeding:
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.num_seeds, status.state, status.save_path)
elif not self.finished.is_set():
self.finished.set()
log.info("Torrent finished: %s", self.name)
def prioritize(self, file_index, start, end, cleanup=False):
first_piece, last_piece = self.byte_range_to_piece_range(file_index, start, end)
priorities = self._handle.get_piece_priorities()
priorities = [0 if cleanup else 1 for _ in priorities]
self._handle.clear_piece_deadlines()
for idx, piece_number in enumerate(range(first_piece.piece, last_piece.piece)):
priorities[piece_number] = 7 - idx if 0 <= idx <= 6 else 1
self._handle.set_piece_deadline(piece_number, idx)
log.debug("Prioritizing pieces for %s: %s", self.name, priorities)
self._handle.prioritize_pieces(priorities)
async def status_loop(self):
while True:
self._show_status()
@@ -142,21 +105,19 @@ class TorrentSession:
self._loop = loop
self._executor = executor
self._session: Optional[libtorrent.session] = None
self._handles: Dict[str, TorrentHandle] = {}
self._handles = {}
self.tasks = []
self.wait_start = True
def add_peer(self, btih, addr, port):
self._handles[btih]._handle.connect_peer((addr, port))
async def add_fake_torrent(self, file_count=3):
async def add_fake_torrent(self):
tmpdir = mkdtemp()
info = _create_fake_torrent(tmpdir, file_count=file_count)
info, btih = _create_fake_torrent(tmpdir)
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
handle = self._session.add_torrent({
'ti': info, 'save_path': tmpdir, 'flags': flags
})
self._handles[str(info.info_hash())] = TorrentHandle(self._loop, self._executor, handle)
return str(info.info_hash())
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
return btih
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
settings = {
@@ -170,14 +131,15 @@ class TorrentSession:
self.tasks.append(self._loop.create_task(self.process_alerts()))
def stop(self):
while self._handles:
self._handles.popitem()[1].stop_tasks()
while self.tasks:
self.tasks.pop().cancel()
if self._session:
self._session.save_state()
self._session.pause()
self._session = None
self._session.save_state()
self._session.pause()
self._session.stop_dht()
self._session.stop_lsd()
self._session.stop_natpmp()
self._session.stop_upnp()
self._session = None
def _pop_alerts(self):
for alert in self._session.pop_alerts():
@@ -211,23 +173,18 @@ class TorrentSession:
handle.force_dht_announce()
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
def full_path(self, btih, file_num) -> Optional[str]:
return self._handles[btih].full_path_at(file_num)
def save_path(self, btih):
return self._handles[btih].save_path
def has_torrent(self, btih):
return btih in self._handles
def full_path(self, btih):
return self._handles[btih].largest_file
async def add_torrent(self, btih, download_path):
if btih in self._handles:
return await self._handles[btih].metadata_completed.wait()
await self._loop.run_in_executor(
self._executor, self._add_torrent, btih, download_path
)
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
await self._handles[btih].metadata_completed.wait()
if self.wait_start:
# fixme: temporary until we add streaming support, otherwise playback fails!
await self._handles[btih].started.wait()
def remove_torrent(self, btih, remove_files=False):
if btih in self._handles:
@@ -240,17 +197,9 @@ class TorrentSession:
handle = self._handles[btih]
await handle.resume()
def get_total_size(self, btih):
def get_size(self, btih):
return self._handles[btih].size
def get_index_from_name(self, btih, file_name):
return self._handles[btih].index_from_name(file_name)
def get_size(self, btih, file_name) -> Optional[int]:
for (path, size) in self.get_files(btih).items():
if os.path.basename(path) == file_name:
return size
def get_name(self, btih):
return self._handles[btih].name
@@ -260,38 +209,23 @@ class TorrentSession:
def is_completed(self, btih):
return self._handles[btih].finished.is_set()
def stream_file(self, btih, file_name, start, end):
handle = self._handles[btih]
return handle.stream_range_as_completed(file_name, start, end)
def get_files(self, btih) -> Dict:
handle = self._handles[btih]
return {
self.full_path(btih, file_num): handle.torrent_file.file_size(file_num)
for file_num in range(handle.torrent_file.num_files())
if '.pad' not in handle.torrent_file.file_path(file_num)
}
def get_magnet_uri(btih):
return f"magnet:?xt=urn:btih:{btih}"
def _create_fake_torrent(tmpdir, file_count=3, largest_index=1):
# layout: subdir/tmp{0..file_count-1} files. v1+v2. automatic piece size.
# largest_index: which file index {0 ... file_count} will be the largest file
def _create_fake_torrent(tmpdir):
# beware, that's just for testing
path = os.path.join(tmpdir, 'tmp')
with open(path, 'wb') as myfile:
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
file_storage = libtorrent.file_storage()
subfolder = os.path.join(tmpdir, "subdir")
os.mkdir(subfolder)
for file_number in range(file_count):
file_name = f"tmp{file_number}"
with open(os.path.join(subfolder, file_name), 'wb') as myfile:
size = myfile.write(
bytes([random.randint(0, 255) for _ in range(10 - abs(file_number - largest_index))]) * 1024)
file_storage.add_file(os.path.join("subdir", file_name), size)
t = libtorrent.create_torrent(file_storage, 0, 0)
file_storage.add_file('tmp', size)
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
libtorrent.set_piece_hashes(t, tmpdir)
return libtorrent.torrent_info(t.generate())
info = libtorrent.torrent_info(t.generate())
btih = sha1(info.metadata()).hexdigest()
return info, btih
async def main():
@@ -304,16 +238,17 @@ async def main():
executor = None
session = TorrentSession(asyncio.get_event_loop(), executor)
await session.bind()
await session.add_torrent(btih, os.path.expanduser("~/Downloads"))
session2 = TorrentSession(asyncio.get_event_loop(), executor)
await session.bind('localhost', port=4040)
await session2.bind('localhost', port=4041)
btih = await session.add_fake_torrent()
session2._session.add_dht_node(('localhost', 4040))
await session2.add_torrent(btih, "/tmp/down")
while True:
session.full_path(btih, 0)
await asyncio.sleep(1)
await asyncio.sleep(100)
await session.pause()
executor.shutdown()
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
log = logging.getLogger(__name__)
asyncio.run(main())
+20 -112
View File
@@ -1,14 +1,12 @@
import asyncio
import binascii
import logging
import os
import typing
from typing import Optional
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbry.error import DownloadMetadataTimeoutError
from aiohttp.web import Request
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
from lbry.schema.mime_types import guess_media_type
if typing.TYPE_CHECKING:
from lbry.torrent.session import TorrentSession
@@ -21,6 +19,12 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
def path_or_none(encoded_path) -> Optional[str]:
if not encoded_path:
return
return binascii.unhexlify(encoded_path).decode()
class TorrentSource(ManagedDownloadSource):
STATUS_STOPPED = "stopped"
filter_fields = SourceManager.filter_fields
@@ -38,55 +42,15 @@ class TorrentSource(ManagedDownloadSource):
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
rowid, content_fee, analytics_manager, added_on)
self.torrent_session = torrent_session
self._suggested_file_name = None
self._full_path = None
@property
def full_path(self) -> Optional[str]:
if not self._full_path:
self._full_path = self.select_path()
self._file_name = os.path.basename(self._full_path)
self.download_directory = self.torrent_session.save_path(self.identifier)
return self._full_path
def select_path(self):
wanted_name = (self.stream_claim_info.claim.stream.source.name or '') if self.stream_claim_info else ''
wanted_index = self.torrent_session.get_index_from_name(self.identifier, wanted_name)
if wanted_index is None:
# maybe warn?
largest = (None, -1)
for (path, size) in self.torrent_session.get_files(self.identifier).items():
largest = (path, size) if size > largest[1] else largest
return largest[0]
else:
return self.torrent_session.full_path(self.identifier, wanted_index or 0)
@property
def suggested_file_name(self):
self._suggested_file_name = self._suggested_file_name or os.path.basename(self.select_path())
return self._suggested_file_name
@property
def mime_type(self) -> Optional[str]:
return guess_media_type(os.path.basename(self.full_path))[0]
async def setup(self, timeout: Optional[float] = None):
try:
metadata_download = self.torrent_session.add_torrent(self.identifier, self.download_directory)
await asyncio.wait_for(metadata_download, timeout, loop=self.loop)
except asyncio.TimeoutError:
self.torrent_session.remove_torrent(btih=self.identifier)
raise DownloadMetadataTimeoutError(self.identifier)
self.download_directory = self.torrent_session.save_path(self.identifier)
self._file_name = os.path.basename(self.full_path)
full_path = self.torrent_session.full_path(self.identifier)
self.download_directory = os.path.dirname(full_path)
return full_path
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
await self.setup(timeout)
if not self.rowid:
await self.storage.add_torrent(self.identifier, self.torrent_length, self.torrent_name)
self.rowid = await self.storage.save_downloaded_file(
self.identifier, self.file_name, self.download_directory, 0.0, added_on=self._added_on
)
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
async def stop(self, finished: bool = False):
await self.torrent_session.remove_torrent(self.identifier)
@@ -96,11 +60,7 @@ class TorrentSource(ManagedDownloadSource):
@property
def torrent_length(self):
return self.torrent_session.get_total_size(self.identifier)
@property
def stream_length(self):
return self.torrent_session.get_size(self.identifier, self.file_name)
return self.torrent_session.get_size(self.identifier)
@property
def written_bytes(self):
@@ -121,56 +81,6 @@ class TorrentSource(ManagedDownloadSource):
def completed(self):
return self.torrent_session.is_completed(self.identifier)
@property
def status(self):
return self.STATUS_FINISHED if self.completed else self.STATUS_RUNNING
async def stream_file(self, request):
log.info("stream torrent to browser for lbry://%s#%s (btih %s...)", self.claim_name, self.claim_id,
self.identifier[:6])
headers, start, end = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
target = self.suggested_file_name
await self.start()
response = StreamResponse(
status=206,
headers=headers
)
await response.prepare(request)
while not os.path.exists(self.full_path):
async for _ in self.torrent_session.stream_file(self.identifier, target, start, end):
break
with open(self.full_path, 'rb') as infile:
infile.seek(start)
async for read_size in self.torrent_session.stream_file(self.identifier, target, start, end):
if infile.tell() + read_size < end:
await response.write(infile.read(read_size))
else:
await response.write_eof(infile.read(end - infile.tell() + 1))
return response
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
if '=' in get_range:
get_range = get_range.split('=')[1]
start, end = get_range.split('-')
size = self.stream_length
start = int(start)
end = int(end) if end else size - 1
if end >= size or not 0 <= start < size:
raise HTTPRequestRangeNotSatisfiable()
final_size = end - start + 1
headers = {
'Accept-Ranges': 'bytes',
'Content-Range': f'bytes {start}-{end}/{size}',
'Content-Length': str(final_size),
'Content-Type': self.mime_type
}
return headers, start, end
class TorrentManager(SourceManager):
_sources: typing.Dict[str, ManagedDownloadSource]
@@ -193,7 +103,7 @@ class TorrentManager(SourceManager):
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int], **kwargs):
added_on: Optional[int]):
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
@@ -201,14 +111,9 @@ class TorrentManager(SourceManager):
torrent_session=self.torrent_session
)
self.add(stream)
await stream.setup()
async def initialize_from_database(self):
for file in await self.storage.get_all_torrent_files():
claim = await self.storage.get_content_claim_for_torrent(file['bt_infohash'])
file['download_directory'] = bytes.fromhex(file['download_directory'] or '').decode() or None
file['file_name'] = bytes.fromhex(file['file_name'] or '').decode() or None
await self._load_stream(claim=claim, **file)
pass
async def start(self):
await super().start()
@@ -227,6 +132,9 @@ class TorrentManager(SourceManager):
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
raise NotImplementedError
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
# await self.storage.delete_stream(source.descriptor)
async def stream_partial_content(self, request: Request, identifier: str):
return await self._sources[identifier].stream_file(request)
async def stream_partial_content(self, request: Request, sd_hash: str):
raise NotImplementedError
+1 -1
View File
@@ -141,7 +141,7 @@ class CoinSelector:
_) -> List[OutputEffectiveAmountEstimator]:
""" Accumulate UTXOs at random until there is enough to cover the target. """
target = self.target + self.cost_of_change
self.random.shuffle(txos, self.random.random)
self.random.shuffle(txos, random=self.random.random) # pylint: disable=deprecated-argument
selection = []
amount = 0
for coin in txos:
+3 -9
View File
@@ -938,9 +938,7 @@ class Ledger(metaclass=LedgerRegistry):
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change,
account.change.gap, channel_count, len(account.channel_keys), claim_count)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
except Exception:
log.exception(
'Failed to display wallet state, please file issue '
'for this bug along with the traceback you see below:')
@@ -963,9 +961,7 @@ class Ledger(metaclass=LedgerRegistry):
claim_ids = [p.purchased_claim_id for p in purchases]
try:
resolved, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
except Exception:
log.exception("Resolve failed while looking up purchased claim ids:")
resolved = []
lookup = {claim.claim_id: claim for claim in resolved}
@@ -1045,9 +1041,7 @@ class Ledger(metaclass=LedgerRegistry):
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
try:
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
except Exception:
log.exception("Resolve failed while looking up collection claim ids:")
return []
claims = []
+1 -1
View File
@@ -117,7 +117,7 @@ class ClientSession(BaseClientSession):
)
else:
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
except Exception as err:
except (Exception, asyncio.CancelledError) as err:
if isinstance(err, asyncio.CancelledError):
log.info("closing connection to %s:%i", *self.server)
else:
+1 -2
View File
@@ -214,7 +214,6 @@ class SPVNode:
self.port = 50001 + node_number # avoid conflict with default daemon
self.udp_port = self.port
self.elastic_notifier_port = 19080 + node_number
self.elastic_services = f'localhost:9200/localhost:{self.elastic_notifier_port}'
self.session_timeout = 600
self.stopped = True
self.index_name = uuid4().hex
@@ -236,7 +235,7 @@ class SPVNode:
'host': self.hostname,
'tcp_port': self.port,
'udp_port': self.udp_port,
'elastic_services': self.elastic_services,
'elastic_notifier_port': self.elastic_notifier_port,
'session_timeout': self.session_timeout,
'max_query_workers': 0,
'es_index_prefix': self.index_name,
-2
View File
@@ -182,8 +182,6 @@ class Wallet:
raise InvalidPasswordError()
if "unknown compression method" in e.args[0].lower():
raise InvalidPasswordError()
if "invalid window size" in e.args[0].lower():
raise InvalidPasswordError()
raise
return json.loads(decompressed)
+4 -4
View File
@@ -18,7 +18,7 @@ setup(
long_description_content_type="text/markdown",
keywords="lbry protocol media",
license='MIT',
python_requires='>=3.7',
python_requires='>=3.9',
packages=find_packages(exclude=('tests',)),
zip_safe=False,
entry_points={
@@ -36,11 +36,11 @@ setup(
'distro==1.4.0',
'base58==1.0.0',
'cffi==1.13.2',
'cryptography==2.5',
'cryptography==3.4.7',
'protobuf==3.17.2',
'prometheus_client==0.7.1',
'ecdsa==0.13.3',
'pyyaml==5.3.1',
'pyyaml==5.4',
'docopt==0.6.2',
'hachoir==3.1.2',
'coincurve==15.0.0',
@@ -57,7 +57,7 @@ setup(
'jsonschema==4.4.0',
],
'hub': [
'hub@git+https://github.com/lbryio/hub.git@929448d64bcbe6c5e476757ec78456beaa85e56a'
'hub@git+https://github.com/lbryio/hub.git@024aceda53fe6d1ab8d519b73584437c25de6975'
]
},
classifiers=[
@@ -31,7 +31,7 @@ STREAM_TYPES = {
def verify(channel, data, signature, channel_hash=None):
pieces = [
signature['salt'].encode(),
signature['signing_ts'].encode(),
channel_hash or channel.claim_hash,
data
]
@@ -1239,13 +1239,8 @@ class ChannelCommands(CommandTestCase):
channel = channel_tx.outputs[0]
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
signature4 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
self.assertNotEqual(signature2, signature3)
self.assertEqual(signature3, signature4)
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature3))
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=99))
self.assertTrue(verify(channel, unhexlify('99'), signature3))
@@ -1,18 +1,14 @@
import time
import unittest
from unittest import skipIf
from unittest import skipIf, skip
import asyncio
import os
from binascii import hexlify
import aiohttp.web
from lbry.schema import Claim
from lbry.stream.background_downloader import BackgroundDownloader
from lbry.stream.descriptor import StreamDescriptor
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.components import TorrentSession, BACKGROUND_DOWNLOADER_COMPONENT
from lbry.utils import aiohttp_request
from lbry.wallet import Transaction
from lbry.torrent.tracker import UDPTrackerServerProtocol
@@ -21,104 +17,56 @@ class FileCommands(CommandTestCase):
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self.skip_libtorrent = False
self.streaming_port = 60818
self.seeder_session = None
async def initialize_torrent(self, tx_to_update=None, pick_a_file=True, name=None):
assert name is None or tx_to_update is None
if not self.seeder_session:
async def add_forever(self):
while True:
for handle in self.client_session._handles.values():
handle._handle.connect_peer(('127.0.0.1', 4040))
await asyncio.sleep(.1)
async def initialize_torrent(self, tx_to_update=None):
if not hasattr(self, 'seeder_session'):
self.seeder_session = TorrentSession(self.loop, None)
self.addCleanup(self.seeder_session.stop)
await self.seeder_session.bind('127.0.0.1', port=4040)
btih = await self.seeder_session.add_fake_torrent(file_count=3)
files = [(size, path) for (path, size) in self.seeder_session.get_files(btih).items()]
files.sort()
# picking a file will pick something in the middle, while automatic selection will pick largest
self.expected_size, self.expected_path = files[1] if pick_a_file else files[-1]
btih = await self.seeder_session.add_fake_torrent()
address = await self.account.receiving.get_or_create_usable_address()
claim = tx_to_update.outputs[0].claim if tx_to_update else Claim()
claim.stream.update(bt_infohash=btih)
if pick_a_file:
claim.stream.source.name = os.path.basename(self.expected_path)
if not tx_to_update:
claim = Claim()
claim.stream.update(bt_infohash=btih)
tx = await Transaction.claim_create(
name or 'torrent', claim, 1, address, [self.account], self.account
'torrent', claim, 1, address, [self.account], self.account
)
else:
claim = tx_to_update.outputs[0].claim
claim.stream.update(bt_infohash=btih)
tx = await Transaction.claim_update(
tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
)
await tx.sign([self.account])
await self.broadcast_and_confirm(tx)
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
self.client_session.wait_start = False # fixme: this is super slow on tests
task = asyncio.create_task(self.add_forever())
self.addCleanup(task.cancel)
return tx, btih
async def assert_torrent_streaming_works(self, btih):
url = f'http://{self.daemon.conf.streaming_host}:{self.streaming_port}/stream/{btih}'
if self.daemon.streaming_runner.server is None:
await self.daemon.streaming_runner.setup()
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
self.streaming_port)
await site.start()
async with aiohttp_request('get', url) as req:
self.assertEqual(req.status, 206)
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
expected_size = self.expected_size
self.assertEqual(expected_size, len(streamed_bytes))
self.assertEqual(content_length, len(streamed_bytes))
self.assertEqual(f"bytes 0-{expected_size - 1}/{expected_size}", content_range)
@skipIf(TorrentSession is None, "libtorrent not installed")
#@skipIf(TorrentSession is None, "libtorrent not installed")
@skip
async def test_download_torrent(self):
tx, btih = await self.initialize_torrent(pick_a_file=False)
tx, btih = await self.initialize_torrent()
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
# second call, see its there and move on
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih)
self.assertIn(btih, self.client_session._handles)
# stream over streaming API (full range of the largest file)
await self.assert_torrent_streaming_works(btih)
# check json encoder fields for torrent sources
file = (await self.out(self.daemon.jsonrpc_file_list()))['items'][0]
self.assertEqual(btih, file['metadata']['source']['bt_infohash'])
self.assertAlmostEqual(time.time(), file['added_on'], delta=12)
self.assertEqual("application/octet-stream", file['mime_type'])
self.assertEqual(os.path.basename(self.expected_path), file['suggested_file_name'])
self.assertEqual(os.path.basename(self.expected_path), file['stream_name'])
while not file['completed']: # improve that
await asyncio.sleep(0.5)
file = (await self.out(self.daemon.jsonrpc_file_list()))['items'][0]
self.assertTrue(file['completed'])
self.assertGreater(file['total_bytes_lower_bound'], 0)
self.assertEqual(file['total_bytes_lower_bound'], file['total_bytes'])
self.assertEqual(file['total_bytes'], file['written_bytes'])
self.assertEqual('finished', file['status'])
# filter by a field which is missing on torrent
self.assertItemCount(await self.daemon.jsonrpc_file_list(stream_hash="abc"), 0)
tx, new_btih = await self.initialize_torrent(tx)
self.assertNotEqual(btih, new_btih)
# claim now points to another torrent, update to it
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
# restart and verify that only one updated stream was recovered
self.daemon.file_manager.stop()
await self.daemon.file_manager.start()
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
# check it was saved properly, once
self.assertEqual(1, len(await self.daemon.storage.get_all_torrent_files()))
self.assertIn(new_btih, self.client_session._handles)
self.assertNotIn(btih, self.client_session._handles)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
@@ -126,11 +74,6 @@ class FileCommands(CommandTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
self.assertNotIn(new_btih, self.client_session._handles)
await self.initialize_torrent(name='torrent2')
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent2')))
file = (await self.out(self.daemon.jsonrpc_file_list()))['items'][0]
self.assertEqual(os.path.basename(self.expected_path), file['stream_name'])
async def create_streams_in_range(self, *args, **kwargs):
self.stream_claim_ids = []
for i in range(*args, **kwargs):
@@ -147,21 +90,6 @@ class FileCommands(CommandTestCase):
await self.reflector.blob_manager.delete_blobs(all_except_sd)
self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
async def test_sd_blob_fields_fallback(self):
claim_id = self.get_claim_id(await self.stream_create('foo', '0.01', suffix='.txt'))
stream = (await self.daemon.jsonrpc_file_list())["items"][0]
stream.descriptor.suggested_file_name = ' '
stream.descriptor.stream_name = ' '
stream.descriptor.stream_hash = stream.descriptor.get_stream_hash()
sd_hash = stream.descriptor.sd_hash = stream.descriptor.calculate_sd_hash()
await stream.descriptor.make_sd_blob()
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.stream_update(claim_id=claim_id, sd_hash=sd_hash)
file_dict = await self.out(self.daemon.jsonrpc_get('lbry://foo', save_file=True))
self.assertEqual(file_dict['suggested_file_name'], stream.file_name)
self.assertEqual(file_dict['stream_name'], stream.file_name)
self.assertEqual(file_dict['mime_type'], 'text/plain')
async def test_file_management(self):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')
@@ -393,12 +321,12 @@ class FileCommands(CommandTestCase):
await self.server.blob_manager.delete_blobs(all_except_sd)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEqual('Failed to download data blobs for %s within timeout.' % sd_hash, resp['error'])
self.assertEqual('Failed to download data blobs for sd hash %s within timeout.' % sd_hash, resp['error'])
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didn't create a file")
await self.server.blob_manager.delete_blobs([sd_hash])
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEqual('Failed to download metadata for %s within timeout.' % sd_hash, resp['error'])
self.assertEqual('Failed to download sd blob %s within timeout.' % sd_hash, resp['error'])
async def wait_files_to_complete(self):
while await self.file_list(status='running'):
@@ -1508,27 +1508,27 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
COIN = int(1E8)
self.assertEqual(self.conductor.spv_node.writer.height, 207)
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
(208, bytes.fromhex(claim_id1)), (0, 10 * COIN)
)
await self.generate(1)
self.assertEqual(self.conductor.spv_node.writer.height, 208)
self.assertEqual(1.7090807854206793, await get_trending_score(claim_id1))
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
(209, bytes.fromhex(claim_id1)), (10 * COIN, 100 * COIN)
)
await self.generate(1)
self.assertEqual(self.conductor.spv_node.writer.height, 209)
self.assertEqual(2.2437974397778886, await get_trending_score(claim_id1))
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
(309, bytes.fromhex(claim_id1)), (100 * COIN, 1000000 * COIN)
)
await self.generate(100)
self.assertEqual(self.conductor.spv_node.writer.height, 309)
self.assertEqual(5.157053472135866, await get_trending_score(claim_id1))
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
(409, bytes.fromhex(claim_id1)), (1000000 * COIN, 1 * COIN)
)
@@ -2,7 +2,7 @@ import asyncio
import unittest
from lbry.testcase import CommandTestCase
from lbry.wallet import Transaction
class TransactionCommandsTestCase(CommandTestCase):
@@ -29,42 +29,17 @@ class TransactionCommandsTestCase(CommandTestCase):
# someone's tx
change_address = await self.blockchain.get_raw_change_address()
sendtxid = await self.blockchain.send_to_address(change_address, 10)
# After a few tries, Hub should have the transaction (in mempool).
for i in range(5):
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
# Retry if Hub is not aware of the transaction.
if isinstance(tx, dict):
# Fields: 'success', 'code', 'message'
self.assertFalse(tx['success'], tx)
self.assertEqual(tx['code'], 404, tx)
self.assertEqual(tx['message'], "transaction not found", tx)
await asyncio.sleep(0.1)
continue
break
# verify transaction show (in mempool)
self.assertTrue(isinstance(tx, Transaction), str(tx))
# Fields: 'txid', 'raw', 'height', 'position', 'is_verified', and more.
self.assertEqual(tx.id, sendtxid, vars(tx))
self.assertEqual(tx.height, -1, vars(tx))
self.assertEqual(tx.is_verified, False, vars(tx))
# transaction is confirmed and leaves mempool
await self.generate(1)
# verify transaction show
await asyncio.sleep(0.2)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertTrue(isinstance(tx, Transaction), str(tx))
self.assertEqual(tx.id, sendtxid, vars(tx))
self.assertEqual(tx.height, self.ledger.headers.height, vars(tx))
self.assertEqual(tx.is_verified, True, vars(tx))
self.assertEqual(tx.id, sendtxid)
self.assertEqual(tx.height, -1)
await self.generate(1)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.height, self.ledger.headers.height)
# inexistent
result = await self.daemon.jsonrpc_transaction_show('0'*64)
self.assertTrue(isinstance(result, dict), result)
# Fields: 'success', 'code', 'message'
self.assertFalse(result['success'], result)
self.assertEqual(result['code'], 404, result)
self.assertEqual(result['message'], "transaction not found", result)
self.assertFalse(result['success'])
async def test_utxo_release(self):
await self.send_to_address_and_wait(
+9 -10
View File
@@ -11,7 +11,7 @@ from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
from lbry.testcase import get_fake_exchange_rate_manager
from lbry.utils import generate_id
from lbry.error import InsufficientFundsError
from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadMetadataTimeoutError, DownloadDataTimeoutError
from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadSDTimeoutError, DownloadDataTimeoutError
from lbry.wallet import WalletManager, Wallet, Ledger, Transaction, Input, Output, Database
from lbry.wallet.constants import CENT, NULL_HASH32
from lbry.wallet.network import ClientSession
@@ -229,10 +229,10 @@ class TestStreamManager(BlobExchangeTestBase):
self.assertFalse(event['properties']['added_fixed_peers'])
self.assertEqual(event['properties']['connection_failures_count'], 1)
self.assertEqual(
event['properties']['error_message'], f'Failed to download metadata for {self.sd_hash} within timeout.'
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.'
)
await self._test_time_to_first_bytes(check_post, DownloadMetadataTimeoutError, after_setup=after_setup)
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup)
async def test_override_fixed_peer_delay_dht_disabled(self):
self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)]
@@ -266,18 +266,18 @@ class TestStreamManager(BlobExchangeTestBase):
def check_post(event):
self.assertEqual(event['event'], 'Time To First Bytes')
self.assertEqual(event['properties']['error'], 'DownloadMetadataTimeoutError')
self.assertEqual(event['properties']['error'], 'DownloadSDTimeoutError')
self.assertEqual(event['properties']['tried_peers_count'], 0)
self.assertEqual(event['properties']['active_peer_count'], 0)
self.assertFalse(event['properties']['use_fixed_peers'])
self.assertFalse(event['properties']['added_fixed_peers'])
self.assertIsNone(event['properties']['fixed_peer_delay'])
self.assertEqual(
event['properties']['error_message'], f'Failed to download metadata for {self.sd_hash} within timeout.'
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.'
)
start = self.loop.time()
await self._test_time_to_first_bytes(check_post, DownloadMetadataTimeoutError)
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError)
duration = self.loop.time() - start
self.assertLessEqual(duration, 5)
self.assertGreaterEqual(duration, 3.0)
@@ -305,6 +305,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.assertEqual(stored_status, "running")
await stream.stop()
await asyncio.sleep(1) # TODO: should not be needed
self.assertFalse(stream.finished)
self.assertFalse(stream.running)
@@ -340,8 +341,6 @@ class TestStreamManager(BlobExchangeTestBase):
try:
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
error = err
self.assertEqual(expected_error, type(error))
@@ -387,7 +386,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.server.stop_server()
await self.setup_stream_manager()
await self._test_download_error_analytics_on_start(
DownloadMetadataTimeoutError, f'Failed to download metadata for {self.sd_hash} within timeout.', timeout=1
DownloadSDTimeoutError, f'Failed to download sd blob {self.sd_hash} within timeout.', timeout=1
)
async def test_download_data_timeout(self):
@@ -396,7 +395,7 @@ class TestStreamManager(BlobExchangeTestBase):
head_blob_hash = json.loads(sdf.read())['blobs'][0]['blob_hash']
self.server_blob_manager.delete_blob(head_blob_hash)
await self._test_download_error_analytics_on_start(
DownloadDataTimeoutError, f'Failed to download data blobs for {self.sd_hash} within timeout.', timeout=1
DownloadDataTimeoutError, f'Failed to download data blobs for sd hash {self.sd_hash} within timeout.', timeout=1
)
async def test_unexpected_error(self):
+1 -1
View File
@@ -470,7 +470,7 @@ class TestUpgrade(AsyncioTestCase):
class TestSQLiteRace(AsyncioTestCase):
max_misuse_attempts = 120000
max_misuse_attempts = 80000
def setup_db(self):
self.db = sqlite3.connect(":memory:", isolation_level=None)