Compare commits

...
Author SHA1 Message Date
Jack Robison b593ac96b2 Bump version 0.16.3rc1 --> 0.16.3rc2
Signed-off-by: Jack Robison <jackrobison@lbry.io>
2017-09-28 16:37:16 -04:00
Jack Robison 5d94f76a1f changelog 2017-09-28 16:36:49 -04:00
Jack Robison da6867203f move download_failed 2017-09-28 16:33:54 -04:00
Jack Robison 3289f5f846 fix error name 2017-09-28 16:33:53 -04:00
Jack Robison b2b121a574 fix download exceptions 2017-09-28 16:33:53 -04:00
Jack Robison 42697bfbff add _download_failed errback 2017-09-28 16:33:53 -04:00
Jack Robison f83e7b3e73 json blobs 2017-09-28 16:33:52 -04:00
Jack Robison a579caaa0d cancel streams on shutdown 2017-09-28 16:33:52 -04:00
Kay Kurokawa 191d83df93 fix test_timeout in test_Downloader 2017-09-28 16:33:52 -04:00
Jack Robison 2b6ce488da changelog 2017-09-28 16:33:52 -04:00
Jack Robison 2fee69ce7c update tests and scripts 2017-09-28 16:33:51 -04:00
Jack Robison b90facab09 fix stop condition 2017-09-28 16:33:51 -04:00
Jack Robison 8d3f1c94cd download analytics 2017-09-28 16:33:51 -04:00
Jack Robison d47b43bff5 add get_host_downloaded_from 2017-09-28 16:33:51 -04:00
Jack Robison 9bf5f6d12a fix blob history and callback from _download_succeeded
previously _download_succeeded did not block on blob_completed,
presumably because even longer ago it did not block on a deriving
immediate_announce call and thus took a long time to return
2017-09-28 16:33:50 -04:00
Jack Robison 29e451a326 stop failed downloader outside of GetStream 2017-09-28 16:33:50 -04:00
Jack Robison 334611ab74 better download errors 2017-09-28 16:33:49 -04:00
Jack Robison 9e82f84da1 Bump version 0.16.1 --> 0.16.2
Signed-off-by: Jack Robison <jackrobison@lbry.io>
2017-09-26 17:32:37 -04:00
Jack Robison 68cfa7d8fe changelog 2017-09-26 17:30:01 -04:00
Jack Robison 223946d35f update requirements 2017-09-26 17:18:31 -04:00
15 changed files with 195 additions and 88 deletions

No files matched your search

+9 -4
View File
@@ -13,16 +13,16 @@ at anytime.
*
### Fixed
*
*
* Fixed redundant blob requests to a peer
* Fixed blob download history
### Deprecated
*
*
### Changed
*
*
* Improved download analytics
* Improved download errors by distinguishing a data timeout from a sd timeout
### Added
*
@@ -33,6 +33,11 @@ at anytime.
*
## [0.16.2] - 2017-09-26
### Fixed
* Fixed https://github.com/lbryio/lbry/issues/771 (handle when a certificate is missing for a signed claim in `claim_list_mine`)
## [0.16.1] - 2017-09-20
### Fixed
* Fixed `transaction_list` doc string
+1 -1
View File
@@ -1,6 +1,6 @@
import logging
__version__ = "0.16.1"
__version__ = "0.16.3rc2"
version = tuple(__version__.split('.'))
logging.getLogger(__name__).addHandler(logging.NullHandler())
+28 -16
View File
@@ -57,15 +57,14 @@ class Manager(object):
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_ERRORED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
def send_download_finished(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_FINISHED, self._download_properties(id_, name, claim_dict))
)
def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
def send_claim_action(self, action):
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
@@ -159,18 +158,31 @@ class Manager(object):
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None):
sd_hash = None
if claim_dict:
try:
sd_hash = claim_dict.source_hash
except (KeyError, TypeError, ValueError):
log.debug('Failed to get sd_hash from %s', claim_dict, exc_info=True)
return {
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash,
'error': error_name(error),
'reason': error.message,
'report': report
}
@staticmethod
def _make_context(platform, wallet):
+10
View File
@@ -112,6 +112,16 @@ class DiskBlobManager(DHTHashSupplier):
d = self._add_blob_to_download_history(blob_hash, host, rate)
return d
@defer.inlineCallbacks
def get_host_downloaded_from(self, blob_hash):
query_str = "SELECT host FROM download WHERE blob=? ORDER BY ts DESC LIMIT 1"
host = yield self.db_conn.runQuery(query_str, (blob_hash,))
if host:
result = host[0][0]
else:
result = None
defer.returnValue(result)
def add_blob_to_upload_history(self, blob_hash, host, rate):
d = self._add_blob_to_upload_history(blob_hash, host, rate)
return d
+15
View File
@@ -9,11 +9,26 @@ class DuplicateStreamHashError(Exception):
class DownloadCanceledError(Exception):
pass
class DownloadSDTimeout(Exception):
def __init__(self, download):
Exception.__init__(self, 'Failed to download sd blob {} within timeout'.format(download))
self.download = download
class DownloadTimeoutError(Exception):
def __init__(self, download):
Exception.__init__(self, 'Failed to download {} within timeout'.format(download))
self.download = download
class DownloadDataTimeout(Exception):
def __init__(self, download):
Exception.__init__(self, 'Failed to download data blobs for sd hash '
'{} within timeout'.format(download))
self.download = download
class RequestCanceledError(Exception):
pass
+5 -4
View File
@@ -514,8 +514,6 @@ class DownloadRequest(RequestHelper):
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
if self._can_pay_peer(blob, arg):
self._pay_peer(blob.length, reserved_points)
d = self.requestor.blob_manager.add_blob_to_download_history(
str(blob), str(self.peer.host), float(self.protocol_prices[self.protocol]))
else:
self._cancel_points(reserved_points)
return arg
@@ -565,8 +563,11 @@ class DownloadRequest(RequestHelper):
self.peer.update_stats('blobs_downloaded', 1)
self.peer.update_score(5.0)
should_announce = blob.blob_hash == self.head_blob_hash
self.requestor.blob_manager.blob_completed(blob, should_announce=should_announce)
return arg
d = self.requestor.blob_manager.blob_completed(blob, should_announce=should_announce)
d.addCallback(lambda _: self.requestor.blob_manager.add_blob_to_download_history(
blob.blob_hash, self.peer.host, self.protocol_prices[self.protocol]))
d.addCallback(lambda _: arg)
return d
def _download_failed(self, reason):
if not reason.check(DownloadCanceledError, PriceDisagreementError):
@@ -5,7 +5,7 @@ from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.Error import InvalidBlobHashError, DownloadTimeoutError
from lbrynet.core.Error import InvalidBlobHashError, DownloadSDTimeout
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
from twisted.python.failure import Failure
from twisted.internet import defer
@@ -136,7 +136,7 @@ class StandaloneBlobDownloader(object):
def _download_timedout(self):
self.stop()
if not self.finished_deferred.called:
self.finished_deferred.errback(DownloadTimeoutError(self.blob_hash))
self.finished_deferred.errback(DownloadSDTimeout(self.blob_hash))
def insufficient_funds(self, err):
self.stop()
+1 -1
View File
@@ -162,7 +162,7 @@ class BlobRequestHandler(object):
def record_transaction(self, blob):
d = self.blob_manager.add_blob_to_upload_history(
str(blob), self.peer.host, self.blob_data_payment_rate)
blob.blob_hash, self.peer.host, self.blob_data_payment_rate)
return d
def _reply_to_send_request(self, response, incoming):
+67 -9
View File
@@ -47,7 +47,7 @@ from lbrynet.core.looping_call_manager import LoopingCallManager
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
from lbrynet.core.Error import InsufficientFundsError, UnknownNameError, NoSuchSDHash
from lbrynet.core.Error import NoSuchStreamHash
from lbrynet.core.Error import NoSuchStreamHash, DownloadDataTimeout, DownloadSDTimeout
from lbrynet.core.Error import NullFundsError, NegativeFundsError
log = logging.getLogger(__name__)
@@ -392,6 +392,11 @@ class Daemon(AuthJSONRPCServer):
def _already_shutting_down(sig_num, frame):
log.info("Already shutting down")
def _stop_streams(self):
"""stop pending GetStream downloads"""
for claim_id, stream in self.streams.iteritems():
stream.cancel(reason="daemon shutdown")
def _shutdown(self):
# ignore INT/TERM signals once shutdown has started
signal.signal(signal.SIGINT, self._already_shutting_down)
@@ -399,6 +404,9 @@ class Daemon(AuthJSONRPCServer):
log.info("Closing lbrynet session")
log.info("Status at time of shutdown: " + self.startup_status[0])
self._stop_streams()
self.looping_call_manager.shutdown()
if self.analytics_manager:
self.analytics_manager.shutdown()
@@ -608,6 +616,39 @@ class Daemon(AuthJSONRPCServer):
timeout = timeout or 30
return download_sd_blob(self.session, blob_hash, rate_manager, timeout)
@defer.inlineCallbacks
def _get_stream_analytics_report(self, claim_dict):
sd_hash = claim_dict.source_hash
try:
stream_hash = yield self.stream_info_manager.get_stream_hash_for_sd_hash(sd_hash)
except Exception:
stream_hash = None
report = {
"sd_hash": sd_hash,
"stream_hash": stream_hash,
}
blobs = {}
try:
sd_host = yield self.session.blob_manager.get_host_downloaded_from(sd_hash)
except Exception:
sd_host = None
report["sd_blob"] = sd_host
if stream_hash:
blob_infos = yield self.stream_info_manager.get_blobs_for_stream(stream_hash)
report["known_blobs"] = len(blob_infos)
else:
blob_infos = []
report["known_blobs"] = 0
for blob_hash, blob_num, iv, length in blob_infos:
try:
host = yield self.session.blob_manager.get_host_downloaded_from(blob_hash)
except Exception:
host = None
if host:
blobs[blob_num] = host
report["blobs"] = json.dumps(blobs)
defer.returnValue(report)
@defer.inlineCallbacks
def _download_name(self, name, claim_dict, claim_id, timeout=None, file_name=None):
"""
@@ -615,6 +656,17 @@ class Daemon(AuthJSONRPCServer):
If it already exists in the file manager, return the existing lbry file
"""
@defer.inlineCallbacks
def _download_finished(download_id, name, claim_dict):
report = yield self._get_stream_analytics_report(claim_dict)
self.analytics_manager.send_download_finished(download_id, name, report, claim_dict)
@defer.inlineCallbacks
def _download_failed(error, download_id, name, claim_dict):
report = yield self._get_stream_analytics_report(claim_dict)
self.analytics_manager.send_download_errored(error, download_id, name, claim_dict,
report)
if claim_id in self.streams:
downloader = self.streams[claim_id]
result = yield downloader.finished_deferred
@@ -630,17 +682,23 @@ class Daemon(AuthJSONRPCServer):
file_name)
try:
lbry_file, finished_deferred = yield self.streams[claim_id].start(claim_dict, name)
finished_deferred.addCallback(
lambda _: self.analytics_manager.send_download_finished(download_id,
name,
claim_dict))
finished_deferred.addCallbacks(lambda _: _download_finished(download_id, name,
claim_dict),
lambda e: _download_failed(e, download_id, name,
claim_dict))
result = yield self._get_lbry_file_dict(lbry_file, full_status=True)
del self.streams[claim_id]
except Exception as err:
log.warning('Failed to get %s: %s', name, err)
self.analytics_manager.send_download_errored(download_id, name, claim_dict)
del self.streams[claim_id]
yield _download_failed(err, download_id, name, claim_dict)
if isinstance(err, (DownloadDataTimeout, DownloadSDTimeout)):
log.warning('Failed to get %s (%s)', name, err)
else:
log.error('Failed to get %s (%s)', name, err)
if self.streams[claim_id].downloader:
yield self.streams[claim_id].downloader.stop(err)
result = {'error': err.message}
finally:
del self.streams[claim_id]
defer.returnValue(result)
@defer.inlineCallbacks
+24 -12
View File
@@ -5,11 +5,11 @@ from twisted.internet.task import LoopingCall
from lbryschema.fee import Fee
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, DownloadTimeoutError
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
from lbrynet.core.Error import DownloadDataTimeout, DownloadCanceledError
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet import conf
INITIALIZING_CODE = 'initializing'
@@ -61,23 +61,22 @@ class GetStream(object):
return os.path.join(self.download_directory, self.downloader.file_name)
def _check_status(self, status):
stop_condition = (status.num_completed > 0 or
status.running_status == ManagedEncryptedFileDownloader.STATUS_STOPPED)
if stop_condition and not self.data_downloading_deferred.called:
if status.num_completed > 0 and not self.data_downloading_deferred.called:
self.data_downloading_deferred.callback(True)
if self.data_downloading_deferred.called:
safe_stop_looping_call(self.checker)
else:
log.info("Downloading stream data (%i seconds)", self.timeout_counter)
log.info("Waiting for stream data (%i seconds)", self.timeout_counter)
def check_status(self):
"""
Check if we've got the first data blob in the stream yet
"""
self.timeout_counter += 1
if self.timeout_counter >= self.timeout:
if self.timeout_counter > self.timeout:
if not self.data_downloading_deferred.called:
self.data_downloading_deferred.errback(DownloadTimeoutError(self.file_name))
self.data_downloading_deferred.errback(DownloadDataTimeout(self.sd_hash))
safe_stop_looping_call(self.checker)
else:
d = self.downloader.status()
@@ -150,6 +149,10 @@ class GetStream(object):
self._check_status(status)
defer.returnValue(self.download_path)
def fail(self, err):
safe_stop_looping_call(self.checker)
raise err
@defer.inlineCallbacks
def _initialize(self, stream_info):
# Set sd_hash and return key_fee from stream_info
@@ -179,7 +182,7 @@ class GetStream(object):
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.finished_deferred = self.downloader.start()
self.finished_deferred.addCallback(self.finish, name)
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
@defer.inlineCallbacks
def start(self, stream_info, name):
@@ -204,9 +207,18 @@ class GetStream(object):
try:
yield self.data_downloading_deferred
except Exception as err:
self.downloader.stop()
except DownloadDataTimeout as err:
safe_stop_looping_call(self.checker)
raise
raise err
defer.returnValue((self.downloader, self.finished_deferred))
def cancel(self, reason=None):
if reason:
msg = "download stream cancelled: %s" % reason
else:
msg = "download stream cancelled"
if self.finished_deferred and not self.finished_deferred.called:
self.finished_deferred.errback(DownloadCanceledError(msg))
if self.data_downloading_deferred and not self.data_downloading_deferred.called:
self.data_downloading_deferred.errback(DownloadCanceledError(msg))
+2 -2
View File
@@ -13,8 +13,8 @@ gmpy==1.17
jsonrpc==1.2
jsonrpclib==0.1.7
jsonschema==2.5.1
git+https://github.com/lbryio/lbryschema.git@v0.0.11#egg=lbryschema
git+https://github.com/lbryio/lbryum.git@v3.1.8#egg=lbryum
git+https://github.com/lbryio/lbryschema.git@v0.0.12rc1#egg=lbryschema
git+https://github.com/lbryio/lbryum.git@v3.1.9rc2#egg=lbryum
miniupnpc==1.9
pbkdf2==1.3
pycrypto==2.6.1
+1 -1
View File
@@ -59,7 +59,7 @@ def main(args=None):
use_upnp=False,
wallet=wallet
)
api = analytics.Api.new_instance()
api = analytics.Api.new_instance(conf.settings['share_usage_data'])
run(args, session, api)
reactor.run()
finally:
+2 -2
View File
@@ -21,8 +21,8 @@ requires = [
'envparse',
'jsonrpc',
'jsonschema',
'lbryum==3.1.8',
'lbryschema==0.0.11',
'lbryum==3.1.9rc2',
'lbryschema==0.0.12rc1',
'miniupnpc',
'pycrypto',
'pyyaml',
+2 -2
View File
@@ -50,10 +50,10 @@ class BlobManagerTest(unittest.TestCase):
finished_d, write, cancel =yield blob.open_for_writing(self.peer)
yield write(data)
yield self.bm.blob_completed(blob)
yield self.bm.add_blob_to_upload_history(blob_hash,'test',len(data))
yield self.bm.add_blob_to_upload_history(blob_hash, 'test', len(data))
# check to see if blob is there
self.assertTrue(os.path.isfile(os.path.join(self.blob_dir,blob_hash)))
self.assertTrue(os.path.isfile(os.path.join(self.blob_dir, blob_hash)))
blobs = yield self.bm.get_all_verified_blobs()
self.assertTrue(blob_hash in blobs)
defer.returnValue(blob_hash)
+26 -32
View File
@@ -1,26 +1,22 @@
import types
import mock
import json
from twisted.trial import unittest
from twisted.internet import defer, task
from lbryschema.claim import ClaimDict
from lbrynet.core import Session, PaymentRateManager, Wallet
from lbrynet.core.Error import DownloadTimeoutError
from lbrynet.core.Error import DownloadDataTimeout, DownloadSDTimeout
from lbrynet.daemon import Downloader
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier,StreamMetadata
from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader, ManagedEncryptedFileDownloaderFactory
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker
from tests.mocks import ExchangeRateManager as DummyExchangeRateManager
from tests.mocks import BTCLBCFeed, USDBTCFeed
from tests.mocks import mock_conf_settings
class MocDownloader(object):
def __init__(self):
self.finish_deferred = defer.Deferred(None)
@@ -106,7 +102,7 @@ class GetStreamTests(unittest.TestCase):
DownloadTimeoutError is raised
"""
def download_sd_blob(self):
raise DownloadTimeoutError(self.file_name)
raise DownloadSDTimeout(self.file_name)
getstream = self.init_getstream_with_mocs()
getstream._initialize = types.MethodType(moc_initialize, getstream)
@@ -115,15 +111,14 @@ class GetStreamTests(unittest.TestCase):
getstream.pay_key_fee = types.MethodType(moc_pay_key_fee, getstream)
name='test'
stream_info = None
with self.assertRaises(DownloadTimeoutError):
with self.assertRaises(DownloadSDTimeout):
yield getstream.start(stream_info,name)
self.assertFalse(getstream.pay_key_fee_called)
@defer.inlineCallbacks
def test_timeout(self):
"""
test that timeout (set to 2 here) exception is raised
test that timeout (set to 3 here) exception is raised
when download times out while downloading first blob, and key fee is paid
"""
getstream = self.init_getstream_with_mocs()
@@ -136,9 +131,9 @@ class GetStreamTests(unittest.TestCase):
start = getstream.start(stream_info,name)
self.clock.advance(1)
self.clock.advance(1)
with self.assertRaises(DownloadTimeoutError):
self.clock.advance(1)
with self.assertRaises(DownloadDataTimeout):
yield start
self.assertTrue(getstream.downloader.stop_called)
self.assertTrue(getstream.pay_key_fee_called)
@defer.inlineCallbacks
@@ -163,21 +158,20 @@ class GetStreamTests(unittest.TestCase):
downloader, f_deferred = yield start
self.assertTrue(getstream.pay_key_fee_called)
@defer.inlineCallbacks
def test_finish_stopped_downloader(self):
"""
test that if we have a stopped downloader, beforfe a blob is downloaded,
start() returns
"""
getstream = self.init_getstream_with_mocs()
getstream._initialize = types.MethodType(moc_initialize, getstream)
getstream._download_sd_blob = types.MethodType(moc_download_sd_blob, getstream)
getstream._download = types.MethodType(moc_download, getstream)
name='test'
stream_info = None
start = getstream.start(stream_info,name)
getstream.downloader.running_status = ManagedEncryptedFileDownloader.STATUS_STOPPED
self.clock.advance(1)
downloader, f_deferred = yield start
# @defer.inlineCallbacks
# def test_finish_stopped_downloader(self):
# """
# test that if we have a stopped downloader, beforfe a blob is downloaded,
# start() returns
# """
# getstream = self.init_getstream_with_mocs()
# getstream._initialize = types.MethodType(moc_initialize, getstream)
# getstream._download_sd_blob = types.MethodType(moc_download_sd_blob, getstream)
# getstream._download = types.MethodType(moc_download, getstream)
# name='test'
# stream_info = None
# start = getstream.start(stream_info,name)
#
# getstream.downloader.running_status = ManagedEncryptedFileDownloader.STATUS_STOPPED
# self.clock.advance(1)
# downloader, f_deferred = yield start