Compare commits

..
5 Commits
Author SHA1 Message Date
Lex Berezhny 8c6c7b655c v0.112.0
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (claims) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
ci / tests / integration (takeovers) (push) Waiting to run
ci / tests / integration (transactions) (push) Waiting to run
2022-10-30 21:56:17 -04:00
Lex Berezhny 48c6873fc4 channel_sign command has customizeable salt 2022-10-30 21:53:53 -04:00
Victor Shyba 15dc52bd9a Merge pull request #3695 from lbryio/3690
Fix claim fields fallback raising errors before download is saved on database
2022-10-28 11:16:52 -03:00
Victor Shyba 52d555078f initialize stored claim field for fallback earlier 2022-10-19 15:13:47 -03:00
Victor Shyba cc976bd010 test for early fallback of suggested_file_name 2022-10-19 15:13:47 -03:00
18 changed files with 82 additions and 36 deletions

No files matched your search

+6 -6
View File
@@ -10,7 +10,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.9'
python-version: '3.7'
- name: extract pip cache
uses: actions/cache@v2
with:
@@ -34,7 +34,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.9'
python-version: '3.7'
- name: set pip cache dir
id: pip-cache
run: echo "::set-output name=dir::$(pip cache dir)"
@@ -96,7 +96,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.9'
python-version: '3.7'
- if: matrix.test == 'other'
run: |
sudo apt-get update
@@ -138,7 +138,7 @@ jobs:
strategy:
matrix:
os:
- ubuntu-20.04
- ubuntu-18.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
@@ -146,7 +146,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.9'
python-version: '3.7'
- id: os-name
uses: ASzc/change-string-case-action@v1
with:
@@ -160,7 +160,7 @@ jobs:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install pyinstaller==5.3
- run: pip install pyinstaller==4.4
- run: pip install -e .
- if: startsWith(github.ref, 'refs/tags/v')
run: python docker/set_build.py
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.111.0"
__version__ = "0.112.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+2
View File
@@ -42,6 +42,8 @@ class BlobAnnouncer:
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
self.announcements_sent_metric.labels(peers=0, error=True).inc()
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
async def _announce(self, batch_size: typing.Optional[int] = 10):
+2
View File
@@ -560,6 +560,8 @@ class UPnPComponent(Component):
self.upnp = await UPnP.discover(loop=self.component_manager.loop)
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.warning("upnp discovery failed: %s", err)
self.upnp = None
+10 -6
View File
@@ -2943,19 +2943,21 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_sign(
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
self, channel_name=None, channel_id=None, hexdata=None, salt=None,
channel_account_id=None, wallet_id=None):
"""
Signs data using the specified channel signing key.
Usage:
channel_sign [<channel_name> | --channel_name=<channel_name>]
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
channel_sign [<channel_name> | --channel_name=<channel_name>] [<channel_id> | --channel_id=<channel_id>]
[<hexdata> | --hexdata=<hexdata>] [<salt> | --salt=<salt>]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
--salt=<salt> : (str) salt to use for signing, default is to use timestamp
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
@@ -2972,11 +2974,13 @@ class Daemon(metaclass=JSONRPCServerType):
signing_channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
timestamp = str(int(time.time()))
signature = signing_channel.sign_data(unhexlify(str(hexdata)), timestamp)
if salt is None:
salt = str(int(time.time()))
signature = signing_channel.sign_data(unhexlify(str(hexdata)), salt)
return {
'signature': signature,
'signing_ts': timestamp
'signing_ts': salt, # DEPRECATED
'salt': salt,
}
@requires(WALLET_COMPONENT)
@@ -80,6 +80,8 @@ class MarketFeed:
self.rate = ExchangeRate(self.market, rate, int(time.time()))
self.last_check = time.time()
return self.rate
except asyncio.CancelledError:
raise
except asyncio.TimeoutError:
log.warning("Timed out fetching exchange rate from %s.", self.name)
except json.JSONDecodeError as e:
+11 -5
View File
@@ -13,11 +13,12 @@ from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
from lbry.extras.daemon.storage import StoredContentClaim
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager, Output
from lbry.wallet import WalletManager
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
@@ -98,6 +99,8 @@ class FileManager:
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
if 'error' in resolved_result:
@@ -192,21 +195,24 @@ class FileManager:
####################
# make downloader and wait for start
####################
# temporary with fields we know so downloader can start. Missing fields are populated later.
stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
amount=txo.amount, height=txo.tx_ref.height,
serialized=claim.to_bytes().hex())
if not claim.stream.source.bt_infohash:
# fixme: this shouldnt be here
stream = ManagedStream(
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager
analytics_manager=self.analytics_manager, claim=stored_claim
)
stream.downloader.node = source_manager.node
else:
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
file_name=file_name, download_directory=download_directory or self.config.download_dir,
status=ManagedStream.STATUS_RUNNING,
analytics_manager=self.analytics_manager,
status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
torrent_session=source_manager.torrent_session
)
log.info("starting download for %s", uri)
@@ -243,7 +249,7 @@ class FileManager:
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except (Exception, asyncio.CancelledError) as err: # forgive data timeout, don't delete stream
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
if isinstance(err, expected):
+2
View File
@@ -22,6 +22,8 @@ class BackgroundDownloader:
await downloader.download_stream_blob(blob_info)
except ValueError:
return
except asyncio.CancelledError:
raise
except Exception:
log.error("Unexpected download error on background downloader")
finally:
+2 -2
View File
@@ -279,7 +279,7 @@ class ManagedStream(ManagedDownloadSource):
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
self.sd_hash[:6], self.full_path)
await self.blob_manager.storage.set_saved_file(self.stream_hash)
except (Exception, asyncio.CancelledError) as err:
except Exception as err:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
@@ -366,7 +366,7 @@ class ManagedStream(ManagedDownloadSource):
return sent
except ConnectionError:
return sent
except (OSError, Exception, asyncio.CancelledError) as err:
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
elif isinstance(err, OSError):
+2
View File
@@ -164,6 +164,8 @@ class StreamManager(SourceManager):
async def reflect_streams(self):
try:
return await self._reflect_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
+3 -3
View File
@@ -22,9 +22,9 @@ class TorrentHandle:
self._loop = loop
self._executor = executor
self._handle: libtorrent.torrent_handle = handle
self.started = asyncio.Event()
self.finished = asyncio.Event()
self.metadata_completed = asyncio.Event()
self.started = asyncio.Event(loop=loop)
self.finished = asyncio.Event(loop=loop)
self.metadata_completed = asyncio.Event(loop=loop)
self.size = 0
self.total_wanted_done = 0
self.name = ''
+1 -1
View File
@@ -141,7 +141,7 @@ class CoinSelector:
_) -> List[OutputEffectiveAmountEstimator]:
""" Accumulate UTXOs at random until there is enough to cover the target. """
target = self.target + self.cost_of_change
self.random.shuffle(txos, random=self.random.random) # pylint: disable=deprecated-argument
self.random.shuffle(txos, self.random.random)
selection = []
amount = 0
for coin in txos:
+9 -3
View File
@@ -938,7 +938,9 @@ class Ledger(metaclass=LedgerRegistry):
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change,
account.change.gap, channel_count, len(account.channel_keys), claim_count)
except Exception:
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception(
'Failed to display wallet state, please file issue '
'for this bug along with the traceback you see below:')
@@ -961,7 +963,9 @@ class Ledger(metaclass=LedgerRegistry):
claim_ids = [p.purchased_claim_id for p in purchases]
try:
resolved, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception:
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception("Resolve failed while looking up purchased claim ids:")
resolved = []
lookup = {claim.claim_id: claim for claim in resolved}
@@ -1041,7 +1045,9 @@ class Ledger(metaclass=LedgerRegistry):
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
try:
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception:
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception("Resolve failed while looking up collection claim ids:")
return []
claims = []
+1 -1
View File
@@ -117,7 +117,7 @@ class ClientSession(BaseClientSession):
)
else:
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
except (Exception, asyncio.CancelledError) as err:
except Exception as err:
if isinstance(err, asyncio.CancelledError):
log.info("closing connection to %s:%i", *self.server)
else:
+3 -3
View File
@@ -18,7 +18,7 @@ setup(
long_description_content_type="text/markdown",
keywords="lbry protocol media",
license='MIT',
python_requires='>=3.9',
python_requires='>=3.7',
packages=find_packages(exclude=('tests',)),
zip_safe=False,
entry_points={
@@ -36,11 +36,11 @@ setup(
'distro==1.4.0',
'base58==1.0.0',
'cffi==1.13.2',
'cryptography==3.4.7',
'cryptography==2.5',
'protobuf==3.17.2',
'prometheus_client==0.7.1',
'ecdsa==0.13.3',
'pyyaml==5.4',
'pyyaml==5.3.1',
'docopt==0.6.2',
'hachoir==3.1.2',
'coincurve==15.0.0',
@@ -31,7 +31,7 @@ STREAM_TYPES = {
def verify(channel, data, signature, channel_hash=None):
pieces = [
signature['signing_ts'].encode(),
signature['salt'].encode(),
channel_hash or channel.claim_hash,
data
]
@@ -1239,8 +1239,13 @@ class ChannelCommands(CommandTestCase):
channel = channel_tx.outputs[0]
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
signature4 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
self.assertNotEqual(signature2, signature3)
self.assertEqual(signature3, signature4)
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature3))
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=99))
self.assertTrue(verify(channel, unhexlify('99'), signature3))
@@ -1,5 +1,5 @@
import unittest
from unittest import skipIf, skip
from unittest import skipIf
import asyncio
import os
from binascii import hexlify
@@ -51,8 +51,7 @@ class FileCommands(CommandTestCase):
self.addCleanup(task.cancel)
return tx, btih
#@skipIf(TorrentSession is None, "libtorrent not installed")
@skip
@skipIf(TorrentSession is None, "libtorrent not installed")
async def test_download_torrent(self):
tx, btih = await self.initialize_torrent()
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
@@ -90,6 +89,21 @@ class FileCommands(CommandTestCase):
await self.reflector.blob_manager.delete_blobs(all_except_sd)
self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
async def test_sd_blob_fields_fallback(self):
claim_id = self.get_claim_id(await self.stream_create('foo', '0.01', suffix='.txt'))
stream = (await self.daemon.jsonrpc_file_list())["items"][0]
stream.descriptor.suggested_file_name = ' '
stream.descriptor.stream_name = ' '
stream.descriptor.stream_hash = stream.descriptor.get_stream_hash()
sd_hash = stream.descriptor.sd_hash = stream.descriptor.calculate_sd_hash()
await stream.descriptor.make_sd_blob()
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.stream_update(claim_id=claim_id, sd_hash=sd_hash)
file_dict = await self.out(self.daemon.jsonrpc_get('lbry://foo', save_file=True))
self.assertEqual(file_dict['suggested_file_name'], stream.file_name)
self.assertEqual(file_dict['stream_name'], stream.file_name)
self.assertEqual(file_dict['mime_type'], 'text/plain')
async def test_file_management(self):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')
+2 -1
View File
@@ -305,7 +305,6 @@ class TestStreamManager(BlobExchangeTestBase):
self.assertEqual(stored_status, "running")
await stream.stop()
await asyncio.sleep(1) # TODO: should not be needed
self.assertFalse(stream.finished)
self.assertFalse(stream.running)
@@ -341,6 +340,8 @@ class TestStreamManager(BlobExchangeTestBase):
try:
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
error = err
self.assertEqual(expected_error, type(error))