Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a75c003016 | ||
|
|
ed89841762 | ||
|
|
974ed29c36 | ||
|
|
53eb033034 | ||
|
|
f2545b98ab | ||
|
|
1bf51e855d | ||
|
|
e4da2a695b | ||
|
|
1a905d9a42 | ||
|
|
7d9d0c15d3 | ||
|
|
d34f5c2712 | ||
|
|
a56dd66c98 | ||
|
|
3946bc6662 | ||
|
|
6647dd8f08 | ||
|
|
9371122bed | ||
|
|
734cd8ee08 | ||
|
|
d85d9d05bb | ||
|
|
1d7b87b3a9 | ||
|
|
f0cfde36f2 | ||
|
|
7e220d2741 | ||
|
|
c4b1351a43 | ||
|
|
90d2597bc3 | ||
|
|
c1508c94c2 |
No files matched your search
+4
-2
@@ -70,7 +70,8 @@ test:json-api:
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install pyinstaller
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.6
|
||||
- python3.7 docker/set_build.py
|
||||
- pip install -e .
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
@@ -125,7 +126,8 @@ build:windows:
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install pyinstaller
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.6
|
||||
- python docker/set_build.py
|
||||
- pip install -e .
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
dist: xenial
|
||||
language: python
|
||||
python: "3.7"
|
||||
|
||||
jobs:
|
||||
include:
|
||||
|
||||
- stage: code quality
|
||||
name: "pylint & mypy"
|
||||
install:
|
||||
- make install tools
|
||||
script: make lint
|
||||
|
||||
- stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- make install tools
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
- name: "Integration Tests - Data Network"
|
||||
install:
|
||||
- pip install tox-travis
|
||||
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
|
||||
script: tox -e datanetwork
|
||||
|
||||
- name: "Integration Tests - Blockchain"
|
||||
install:
|
||||
- pip install tox-travis
|
||||
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
|
||||
script: tox -e blockchain
|
||||
|
||||
- name: "Integration Tests - Other"
|
||||
install:
|
||||
- pip install tox-travis
|
||||
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
|
||||
script: tox -e other
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- make install tools
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/venv
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $HOME/Library/Caches/Homebrew
|
||||
- $TRAVIS_BUILD_DIR/.tox
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.52.0"
|
||||
__version__ = "0.53.2"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -33,6 +33,8 @@ def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
|
||||
return result.decode(), init_vector
|
||||
except UnicodeDecodeError:
|
||||
raise InvalidPasswordError()
|
||||
except ValueError as e:
|
||||
if e.args[0] == 'Invalid padding bytes.':
|
||||
raise InvalidPasswordError()
|
||||
|
||||
@@ -55,6 +55,7 @@ Code | Name | Message
|
||||
409 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
410 | KeyFeeAboveMaxAllowed | {message}
|
||||
411 | InvalidPassword | Password is invalid.
|
||||
412 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
@@ -209,6 +209,12 @@ class InvalidPasswordError(WalletError):
|
||||
super().__init__("Password is invalid.")
|
||||
|
||||
|
||||
class IncompatibleWalletServerError(WalletError):
|
||||
|
||||
def __init__(self, server, port):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
|
||||
@@ -136,18 +136,13 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
stream_hashes_and_bt_infohashes = transaction.execute("select stream_hash, bt_infohash from file").fetchall()
|
||||
stream_hashes = tuple(
|
||||
stream_hash for stream_hash, _ in stream_hashes_and_bt_infohashes if stream_hash is not None
|
||||
)
|
||||
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
|
||||
"select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", stream_hashes):
|
||||
"order by c.rowid desc").fetchall():
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -539,22 +534,30 @@ class SQLiteStorage(SQLiteMixin):
|
||||
Update files that have been removed from the downloads directory since the last run
|
||||
"""
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
removed = []
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1 "
|
||||
files = {}
|
||||
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
|
||||
"and stream_hash is not null"
|
||||
).fetchall():
|
||||
if download_directory and file_name and os.path.isfile(
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
).fetchall()
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
|
||||
if download_directory and file_name:
|
||||
files[stream_hash] = download_directory, file_name
|
||||
return files
|
||||
|
||||
def detect_removed(files):
|
||||
return [
|
||||
stream_hash for stream_hash, (download_directory, file_name) in files.items()
|
||||
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode()))
|
||||
]
|
||||
|
||||
def update_db_removed(transaction: sqlite3.Connection, removed):
|
||||
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
|
||||
for cur in _batched_select(transaction, query, removed):
|
||||
cur.fetchall()
|
||||
|
||||
stream_and_file = await self.db.run(update_manually_removed_files)
|
||||
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
|
||||
if removed:
|
||||
await self.db.run(update_db_removed, removed)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
|
||||
@@ -144,6 +144,7 @@ class StreamManager:
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Recover %i files", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
|
||||
@@ -271,7 +271,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
async def get_local_status_and_history(self, address, history=None):
|
||||
if not history:
|
||||
address_details = await self.db.get_address(address=address)
|
||||
history = address_details['history'] or ''
|
||||
history = (address_details['history'] if address_details else '') or ''
|
||||
parts = history.split(':')[:-1]
|
||||
return (
|
||||
hexlify(sha256(history.encode())).decode() if history else None,
|
||||
@@ -616,9 +616,12 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
)) for address_record in records
|
||||
], timeout=timeout)
|
||||
if pending:
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
for record in records:
|
||||
found = False
|
||||
_, local_history = await self.get_local_status_and_history(None, history=record['history'])
|
||||
local_history = (await self.get_local_status_and_history(
|
||||
record['address'], history=record['history']
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id and local_height >= height:
|
||||
found = True
|
||||
|
||||
+17
-13
@@ -5,6 +5,7 @@ from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
|
||||
@@ -77,7 +78,7 @@ class ClientSession(BaseClientSession):
|
||||
self.synchronous_close()
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
@@ -90,7 +91,6 @@ class ClientSession(BaseClientSession):
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
log.warning("is closing, reconnect")
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
@@ -98,16 +98,17 @@ class ClientSession(BaseClientSession):
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
log.warning("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
await self.close()
|
||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except asyncio.TimeoutError:
|
||||
log.error("closing due to timeout")
|
||||
except IncompatibleWalletServerError:
|
||||
await self.close()
|
||||
retry_delay = 60 * 60
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.warning("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
except Exception:
|
||||
log.exception("unexpected error")
|
||||
raise
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
@@ -117,9 +118,12 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
return await asyncio.wait_for(
|
||||
response = await asyncio.wait_for(
|
||||
self.send_request('server.version', [__version__, required]), timeout=timeout
|
||||
)
|
||||
if tuple(int(piece) for piece in response[0].split(".")) < self.network.MINIMUM_REQUIRED:
|
||||
raise IncompatibleWalletServerError(*self.server)
|
||||
return response
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
@@ -132,7 +136,7 @@ class ClientSession(BaseClientSession):
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.warning("Connection lost: %s:%d", *self.server)
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
@@ -144,6 +148,7 @@ class ClientSession(BaseClientSession):
|
||||
class Network:
|
||||
|
||||
PROTOCOL_VERSION = __version__
|
||||
MINIMUM_REQUIRED = (0, 53, 2)
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
@@ -228,8 +233,7 @@ class Network:
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("Ignoring connection error")
|
||||
log.warning("Raising cancelled error")
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
|
||||
@@ -115,7 +115,6 @@ class SessionBase(asyncio.Protocol):
|
||||
try:
|
||||
await asyncio.wait_for(self._can_send.wait(), secs)
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.warning("abort connection after limited wait (%s)", secs)
|
||||
self.abort()
|
||||
raise asyncio.TimeoutError(f'task timed out after {secs}s')
|
||||
|
||||
@@ -132,7 +131,6 @@ class SessionBase(asyncio.Protocol):
|
||||
self.transport.write(framed_message)
|
||||
|
||||
def _bump_errors(self):
|
||||
self.logger.warning("bump errors %i", self.errors)
|
||||
self.errors += 1
|
||||
if self.errors >= self.max_errors:
|
||||
# Don't await self.close() because that is self-cancelling
|
||||
@@ -183,7 +181,6 @@ class SessionBase(asyncio.Protocol):
|
||||
"""Called by asyncio when the connection closes.
|
||||
|
||||
Tear down things done in connection_made."""
|
||||
self.logger.error("connection lost to %s - %s", self.peer_address_str(), exc)
|
||||
self._address = None
|
||||
self.transport = None
|
||||
self._task_group.cancel()
|
||||
@@ -223,14 +220,12 @@ class SessionBase(asyncio.Protocol):
|
||||
|
||||
def abort(self):
|
||||
"""Forcefully close the connection."""
|
||||
self.logger.warning("abort connection")
|
||||
if self.transport:
|
||||
self.transport.abort()
|
||||
|
||||
# TODO: replace with synchronous_close
|
||||
async def close(self, *, force_after=30):
|
||||
"""Close the connection and return when closed."""
|
||||
self.logger.warning("close connection")
|
||||
self._close()
|
||||
if self._pm_task:
|
||||
with suppress(CancelledError):
|
||||
@@ -239,7 +234,6 @@ class SessionBase(asyncio.Protocol):
|
||||
await self._pm_task
|
||||
|
||||
def synchronous_close(self):
|
||||
self.logger.warning("synchronous close")
|
||||
self._close()
|
||||
if self._pm_task and not self._pm_task.done():
|
||||
self._pm_task.cancel()
|
||||
@@ -400,7 +394,7 @@ class RPCSession(SessionBase):
|
||||
try:
|
||||
requests = self.connection.receive_message(message)
|
||||
except ProtocolError as e:
|
||||
self.logger.warning(f'protocol error {e}')
|
||||
self.logger.debug(f'{e}')
|
||||
if e.error_message:
|
||||
await self._send_message(e.error_message)
|
||||
if e.code == JSONRPC.PARSE_ERROR:
|
||||
|
||||
+1
-1
@@ -127,7 +127,7 @@ def release(args):
|
||||
new_version = current_version.increment(args.action)
|
||||
print(f' New Version: {new_version}')
|
||||
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
|
||||
|
||||
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
|
||||
print()
|
||||
|
||||
@@ -54,8 +54,6 @@ class NetworkTests(IntegrationTestCase):
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_multiple_servers(self):
|
||||
# we have a secondary node that connects later, so
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
@@ -80,7 +78,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.stop()
|
||||
initial_height = self.ledger.local_height_including_downloaded_height
|
||||
await self.blockchain.generate(100)
|
||||
while self.conductor.spv_node.server.bp.height < initial_height + 99: # off by 1
|
||||
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
|
||||
await asyncio.sleep(0.1)
|
||||
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
|
||||
# locks header processing so we make sure we are the only ones modifying it
|
||||
@@ -147,7 +145,9 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
class FakeSession(RPCSession):
|
||||
async def handle_request(self, request):
|
||||
await asyncio.sleep(latency)
|
||||
return {"height": 1}
|
||||
if request.method == 'server.version':
|
||||
return tuple(request.args)
|
||||
return {'height': 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
@@ -375,6 +375,9 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
|
||||
async def test_empty_history(self):
|
||||
self.assertEqual((None, []), await self.ledger.get_local_status_and_history(''))
|
||||
|
||||
|
||||
class TestUpgrade(AsyncioTestCase):
|
||||
|
||||
|
||||
@@ -39,6 +39,14 @@ class TestAESEncryptDecrypt(TestCase):
|
||||
with self.assertRaises(InvalidPasswordError):
|
||||
aes_decrypt('notbubblegum', aes_encrypt('bubblegum', self.message))
|
||||
|
||||
def test_edge_case_invalid_password_valid_padding_invalid_unicode(self):
|
||||
with self.assertRaises(InvalidPasswordError):
|
||||
aes_decrypt(
|
||||
'notbubblegum',
|
||||
'gy3/mNq3FWB/xAXirOQnlAqQLuvhLGXZaeGBUIg1w6yY4PDLDT7BU83XOfBsJol'
|
||||
'uWU5zEU4+upOFH35HDqyV8EMQhcKSufN9WkT1izEbFtweBUTK8nTSkV7NBppE1Jaz'
|
||||
)
|
||||
|
||||
def test_better_encrypt_decrypt(self):
|
||||
self.assertEqual(
|
||||
b'valuable value',
|
||||
@@ -46,9 +54,15 @@ class TestAESEncryptDecrypt(TestCase):
|
||||
'super secret',
|
||||
better_aes_encrypt('super secret', b'valuable value')))
|
||||
|
||||
def test_better_decrypt_error(self):
|
||||
@mock.patch('os.urandom', side_effect=lambda i: b'd'*i)
|
||||
def test_better_decrypt_error(self, _):
|
||||
with self.assertRaises(InvalidPasswordError):
|
||||
better_aes_decrypt(
|
||||
'super secret but wrong',
|
||||
better_aes_encrypt('super secret', b'valuable value')
|
||||
)
|
||||
|
||||
def test_edge_case_invalid_password_valid_everything(self):
|
||||
value = b'czo4MTkyOjE2OjE6VrwsN8FSJlegxHVEQePoyjWT1k8yAXBCUbbGCFKcsNY='
|
||||
self.assertEqual(b'valuable value', better_aes_decrypt('super secret', value))
|
||||
self.assertNotEqual(b'valuable value', better_aes_decrypt('super secret but wrong', value))
|
||||
Reference in new issue
Block a user