Compare commits

..
Author SHA1 Message Date
Jack Robison b75077c1e4 debug 2020-01-12 19:14:45 -05:00
Jack Robison 8098438882 debug session closing 2020-01-12 18:21:09 -05:00
15 changed files with 105 additions and 87 deletions

No files matched your search

+2 -4
View File
@@ -70,8 +70,7 @@ test:json-api:
paths:
- lbrynet-${OS}.zip
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.6
- pip install pyinstaller
- python3.7 docker/set_build.py
- pip install -e .
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
@@ -126,8 +125,7 @@ build:windows:
after_script:
- rmdir -Recurse venv
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.6
- pip install pyinstaller
- python docker/set_build.py
- pip install -e .
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
+51
View File
@@ -0,0 +1,51 @@
dist: xenial
language: python
python: "3.7"
jobs:
include:
- stage: code quality
name: "pylint & mypy"
install:
- make install tools
script: make lint
- stage: test
name: "Unit Tests"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
- name: "Integration Tests - Data Network"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e datanetwork
- name: "Integration Tests - Blockchain"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e blockchain
- name: "Integration Tests - Other"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e other
- name: "Run Examples"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.53.2"
__version__ = "0.52.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
-2
View File
@@ -33,8 +33,6 @@ def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
unpadder = PKCS7(AES.block_size).unpadder()
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
return result.decode(), init_vector
except UnicodeDecodeError:
raise InvalidPasswordError()
except ValueError as e:
if e.args[0] == 'Invalid padding bytes.':
raise InvalidPasswordError()
-1
View File
@@ -55,7 +55,6 @@ Code | Name | Message
409 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
410 | KeyFeeAboveMaxAllowed | {message}
411 | InvalidPassword | Password is invalid.
412 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
-6
View File
@@ -209,12 +209,6 @@ class InvalidPasswordError(WalletError):
super().__init__("Password is invalid.")
class IncompatibleWalletServerError(WalletError):
def __init__(self, server, port):
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class BlobError(BaseError):
"""
**Blobs**
+23 -26
View File
@@ -136,13 +136,18 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
stream_hashes_and_bt_infohashes = transaction.execute("select stream_hash, bt_infohash from file").fetchall()
stream_hashes = tuple(
stream_hash for stream_hash, _ in stream_hashes_and_bt_infohashes if stream_hash is not None
)
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.* "
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"order by c.rowid desc").fetchall():
"where file.stream_hash in {} "
"order by c.rowid desc", stream_hashes):
claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -534,30 +539,22 @@ class SQLiteStorage(SQLiteMixin):
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
files = {}
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1 "
"and stream_hash is not null"
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
if download_directory and file_name:
files[stream_hash] = download_directory, file_name
return files
def detect_removed(files):
return [
stream_hash for stream_hash, (download_directory, file_name) in files.items()
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode()))
]
def update_db_removed(transaction: sqlite3.Connection, removed):
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
for cur in _batched_select(transaction, query, removed):
cur.fetchall()
stream_and_file = await self.db.run(update_manually_removed_files)
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
if removed:
await self.db.run(update_db_removed, removed)
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
).fetchall()
return await self.db.run(update_manually_removed_files)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
-1
View File
@@ -144,7 +144,6 @@ class StreamManager:
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Recover %i files", len(to_recover))
await self.recover_streams(to_recover)
log.info("Initializing %i files", len(to_start))
+2 -5
View File
@@ -271,7 +271,7 @@ class Ledger(metaclass=LedgerRegistry):
async def get_local_status_and_history(self, address, history=None):
if not history:
address_details = await self.db.get_address(address=address)
history = (address_details['history'] if address_details else '') or ''
history = address_details['history'] or ''
parts = history.split(':')[:-1]
return (
hexlify(sha256(history.encode())).decode() if history else None,
@@ -616,12 +616,9 @@ class Ledger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
found = False
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
_, local_history = await self.get_local_status_and_history(None, history=record['history'])
for txid, local_height in local_history:
if txid == tx.id and local_height >= height:
found = True
+13 -17
View File
@@ -5,7 +5,6 @@ from operator import itemgetter
from typing import Dict, Optional, Tuple
from lbry import __version__
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
from lbry.wallet.stream import StreamController
@@ -78,7 +77,7 @@ class ClientSession(BaseClientSession):
self.synchronous_close()
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
log.warning("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
raise
finally:
@@ -91,6 +90,7 @@ class ClientSession(BaseClientSession):
while True:
try:
if self.is_closing():
log.warning("is closing, reconnect")
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
@@ -98,17 +98,16 @@ class ClientSession(BaseClientSession):
await self.ensure_server_version()
retry_delay = default_delay
except RPCError as e:
await self.close()
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
log.warning("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
retry_delay = 60 * 60
except IncompatibleWalletServerError:
await self.close()
retry_delay = 60 * 60
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
except (asyncio.TimeoutError, OSError):
except asyncio.TimeoutError:
log.error("closing due to timeout")
await self.close()
retry_delay = min(60, retry_delay * 2)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
log.warning("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
except Exception:
log.exception("unexpected error")
raise
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
@@ -118,12 +117,9 @@ class ClientSession(BaseClientSession):
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
response = await asyncio.wait_for(
return await asyncio.wait_for(
self.send_request('server.version', [__version__, required]), timeout=timeout
)
if tuple(int(piece) for piece in response[0].split(".")) < self.network.MINIMUM_REQUIRED:
raise IncompatibleWalletServerError(*self.server)
return response
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
@@ -136,7 +132,7 @@ class ClientSession(BaseClientSession):
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
log.warning("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
@@ -148,7 +144,6 @@ class ClientSession(BaseClientSession):
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 53, 2)
def __init__(self, ledger):
self.ledger = ledger
@@ -233,7 +228,8 @@ class Network:
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
pass
log.warning("Ignoring connection error")
log.warning("Raising cancelled error")
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
+7 -1
View File
@@ -115,6 +115,7 @@ class SessionBase(asyncio.Protocol):
try:
await asyncio.wait_for(self._can_send.wait(), secs)
except asyncio.TimeoutError:
self.logger.warning("abort connection after limited wait (%s)", secs)
self.abort()
raise asyncio.TimeoutError(f'task timed out after {secs}s')
@@ -131,6 +132,7 @@ class SessionBase(asyncio.Protocol):
self.transport.write(framed_message)
def _bump_errors(self):
self.logger.warning("bump errors %i", self.errors)
self.errors += 1
if self.errors >= self.max_errors:
# Don't await self.close() because that is self-cancelling
@@ -181,6 +183,7 @@ class SessionBase(asyncio.Protocol):
"""Called by asyncio when the connection closes.
Tear down things done in connection_made."""
self.logger.error("connection lost to %s - %s", self.peer_address_str(), exc)
self._address = None
self.transport = None
self._task_group.cancel()
@@ -220,12 +223,14 @@ class SessionBase(asyncio.Protocol):
def abort(self):
"""Forcefully close the connection."""
self.logger.warning("abort connection")
if self.transport:
self.transport.abort()
# TODO: replace with synchronous_close
async def close(self, *, force_after=30):
"""Close the connection and return when closed."""
self.logger.warning("close connection")
self._close()
if self._pm_task:
with suppress(CancelledError):
@@ -234,6 +239,7 @@ class SessionBase(asyncio.Protocol):
await self._pm_task
def synchronous_close(self):
self.logger.warning("synchronous close")
self._close()
if self._pm_task and not self._pm_task.done():
self._pm_task.cancel()
@@ -394,7 +400,7 @@ class RPCSession(SessionBase):
try:
requests = self.connection.receive_message(message)
except ProtocolError as e:
self.logger.debug(f'{e}')
self.logger.warning(f'protocol error {e}')
if e.error_message:
await self._send_message(e.error_message)
if e.code == JSONRPC.PARSE_ERROR:
+1 -1
View File
@@ -127,7 +127,7 @@ def release(args):
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
previous_release = repo.release_from_tag(current_version.tag)
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
print()
+4 -4
View File
@@ -54,6 +54,8 @@ class NetworkTests(IntegrationTestCase):
class ReconnectTests(IntegrationTestCase):
VERBOSITY = logging.WARN
async def test_multiple_servers(self):
# we have a secondary node that connects later, so
node2 = SPVNode(self.conductor.spv_module, node_number=2)
@@ -78,7 +80,7 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.stop()
initial_height = self.ledger.local_height_including_downloaded_height
await self.blockchain.generate(100)
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
while self.conductor.spv_node.server.bp.height < initial_height + 99: # off by 1
await asyncio.sleep(0.1)
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
# locks header processing so we make sure we are the only ones modifying it
@@ -145,9 +147,7 @@ class ServerPickingTestCase(AsyncioTestCase):
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
if request.method == 'server.version':
return tuple(request.args)
return {'height': 1}
return {"height": 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
-3
View File
@@ -375,9 +375,6 @@ class TestQueries(AsyncioTestCase):
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
async def test_empty_history(self):
self.assertEqual((None, []), await self.ledger.get_local_status_and_history(''))
class TestUpgrade(AsyncioTestCase):
+1 -15
View File
@@ -39,14 +39,6 @@ class TestAESEncryptDecrypt(TestCase):
with self.assertRaises(InvalidPasswordError):
aes_decrypt('notbubblegum', aes_encrypt('bubblegum', self.message))
def test_edge_case_invalid_password_valid_padding_invalid_unicode(self):
with self.assertRaises(InvalidPasswordError):
aes_decrypt(
'notbubblegum',
'gy3/mNq3FWB/xAXirOQnlAqQLuvhLGXZaeGBUIg1w6yY4PDLDT7BU83XOfBsJol'
'uWU5zEU4+upOFH35HDqyV8EMQhcKSufN9WkT1izEbFtweBUTK8nTSkV7NBppE1Jaz'
)
def test_better_encrypt_decrypt(self):
self.assertEqual(
b'valuable value',
@@ -54,15 +46,9 @@ class TestAESEncryptDecrypt(TestCase):
'super secret',
better_aes_encrypt('super secret', b'valuable value')))
@mock.patch('os.urandom', side_effect=lambda i: b'd'*i)
def test_better_decrypt_error(self, _):
def test_better_decrypt_error(self):
with self.assertRaises(InvalidPasswordError):
better_aes_decrypt(
'super secret but wrong',
better_aes_encrypt('super secret', b'valuable value')
)
def test_edge_case_invalid_password_valid_everything(self):
value = b'czo4MTkyOjE2OjE6VrwsN8FSJlegxHVEQePoyjWT1k8yAXBCUbbGCFKcsNY='
self.assertEqual(b'valuable value', better_aes_decrypt('super secret', value))
self.assertNotEqual(b'valuable value', better_aes_decrypt('super secret but wrong', value))