Compare commits

..
13 changed files with 63 additions and 100 deletions

No files matched your search

+2
View File
@@ -70,6 +70,7 @@ test:json-api:
paths:
- lbrynet-${OS}.zip
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller
- python3.7 docker/set_build.py
- pip install -e .
@@ -125,6 +126,7 @@ build:windows:
after_script:
- rmdir -Recurse venv
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller
- python docker/set_build.py
- pip install -e .
-51
View File
@@ -1,51 +0,0 @@
dist: xenial
language: python
python: "3.7"
jobs:
include:
- stage: code quality
name: "pylint & mypy"
install:
- make install tools
script: make lint
- stage: test
name: "Unit Tests"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
- name: "Integration Tests - Data Network"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e datanetwork
- name: "Integration Tests - Blockchain"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e blockchain
- name: "Integration Tests - Other"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e other
- name: "Run Examples"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.52.0"
__version__ = "0.53.2"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+2
View File
@@ -33,6 +33,8 @@ def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
unpadder = PKCS7(AES.block_size).unpadder()
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
return result.decode(), init_vector
except UnicodeDecodeError:
raise InvalidPasswordError()
except ValueError as e:
if e.args[0] == 'Invalid padding bytes.':
raise InvalidPasswordError()
+26 -23
View File
@@ -136,18 +136,13 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
stream_hashes_and_bt_infohashes = transaction.execute("select stream_hash, bt_infohash from file").fetchall()
stream_hashes = tuple(
stream_hash for stream_hash, _ in stream_hashes_and_bt_infohashes if stream_hash is not None
)
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", stream_hashes):
"order by c.rowid desc").fetchall():
claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -539,22 +534,30 @@ class SQLiteStorage(SQLiteMixin):
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1 "
files = {}
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
"and stream_hash is not null"
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
).fetchall()
return await self.db.run(update_manually_removed_files)
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
if download_directory and file_name:
files[stream_hash] = download_directory, file_name
return files
def detect_removed(files):
return [
stream_hash for stream_hash, (download_directory, file_name) in files.items()
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode()))
]
def update_db_removed(transaction: sqlite3.Connection, removed):
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
for cur in _batched_select(transaction, query, removed):
cur.fetchall()
stream_and_file = await self.db.run(update_manually_removed_files)
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
if removed:
await self.db.run(update_db_removed, removed)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
+1
View File
@@ -144,6 +144,7 @@ class StreamManager:
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Recover %i files", len(to_recover))
await self.recover_streams(to_recover)
log.info("Initializing %i files", len(to_start))
+5 -2
View File
@@ -271,7 +271,7 @@ class Ledger(metaclass=LedgerRegistry):
async def get_local_status_and_history(self, address, history=None):
if not history:
address_details = await self.db.get_address(address=address)
history = address_details['history'] or ''
history = (address_details['history'] if address_details else '') or ''
parts = history.split(':')[:-1]
return (
hexlify(sha256(history.encode())).decode() if history else None,
@@ -616,9 +616,12 @@ class Ledger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
found = False
_, local_history = await self.get_local_status_and_history(None, history=record['history'])
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id and local_height >= height:
found = True
+5 -11
View File
@@ -77,7 +77,7 @@ class ClientSession(BaseClientSession):
self.synchronous_close()
raise
except asyncio.CancelledError:
log.warning("cancelled sending %s to %s:%i", method, *self.server)
log.info("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
raise
finally:
@@ -90,7 +90,6 @@ class ClientSession(BaseClientSession):
while True:
try:
if self.is_closing():
log.warning("is closing, reconnect")
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
@@ -100,14 +99,10 @@ class ClientSession(BaseClientSession):
except RPCError as e:
log.warning("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
retry_delay = 60 * 60
except asyncio.TimeoutError:
log.error("closing due to timeout")
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.warning("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
except Exception:
log.exception("unexpected error")
raise
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
@@ -132,7 +127,7 @@ class ClientSession(BaseClientSession):
controller.add(request.args)
def connection_lost(self, exc):
log.warning("Connection lost: %s:%d", *self.server)
log.debug("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
@@ -228,8 +223,7 @@ class Network:
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("Ignoring connection error")
log.warning("Raising cancelled error")
pass
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
+1 -7
View File
@@ -115,7 +115,6 @@ class SessionBase(asyncio.Protocol):
try:
await asyncio.wait_for(self._can_send.wait(), secs)
except asyncio.TimeoutError:
self.logger.warning("abort connection after limited wait (%s)", secs)
self.abort()
raise asyncio.TimeoutError(f'task timed out after {secs}s')
@@ -132,7 +131,6 @@ class SessionBase(asyncio.Protocol):
self.transport.write(framed_message)
def _bump_errors(self):
self.logger.warning("bump errors %i", self.errors)
self.errors += 1
if self.errors >= self.max_errors:
# Don't await self.close() because that is self-cancelling
@@ -183,7 +181,6 @@ class SessionBase(asyncio.Protocol):
"""Called by asyncio when the connection closes.
Tear down things done in connection_made."""
self.logger.error("connection lost to %s - %s", self.peer_address_str(), exc)
self._address = None
self.transport = None
self._task_group.cancel()
@@ -223,14 +220,12 @@ class SessionBase(asyncio.Protocol):
def abort(self):
"""Forcefully close the connection."""
self.logger.warning("abort connection")
if self.transport:
self.transport.abort()
# TODO: replace with synchronous_close
async def close(self, *, force_after=30):
"""Close the connection and return when closed."""
self.logger.warning("close connection")
self._close()
if self._pm_task:
with suppress(CancelledError):
@@ -239,7 +234,6 @@ class SessionBase(asyncio.Protocol):
await self._pm_task
def synchronous_close(self):
self.logger.warning("synchronous close")
self._close()
if self._pm_task and not self._pm_task.done():
self._pm_task.cancel()
@@ -400,7 +394,7 @@ class RPCSession(SessionBase):
try:
requests = self.connection.receive_message(message)
except ProtocolError as e:
self.logger.warning(f'protocol error {e}')
self.logger.debug(f'{e}')
if e.error_message:
await self._send_message(e.error_message)
if e.code == JSONRPC.PARSE_ERROR:
+1 -1
View File
@@ -127,7 +127,7 @@ def release(args):
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(current_version.tag)
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
print()
+1 -3
View File
@@ -54,8 +54,6 @@ class NetworkTests(IntegrationTestCase):
class ReconnectTests(IntegrationTestCase):
VERBOSITY = logging.WARN
async def test_multiple_servers(self):
# we have a secondary node that connects later, so
node2 = SPVNode(self.conductor.spv_module, node_number=2)
@@ -80,7 +78,7 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.stop()
initial_height = self.ledger.local_height_including_downloaded_height
await self.blockchain.generate(100)
while self.conductor.spv_node.server.bp.height < initial_height + 99: # off by 1
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
await asyncio.sleep(0.1)
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
# locks header processing so we make sure we are the only ones modifying it
+3
View File
@@ -375,6 +375,9 @@ class TestQueries(AsyncioTestCase):
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
async def test_empty_history(self):
self.assertEqual((None, []), await self.ledger.get_local_status_and_history(''))
class TestUpgrade(AsyncioTestCase):
+15 -1
View File
@@ -39,6 +39,14 @@ class TestAESEncryptDecrypt(TestCase):
with self.assertRaises(InvalidPasswordError):
aes_decrypt('notbubblegum', aes_encrypt('bubblegum', self.message))
def test_edge_case_invalid_password_valid_padding_invalid_unicode(self):
with self.assertRaises(InvalidPasswordError):
aes_decrypt(
'notbubblegum',
'gy3/mNq3FWB/xAXirOQnlAqQLuvhLGXZaeGBUIg1w6yY4PDLDT7BU83XOfBsJol'
'uWU5zEU4+upOFH35HDqyV8EMQhcKSufN9WkT1izEbFtweBUTK8nTSkV7NBppE1Jaz'
)
def test_better_encrypt_decrypt(self):
self.assertEqual(
b'valuable value',
@@ -46,9 +54,15 @@ class TestAESEncryptDecrypt(TestCase):
'super secret',
better_aes_encrypt('super secret', b'valuable value')))
def test_better_decrypt_error(self):
@mock.patch('os.urandom', side_effect=lambda i: b'd'*i)
def test_better_decrypt_error(self, _):
with self.assertRaises(InvalidPasswordError):
better_aes_decrypt(
'super secret but wrong',
better_aes_encrypt('super secret', b'valuable value')
)
def test_edge_case_invalid_password_valid_everything(self):
value = b'czo4MTkyOjE2OjE6VrwsN8FSJlegxHVEQePoyjWT1k8yAXBCUbbGCFKcsNY='
self.assertEqual(b'valuable value', better_aes_decrypt('super secret', value))
self.assertNotEqual(b'valuable value', better_aes_decrypt('super secret but wrong', value))