Compare commits

...
21 Commits
Author SHA1 Message Date
Jack Robison a75c003016 pin pyinstaller to 3.6 2020-01-16 13:44:09 -05:00
Jack Robison ed89841762 pin pyinstaller 2020-01-16 13:24:30 -05:00
Jack Robison 974ed29c36 Merge pull request #2729 from lbryio/fix-headers-subscription-to-old
disconnect from server that returns rpc error for server.version
2020-01-15 17:59:17 -05:00
Jack Robison 53eb033034 disconnect from server that returns rpc error for server.version
-the server returns this error if it thinks we are incompatible with it
2020-01-15 17:06:13 -05:00
Jack Robison f2545b98ab Merge pull request #2728 from lbryio/check-server-version-from-client
check minimum server version from wallet client
2020-01-15 16:53:10 -05:00
Jack Robison 1bf51e855d check minimum server version from wallet client 2020-01-15 15:55:39 -05:00
Jack Robison e4da2a695b v0.53.2 2020-01-14 16:47:56 -05:00
Lex Berezhny 1a905d9a42 make --start-tag work again in release/script.py 2020-01-13 21:40:07 -05:00
Victor Shyba 7d9d0c15d3 fix test_direct_sync flakiness 2020-01-13 21:17:10 -05:00
Victor Shyba d34f5c2712 fix None in history, check records after wait 2020-01-13 21:17:10 -05:00
Victor Shyba a56dd66c98 separate cases and make it deterministic 2020-01-13 21:17:10 -05:00
Victor Shyba 3946bc6662 fix and document new edge case on tests 2020-01-13 21:17:10 -05:00
Victor Shyba 6647dd8f08 fix decrypting invalid bytes with valid padding 2020-01-13 21:17:10 -05:00
Jack Robison 9371122bed Merge pull request #2725 from lbryio/stream-manager-startup
faster stream manager startup
2020-01-13 18:09:49 -05:00
Jack Robison 734cd8ee08 logging 2020-01-13 17:51:05 -05:00
Jack Robison d85d9d05bb fast get_all_lbry_files 2020-01-13 17:51:05 -05:00
Jack Robison 1d7b87b3a9 fix inefficient loop in update_manually_removed_files_since_last_run 2020-01-13 17:51:02 -05:00
Lex Berezhny f0cfde36f2 v0.53.1 2020-01-13 14:12:30 -05:00
Lex Berezhny 7e220d2741 downgrade setuptools 2020-01-13 14:03:45 -05:00
Lex Berezhny c4b1351a43 stop building on travis 2020-01-13 14:03:45 -05:00
Lex Berezhny 90d2597bc3 pin to pyinstaller==3.5 2020-01-13 14:03:45 -05:00
14 changed files with 81 additions and 87 deletions

No files matched your search

+4 -2
View File
@@ -70,7 +70,8 @@ test:json-api:
paths:
- lbrynet-${OS}.zip
script:
- pip install pyinstaller
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.6
- python3.7 docker/set_build.py
- pip install -e .
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
@@ -125,7 +126,8 @@ build:windows:
after_script:
- rmdir -Recurse venv
script:
- pip install pyinstaller
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.6
- python docker/set_build.py
- pip install -e .
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
-51
View File
@@ -1,51 +0,0 @@
dist: xenial
language: python
python: "3.7"
jobs:
include:
- stage: code quality
name: "pylint & mypy"
install:
- make install tools
script: make lint
- stage: test
name: "Unit Tests"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
- name: "Integration Tests - Data Network"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e datanetwork
- name: "Integration Tests - Blockchain"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e blockchain
- name: "Integration Tests - Other"
install:
- pip install tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: tox -e other
- name: "Run Examples"
install:
- make install tools
script:
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.53.0"
__version__ = "0.53.2"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+2
View File
@@ -33,6 +33,8 @@ def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
unpadder = PKCS7(AES.block_size).unpadder()
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
return result.decode(), init_vector
except UnicodeDecodeError:
raise InvalidPasswordError()
except ValueError as e:
if e.args[0] == 'Invalid padding bytes.':
raise InvalidPasswordError()
+1
View File
@@ -55,6 +55,7 @@ Code | Name | Message
409 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
410 | KeyFeeAboveMaxAllowed | {message}
411 | InvalidPassword | Password is invalid.
412 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
+6
View File
@@ -209,6 +209,12 @@ class InvalidPasswordError(WalletError):
super().__init__("Password is invalid.")
class IncompatibleWalletServerError(WalletError):
def __init__(self, server, port):
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class BlobError(BaseError):
"""
**Blobs**
+26 -23
View File
@@ -136,18 +136,13 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
stream_hashes_and_bt_infohashes = transaction.execute("select stream_hash, bt_infohash from file").fetchall()
stream_hashes = tuple(
stream_hash for stream_hash, _ in stream_hashes_and_bt_infohashes if stream_hash is not None
)
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", stream_hashes):
"order by c.rowid desc").fetchall():
claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -539,22 +534,30 @@ class SQLiteStorage(SQLiteMixin):
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1 "
files = {}
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
"and stream_hash is not null"
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
).fetchall()
return await self.db.run(update_manually_removed_files)
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
if download_directory and file_name:
files[stream_hash] = download_directory, file_name
return files
def detect_removed(files):
return [
stream_hash for stream_hash, (download_directory, file_name) in files.items()
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode()))
]
def update_db_removed(transaction: sqlite3.Connection, removed):
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
for cur in _batched_select(transaction, query, removed):
cur.fetchall()
stream_and_file = await self.db.run(update_manually_removed_files)
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
if removed:
await self.db.run(update_db_removed, removed)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
+1
View File
@@ -144,6 +144,7 @@ class StreamManager:
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Recover %i files", len(to_recover))
await self.recover_streams(to_recover)
log.info("Initializing %i files", len(to_start))
+5 -2
View File
@@ -271,7 +271,7 @@ class Ledger(metaclass=LedgerRegistry):
async def get_local_status_and_history(self, address, history=None):
if not history:
address_details = await self.db.get_address(address=address)
history = address_details['history'] or ''
history = (address_details['history'] if address_details else '') or ''
parts = history.split(':')[:-1]
return (
hexlify(sha256(history.encode())).decode() if history else None,
@@ -616,9 +616,12 @@ class Ledger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
found = False
_, local_history = await self.get_local_status_and_history(None, history=record['history'])
local_history = (await self.get_local_status_and_history(
record['address'], history=record['history']
))[1] if record['history'] else []
for txid, local_height in local_history:
if txid == tx.id and local_height >= height:
found = True
+12 -2
View File
@@ -5,6 +5,7 @@ from operator import itemgetter
from typing import Dict, Optional, Tuple
from lbry import __version__
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
from lbry.wallet.stream import StreamController
@@ -97,8 +98,13 @@ class ClientSession(BaseClientSession):
await self.ensure_server_version()
retry_delay = default_delay
except RPCError as e:
log.warning("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
await self.close()
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
retry_delay = 60 * 60
except IncompatibleWalletServerError:
await self.close()
retry_delay = 60 * 60
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
@@ -112,9 +118,12 @@ class ClientSession(BaseClientSession):
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
return await asyncio.wait_for(
response = await asyncio.wait_for(
self.send_request('server.version', [__version__, required]), timeout=timeout
)
if tuple(int(piece) for piece in response[0].split(".")) < self.network.MINIMUM_REQUIRED:
raise IncompatibleWalletServerError(*self.server)
return response
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
@@ -139,6 +148,7 @@ class ClientSession(BaseClientSession):
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 53, 2)
def __init__(self, ledger):
self.ledger = ledger
+1 -1
View File
@@ -127,7 +127,7 @@ def release(args):
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(current_version.tag)
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
print()
+4 -4
View File
@@ -54,8 +54,6 @@ class NetworkTests(IntegrationTestCase):
class ReconnectTests(IntegrationTestCase):
VERBOSITY = logging.WARN
async def test_multiple_servers(self):
# we have a secondary node that connects later, so
node2 = SPVNode(self.conductor.spv_module, node_number=2)
@@ -80,7 +78,7 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.stop()
initial_height = self.ledger.local_height_including_downloaded_height
await self.blockchain.generate(100)
while self.conductor.spv_node.server.bp.height < initial_height + 99: # off by 1
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
await asyncio.sleep(0.1)
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
# locks header processing so we make sure we are the only ones modifying it
@@ -147,7 +145,9 @@ class ServerPickingTestCase(AsyncioTestCase):
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
return {"height": 1}
if request.method == 'server.version':
return tuple(request.args)
return {'height': 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
+3
View File
@@ -375,6 +375,9 @@ class TestQueries(AsyncioTestCase):
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
async def test_empty_history(self):
self.assertEqual((None, []), await self.ledger.get_local_status_and_history(''))
class TestUpgrade(AsyncioTestCase):
+15 -1
View File
@@ -39,6 +39,14 @@ class TestAESEncryptDecrypt(TestCase):
with self.assertRaises(InvalidPasswordError):
aes_decrypt('notbubblegum', aes_encrypt('bubblegum', self.message))
def test_edge_case_invalid_password_valid_padding_invalid_unicode(self):
with self.assertRaises(InvalidPasswordError):
aes_decrypt(
'notbubblegum',
'gy3/mNq3FWB/xAXirOQnlAqQLuvhLGXZaeGBUIg1w6yY4PDLDT7BU83XOfBsJol'
'uWU5zEU4+upOFH35HDqyV8EMQhcKSufN9WkT1izEbFtweBUTK8nTSkV7NBppE1Jaz'
)
def test_better_encrypt_decrypt(self):
self.assertEqual(
b'valuable value',
@@ -46,9 +54,15 @@ class TestAESEncryptDecrypt(TestCase):
'super secret',
better_aes_encrypt('super secret', b'valuable value')))
def test_better_decrypt_error(self):
@mock.patch('os.urandom', side_effect=lambda i: b'd'*i)
def test_better_decrypt_error(self, _):
with self.assertRaises(InvalidPasswordError):
better_aes_decrypt(
'super secret but wrong',
better_aes_encrypt('super secret', b'valuable value')
)
def test_edge_case_invalid_password_valid_everything(self):
value = b'czo4MTkyOjE2OjE6VrwsN8FSJlegxHVEQePoyjWT1k8yAXBCUbbGCFKcsNY='
self.assertEqual(b'valuable value', better_aes_decrypt('super secret', value))
self.assertNotEqual(b'valuable value', better_aes_decrypt('super secret but wrong', value))