Compare commits

...
120 Commits
Author SHA1 Message Date
Lex Berezhny 7743a6a71d v0.43.2 2019-10-17 13:38:52 -04:00
Lex Berezhny 65a18e78dd publish checks all accounts before forwarding to stream_create/stream_update 2019-10-17 13:38:19 -04:00
Lex Berezhny 12a0cbf4e1 improve preference sync integration test 2019-10-16 17:19:10 -04:00
Lex Berezhny 55c2073813 v0.43.1 2019-10-16 15:50:10 -04:00
Lex Berezhny 39bc8c45c3 test fix 2019-10-16 15:49:10 -04:00
Lex Berezhny 46465f5409 removed WALLET_IS_UNLOCKED condition since it does not take into account multi-wallet 2019-10-16 15:49:10 -04:00
Lex Berezhny 7b5f23e17d incorporate password in the wallet.hash 2019-10-16 15:49:10 -04:00
Lex Berezhny 380efc5ae0 regen docs 2019-10-16 12:06:05 -04:00
Lex Berezhny d1a123997d sync_apply always updates local password if encrypt-on-disk is enabled 2019-10-16 12:06:05 -04:00
Lex Berezhny 722419dffc migrate per-account on disk encryption to encrypt-on-disk preference 2019-10-16 12:06:05 -04:00
Lex Berezhny bcd69f812e sync_apply overwrites local password when encrypt-on-disk is true 2019-10-16 12:06:05 -04:00
Lex Berezhny 1075cd9995 regenerated api.json 2019-10-16 12:06:05 -04:00
Lex Berezhny e5be1ff157 bug fixes and encrypt-on-disk preference 2019-10-16 12:06:05 -04:00
Lex Berezhny 398fe55083 renamed: test_sync.py -> test_wallet_commands.py 2019-10-16 12:06:05 -04:00
shyba b4ee537f7c Merge pull request #2553 from lbryio/choco_37
pin choco py to 37 on win build
2019-10-16 12:11:06 -03:00
Victor Shyba b0ad30a5d8 pin choco py to 37 on win build 2019-10-16 11:48:33 -03:00
Lex Berezhny 26d17e2e69 updated api.json docs 2019-10-15 12:12:35 -04:00
Lex Berezhny 43b619b2c6 release script fix for PRs with null body 2019-10-14 19:09:32 -04:00
Lex Berezhny 0e4426e084 v0.43.0 2019-10-14 13:07:15 -04:00
Jack Robison a27943ea14 update time_to_first_byte.py 2019-10-14 12:59:59 -04:00
Lex Berezhny b8b24da292 encrypt/decrypt/lock/unlock commands return True 2019-10-14 12:57:09 -04:00
Lex Berezhny 45eef03576 fixes 2019-10-14 12:57:09 -04:00
Lex Berezhny 270f77df24 wallet encryption 2019-10-14 12:57:09 -04:00
Lex Berezhny dc580b9eff wip wallet encrypt/decrypt API 2019-10-14 12:57:09 -04:00
Lex Berezhny fed7e94d8d added wallet send command 2019-10-14 10:43:54 -04:00
Lex Berezhny 9bb0ca2f0d async def 2019-10-13 22:32:09 -04:00
Lex Berezhny 3d82becadc utxo release applies to all accounts in wallet 2019-10-13 22:32:09 -04:00
Lex Berezhny e2b92c99ef added wallet balance command 2019-10-13 20:06:11 -04:00
Lex Berezhny fea7275148 mypy fixes 2019-10-12 20:23:29 -04:00
Lex Berezhny 64581fc81b pylint 2019-10-12 20:23:29 -04:00
Lex Berezhny 37ae302fc6 timestamped top-level wallet preferences 2019-10-12 20:23:29 -04:00
Alex Grin c51cc02a87 Merge pull request #2527 from lbryio/faster_trending
only do trending calculations near the tip of the chain
2019-10-11 13:20:35 -04:00
Thomas Zarebczan 6c92fbd8de add oga and ogv mime types
https://gingertech.net/2008/09/05/new-ogg-mime-types-ratified/#targetText=The%20new%20Ogg%20MIME%20types,I%20files%20for%20backwards%20compatibility.
2019-10-11 11:40:36 -04:00
Alex Grintsvayg 065215cc27 only do trending calculations near the tip of the chain 2019-10-11 10:51:50 -04:00
StrikerRUS eb8a88dbd6 refactored integration folder as well 2019-10-11 10:31:54 -04:00
StrikerRUS 62e3258dda revert spaces fixes 2019-10-11 10:31:54 -04:00
StrikerRUS 9959d2c70d use more specific asserts in torba tests 2019-10-11 10:31:54 -04:00
Miroslav Kovar 19e8b6e11f Add test 2019-10-11 10:12:18 -04:00
Miroslav Kovar 63447d1ec3 Make more readable 2019-10-11 10:12:18 -04:00
Miroslav Kovar 75fac93544 Make --verbose behave as per doc 2019-10-11 10:12:18 -04:00
Alex Grin 23f7faa74c Merge pull request #2502 from lbryio/minor_changes
small clarifying changes
2019-10-10 08:59:47 -04:00
Alex Grintsvayg 284fdb2fcb small clarifying changes 2019-10-09 15:42:32 -04:00
Jack Robison 44b27bbfc8 Merge pull request #2487 from lbryio/fix-connection-reset
handle error if range request transport is reset
2019-10-08 16:20:07 -04:00
Jack Robison 35d2d55a27 handle error if range request transport is reset 2019-10-08 14:37:51 -04:00
Jack Robison 292f1953e1 Merge pull request #2521 from lbryio/resolve-ipv4-only
fix resolving fixed peers from an ipv6 connection
2019-10-08 14:37:49 -04:00
Jack Robison 5cd0828dd3 force ipv4 when resolving a url
-log violating ip addresses
2019-10-08 13:31:06 -04:00
Jack Robison 077281a77a v0.42.2 2019-10-07 13:03:51 -04:00
Jack Robison 7055d250d0 Merge pull request #2503 from lbryio/reuse-loggly
Reuse loggly connection
2019-10-07 12:30:04 -04:00
Jack Robison 8c0196301d Merge pull request #2489 from lbryio/dht_object_cache
DHT object cache revamp
2019-10-07 12:27:07 -04:00
Jack Robison f99aacd9dc improve store_to_peer retry, reduce error verbosity 2019-10-07 12:26:49 -04:00
Jack Robison 43b83bec83 remove dead code 2019-10-07 12:26:49 -04:00
Jack Robison fc1b4658fc re-use loggly connection 2019-10-07 12:26:49 -04:00
Victor Shyba 05a28ea346 handle attempts to add peers with no node id and issue a warning 2019-10-07 13:20:38 -03:00
Victor Shyba 624f986978 handle peer_is_good for unknown node id 2019-10-07 13:20:38 -03:00
Victor Shyba 91a9957918 avoid peer mutability
Mutability can be harmful here, specially on fields used for __hash__
It can also make the cache less efficient
2019-10-07 13:20:38 -03:00
Victor Shyba 8cd18e47e7 get_kad... -> make_kad.. 2019-10-07 13:20:38 -03:00
Victor Shyba 517868f73c fix unit tests 2019-10-07 13:20:38 -03:00
Victor Shyba 18d80f4ed5 1 and 65536 arent valid ports 2019-10-07 13:20:38 -03:00
Victor Shyba 2e2c751d75 check for none on tcp port 2019-10-07 13:20:38 -03:00
Victor Shyba 7aa5e2cbe1 use a dataclass experiment 2019-10-07 13:20:38 -03:00
Victor Shyba 7d6db20081 use cache for dht peer objects 2019-10-07 13:20:38 -03:00
Miroslav Kovar 911ec9444b Decrease the status refresh rate 2019-10-07 11:42:40 -04:00
Miroslav Kovar 2883c6bab3 Fix incorrect counting of total bytes sent / received 2019-10-07 11:42:40 -04:00
OrginalS f5fed54195 Punctuation
Added missing punctuation and changed existing to more fitting one.
2019-10-07 10:58:13 -04:00
Rishav Sharma 91b1446375 Grammar correction 2019-10-07 10:50:14 -04:00
Lex Berezhny 56df145052 updating/abanding streams/channels works across all accounts 2019-10-07 09:57:22 -04:00
Lex Berezhny 9e873f77f9 bug fixes related to cross-account / wallet operations 2019-10-07 09:57:22 -04:00
Alex Grintsvayg 27bbd91635 include fn name and args in jsonrpc error message 2019-10-04 11:25:01 -04:00
Alex Grintsvayg 2c73c80c9f fix formatting 2019-10-04 09:52:29 -04:00
Hugo 3a3a9b5f4e Fix spelling 2019-10-03 21:40:06 -04:00
Lex Berezhny 3f89ed4579 v0.42.1 2019-09-30 12:59:45 -04:00
Lex Berezhny 02b01b89f2 fix for wallet_remove command 2019-09-30 12:57:33 -04:00
Victor Shyba 4a6af0b938 retriable resolve 2019-09-30 12:57:18 -04:00
Alex Grintsvayg 9c5b4e3cc9 change trending window to 6 hours 2019-09-30 11:25:52 -04:00
Victor Shyba b3f087dbc7 fix fastest session for when online but unavailable 2019-09-30 10:58:11 -04:00
Jack Robison aa90257ec7 Merge pull request #2481 from lbryio/fix_new_pylint
Fix new pylint
2019-09-25 10:04:01 -04:00
Victor Shyba 348bbc81ca fix for new pylint in lbry/ 2019-09-24 12:53:23 -03:00
Victor Shyba 4231e7f3ff fix for new pylint in torba 2019-09-24 12:23:34 -03:00
Lex Berezhny 4cbc82619d v0.42.0 2019-09-23 13:50:21 -04:00
Lex Berezhny 1b0d802941 is_change bug fix 2019-09-23 13:49:50 -04:00
Lex Berezhny dc72a1e58d fix wallet_create single_key argument 2019-09-23 13:49:50 -04:00
Lex Berezhny 1d5333c737 integration test fix 2019-09-23 13:49:50 -04:00
Lex Berezhny d289041ff3 regnerated docs 2019-09-23 13:49:50 -04:00
Lex Berezhny f287483397 split wallet creation into wallet_add and wallet_create 2019-09-23 13:49:50 -04:00
Lex Berezhny 1ac3b6c173 wallet_list optionally takes wallet_id 2019-09-23 13:49:50 -04:00
Lex Berezhny 1d090765ba pylint dict comp 2019-09-23 13:49:50 -04:00
Lex Berezhny 6c6050528a windows doesnt have json_group_object sqlite function available, switching to group_concat 2019-09-23 13:49:50 -04:00
Lex Berezhny 835d376ca6 wallet_id arg added to more commands 2019-09-23 13:49:50 -04:00
Lex Berezhny 68e9fd8541 account integration test fix 2019-09-23 13:49:50 -04:00
Lex Berezhny e49f8f8c61 regenerated docs 2019-09-23 13:49:50 -04:00
Lex Berezhny b396322f1e removed default_account and default_wallet on Daemon instance 2019-09-23 13:49:50 -04:00
Lex Berezhny ea434e3469 wallet_id related test fixes 2019-09-23 13:49:50 -04:00
Lex Berezhny 84587ac232 wallet commands and wallet_id argument to other commands 2019-09-23 13:49:50 -04:00
Miroslav Kovar 9fbc83fe9d Always handle claim_name as a string 2019-09-20 14:22:01 -04:00
Victor Shyba 965972a719 match regex from app into ours 2019-09-19 17:30:22 -04:00
Miroslav Kovar 34d19c4a72 Remove superfluous test 2019-09-19 17:16:18 -04:00
Miroslav Kovar 9b0910767d Fix formatting, support null argument only 2019-09-19 17:16:18 -04:00
Miroslav Kovar 32cc077e1f enable to set max_key_fee to null via cli 2019-09-19 17:16:18 -04:00
Miroslav Kovar 1e7bec48b6 Add integration test, remove unit test 2019-09-19 15:21:01 -04:00
Miroslav Kovar c035837a14 Fix failing key payment for null max key fee 2019-09-19 15:21:01 -04:00
Miroslav Kovar cb5eb8983b Fix docopt for claim search 2019-09-18 19:30:58 -04:00
Victor Shyba 9c86cb7954 proper dict formatting on tests 2019-09-18 19:28:11 -04:00
Victor Shyba ef94c1eccc use existing donation_address instead 2019-09-18 19:28:11 -04:00
Victor Shyba 64eaa5e3aa add options to set server description, payment address and daily fee 2019-09-18 19:28:11 -04:00
Thomas Zarebczan 826f584aa8 add Flac mime type 2019-09-18 19:20:09 -04:00
Victor Shyba c5530aee5f refactor tests 2019-09-18 19:19:34 -04:00
Victor Shyba 817f01593e call the correct method on is_valid_address 2019-09-18 19:19:34 -04:00
Jack Robison 79727f0e97 Merge pull request #2459 from lbryio/test-sqlite-error-handling
Test sqlite error handling
2019-09-18 10:16:31 -04:00
Jack Robison abac82c1c6 skip test_fetchall_prevents_sqlite_misuse on travis
-print how many attempts the race condition took to trigger
2019-09-17 12:44:43 -04:00
Jack Robison 02c51c6059 typing 2019-09-17 12:09:19 -04:00
Jack Robison 46b45d0387 gotta fetch 'em all 2019-09-17 12:09:19 -04:00
Jack Robison e6c549c457 remove redundant query from sync_missing_blobs 2019-09-17 12:09:19 -04:00
Jack Robison 1aa2983216 fix db errors during shutdown 2019-09-17 12:09:19 -04:00
Jack Robison a095f89c81 remove redundant test 2019-09-17 12:09:19 -04:00
Jack Robison 287902f01b test fetchall prevents SQLITE_MISUSE 2019-09-17 12:09:19 -04:00
Jack Robison 7dc3e4e7a1 test verbosity 2019-09-17 12:09:19 -04:00
Jack Robison 0b31aad901 test binding errors vs sqlite_misuse race errors 2019-09-17 12:09:19 -04:00
Alex Grintsvayg 1791d8e4ee rebased oleg's PR. closes #2446 2019-09-17 11:04:34 -04:00
Lex Berezhny b7c2054a66 mypy fixes 2019-09-17 09:08:29 -04:00
Lex Berezhny 81f16d7061 refactored address_list API to be more detailed 2019-09-17 09:08:29 -04:00
101 changed files with 2988 additions and 1685 deletions

No files matched your search

+1 -1
View File
@@ -110,7 +110,7 @@ jobs:
- OS=windows
- PATH=/c/Python37:/c/Python37/Scripts:/C/Windows/System32/downlevel:$PATH
before_install:
- choco install python --x86
- choco install python --version=3.7.4 --x86
- python -m pip install --upgrade pip
- pip install pywin32
script:
+4 -4
View File
@@ -1,6 +1,6 @@
# Installing LBRY
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md)!
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
@@ -9,7 +9,7 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
## Prerequisites
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
After installing python 3, you'll need to install some additional libraries depending on your operating system.
@@ -17,7 +17,7 @@ After installing python 3, you'll need to install some additional libraries depe
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set
These environment variables also need to be set:
1. PYTHONUNBUFFERED=1
2. EVENT_NOKQUEUE=1
@@ -27,7 +27,7 @@ Remaining dependencies can then be installed by running:
brew install python protobuf
```
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/.
### Linux
+1 -1
View File
@@ -2,7 +2,7 @@
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
+564 -188
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.41.0"
__version__ = "0.43.2"
version = tuple(__version__.split('.'))
+6 -9
View File
@@ -37,18 +37,14 @@ class BlobManager:
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
if self.config.save_blobs or (
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
@@ -82,6 +78,7 @@ class BlobManager:
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
+2 -4
View File
@@ -67,7 +67,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("%s started sending blob we didnt request %s instead of %s", self.peer_address,
log.warning("%s started sending blob we didn't request %s instead of %s", self.peer_address,
blob_response.blob_hash, self.blob.blob_hash)
return
if response.responses:
@@ -83,8 +83,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
else:
data = data
self._blob_bytes_received += len(data)
try:
self.writer.write(data)
@@ -141,7 +139,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
log.warning("blob can't be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
+13 -3
View File
@@ -148,7 +148,12 @@ class MaxKeyFee(Setting[dict]):
@staticmethod
def _parse_list(l):
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
if l == ['null']:
return None
assert len(l) == 2, (
'Max key fee is made up of either two values: '
'"AMOUNT CURRENCY", or "null" (to set no limit)'
)
try:
amount = float(l[0])
except ValueError:
@@ -176,7 +181,7 @@ class MaxKeyFee(Setting[dict]):
parser.add_argument(
self.cli_name,
help=self.doc,
nargs=2,
nargs='+',
metavar=('AMOUNT', 'CURRENCY'),
default=NOT_SET
)
@@ -480,6 +485,10 @@ class Config(CLIConfig):
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
wallets = Strings(
"Wallet files in 'wallet_dir' to load at startup.",
['default_wallet']
)
# network
use_upnp = Toggle(
@@ -527,7 +536,8 @@ class Config(CLIConfig):
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
"Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
)
# reflector settings
+10 -12
View File
@@ -19,8 +19,6 @@ class ConnectionManager:
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._total_sent = 0
self._total_received = 0
self._running = False
self._task: typing.Optional[asyncio.Task] = None
@@ -58,29 +56,29 @@ class ConnectionManager:
'outgoing_bps': {},
'total_incoming_mbs': 0.0,
'total_outgoing_mbs': 0.0,
'total_sent': self._total_sent,
'total_received': self._total_received,
'total_sent': 0,
'total_received': 0,
'time': self.loop.time()
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
await asyncio.sleep(5, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
now = self.loop.time()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
self._status['outgoing_bps'][k] = v / (now - last)
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['incoming_bps'][k] = v / (now - last)
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
) / (now - last)) / 1000000.0
)) / 1000000.0
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
) / (now - last)) / 1000000.0
self._total_sent += sum(list(self._status['outgoing_bps'].values()))
self._total_received += sum(list(self._status['incoming_bps'].values()))
)) / 1000000.0
self._status['total_sent'] += sum(list(self._status['outgoing_bps'].values()))
self._status['total_received'] += sum(list(self._status['incoming_bps'].values()))
self._status['time'] = now
+5 -4
View File
@@ -4,13 +4,14 @@ import typing
import binascii
from lbry.utils import resolve_host
from lbry.dht import constants
from lbry.dht.peer import make_kademlia_peer
from lbry.dht.protocol.distance import Distance
from lbry.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
from lbry.dht.protocol.protocol import KademliaProtocol
from lbry.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbry.dht.peer import PeerManager
from lbry.dht.peer import KademliaPeer
log = logging.getLogger(__name__)
@@ -36,7 +37,7 @@ class Node:
total_peers: typing.List['KademliaPeer'] = []
# add all peers in the routing table
total_peers.extend(self.protocol.routing_table.get_peers())
# add all the peers who have announed blobs to us
# add all the peers who have announced blobs to us
total_peers.extend(self.protocol.data_store.get_storing_contacts())
# get ids falling in the midpoint of each bucket that hasn't been recently updated
@@ -141,7 +142,7 @@ class Node:
if known_node_addresses:
peers = [
KademliaPeer(self.loop, address, udp_port=port)
make_kademlia_peer(None, address, port)
for (address, port) in known_node_addresses
]
while True:
@@ -232,7 +233,7 @@ class Node:
if not peer.udp_port:
udp_port_to_try = peer.tcp_port
if not peer.udp_port:
peer.update_udp_port(udp_port_to_try)
peer = make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
self.loop.create_task(ping(peer))
else:
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
+28 -46
View File
@@ -3,6 +3,7 @@ import asyncio
import logging
import ipaddress
from binascii import hexlify
from dataclasses import dataclass, field
from functools import lru_cache
from lbry.dht import constants
@@ -11,6 +12,13 @@ from lbry.dht.serialization.datagram import make_compact_address, make_compact_i
log = logging.getLogger(__name__)
@lru_cache(1024)
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
udp_port: typing.Optional[int] = None,
tcp_port: typing.Optional[int] = None) -> 'KademliaPeer':
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port)
def is_valid_ipv4(address):
try:
ip = ipaddress.ip_address(address)
@@ -81,10 +89,6 @@ class PeerManager:
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
@lru_cache(maxsize=400)
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
return KademliaPeer(self._loop, address, node_id, udp_port)
def prune(self): # TODO: periodically call this
now = self._loop.time()
to_pop = []
@@ -108,14 +112,16 @@ class PeerManager:
delay = self._loop.time() - constants.check_refresh_interval
# fixme: find a way to re-enable that without breaking other parts
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
# if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
# return
#addr_tup = (address, udp_port)
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
# addr_tup = (address, udp_port)
# if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
# return
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
last_requested = self._last_requested.get((address, udp_port))
last_replied = self._last_replied.get((address, udp_port))
if node_id is None:
return None
if most_recent_failure and last_replied:
if delay < last_replied > most_recent_failure:
return True
@@ -135,47 +141,31 @@ class PeerManager:
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
node_id, address, tcp_port = decode_compact_address(compact_address)
return KademliaPeer(self._loop, address, node_id, tcp_port=tcp_port)
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
@dataclass(unsafe_hash=True)
class KademliaPeer:
__slots__ = [
'loop',
'_node_id',
'address',
'udp_port',
'tcp_port',
'protocol_version',
]
address: str = field(hash=True)
_node_id: typing.Optional[bytes] = field(hash=True)
udp_port: typing.Optional[int] = field(hash=True)
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
def __init__(self, loop: asyncio.AbstractEventLoop, address: str, node_id: typing.Optional[bytes] = None,
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
if node_id is not None:
if not len(node_id) == constants.hash_length:
raise ValueError("invalid node_id: {}".format(hexlify(node_id).decode()))
if udp_port is not None and not 0 <= udp_port <= 65536:
def __post_init__(self):
if self._node_id is not None:
if not len(self._node_id) == constants.hash_length:
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
raise ValueError("invalid udp port")
if tcp_port and not 0 <= tcp_port <= 65536:
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
raise ValueError("invalid tcp port")
if not is_valid_ipv4(address):
raise ValueError("invalid ip address")
self.loop = loop
self._node_id = node_id
self.address = address
self.udp_port = udp_port
self.tcp_port = tcp_port
self.protocol_version = 1
if not is_valid_ipv4(self.address):
raise ValueError(f"invalid ip address: '{self.address}'")
def update_tcp_port(self, tcp_port: int):
self.tcp_port = tcp_port
def update_udp_port(self, udp_port: int):
self.udp_port = udp_port
def set_id(self, node_id):
if not self._node_id:
self._node_id = node_id
@property
def node_id(self) -> bytes:
return self._node_id
@@ -188,11 +178,3 @@ class KademliaPeer:
def compact_ip(self):
return make_compact_ip(self.address)
def __eq__(self, other):
if not isinstance(other, KademliaPeer):
raise TypeError("invalid type to compare with Peer: %s" % str(type(other)))
return (self.node_id, self.address, self.udp_port) == (other.node_id, other.address, other.udp_port)
def __hash__(self):
return hash((self.node_id, self.address, self.udp_port))
+2 -1
View File
@@ -7,6 +7,7 @@ import logging
from lbry.dht import constants
from lbry.dht.error import RemoteException, TransportNotConnected
from lbry.dht.protocol.distance import Distance
from lbry.dht.peer import make_kademlia_peer
from typing import TYPE_CHECKING
if TYPE_CHECKING:
@@ -150,7 +151,7 @@ class IterativeFinder:
self._add_active(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
self._add_active(make_kademlia_peer(node_id, address, udp_port))
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
+18 -33
View File
@@ -15,6 +15,7 @@ from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
from lbry.dht.error import RemoteException, TransportNotConnected
from lbry.dht.protocol.routing_table import TreeRoutingTable
from lbry.dht.protocol.data_store import DictDataStore
from lbry.dht.peer import make_kademlia_peer
if typing.TYPE_CHECKING:
from lbry.dht.peer import PeerManager, KademliaPeer
@@ -322,6 +323,9 @@ class KademliaProtocol(DatagramProtocol):
return args, {}
async def _add_peer(self, peer: 'KademliaPeer'):
if not peer.node_id:
log.warning("Tried adding a peer with no node id!")
return False
for p in self.routing_table.get_peers():
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
self.routing_table.remove_peer(p)
@@ -447,7 +451,7 @@ class KademliaProtocol(DatagramProtocol):
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
@@ -494,8 +498,7 @@ class KademliaProtocol(DatagramProtocol):
elif response_datagram.node_id == self.node_id:
df.set_exception(RemoteException("incoming message is from our node id"))
return
peer.set_id(response_datagram.node_id)
peer.update_udp_port(address[1])
peer = make_kademlia_peer(response_datagram.node_id, address[0], address[1])
self.peer_manager.report_last_replied(address[0], address[1])
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
@@ -529,7 +532,7 @@ class KademliaProtocol(DatagramProtocol):
elif error_datagram.response not in old_protocol_errors:
log.warning(error_msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
df.set_exception(remote_exception)
return
@@ -539,7 +542,7 @@ class KademliaProtocol(DatagramProtocol):
f"pending request: {str(remote_exception)}"
log.warning(msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
address[0], address[1], old_protocol_errors[error_datagram.response])
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
@@ -643,7 +646,8 @@ class KademliaProtocol(DatagramProtocol):
return False
return True
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer',
retry: bool = True) -> typing.Tuple[bytes, bool]:
async def __store():
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
@@ -655,34 +659,15 @@ class KademliaProtocol(DatagramProtocol):
return await __store()
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, False
except ValueError as err:
log.error("Unexpected response: %s" % err)
return peer.node_id, False
except RemoteException as err:
if 'Invalid token' in str(err):
self.peer_manager.clear_token(peer.node_id)
try:
return await __store()
except (ValueError, asyncio.TimeoutError, RemoteException):
return peer.node_id, False
else:
if 'Invalid token' not in str(err):
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
def _write(self, data: bytes, address: typing.Tuple[str, int]):
if self.transport:
try:
self.transport.sendto(data, address)
except OSError as err:
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
# elif err.errno == socket.ENETUNREACH:
# # this should probably try to retransmit when the network connection is back
# log.error("Network is unreachable")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), address[0], address[1], str(err), err.errno)
raise err
else:
raise TransportNotConnected()
return peer.node_id, False
self.peer_manager.clear_token(peer.node_id)
if not retry:
return peer.node_id, False
return await self.store_to_peer(hash_value, peer, retry=False)
+16 -11
View File
@@ -45,7 +45,7 @@ async def execute_command(conf, method, params, callback=display):
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
@@ -65,10 +65,10 @@ def remove_brackets(key):
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
k = None
if arg is None:
continue
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = None
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
@@ -189,7 +189,7 @@ def get_argument_parser():
)
start.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
Config.contribute_to_argparse(start)
@@ -221,7 +221,7 @@ def ensure_directory_exists(path: str):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
def run_daemon(args: list, conf: Config):
def setup_logging(args: argparse.Namespace, conf: Config, loop: asyncio.AbstractEventLoop):
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
file_handler = logging.handlers.RotatingFileHandler(
conf.log_file_path, maxBytes=2097152, backupCount=5
@@ -240,19 +240,24 @@ def run_daemon(args: list, conf: Config):
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
loop = asyncio.get_event_loop()
if args.verbose:
log.setLevel(logging.DEBUG)
log.setLevel(logging.INFO)
if args.verbose is not None:
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if len(args.verbose) > 0:
for module in args.verbose:
logging.getLogger(module).setLevel(logging.DEBUG)
else:
log.setLevel(logging.DEBUG)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config):
loop = asyncio.get_event_loop()
setup_logging(args, conf, loop)
daemon = Daemon(conf)
def __exit():
-2
View File
@@ -252,8 +252,6 @@ class WalletComponent(Component):
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
'best_blockhash': best_hash,
'is_encrypted': self.wallet_manager.use_encryption,
'is_locked': not self.wallet_manager.is_wallet_unlocked,
}
async def start(self):
File diff suppressed because it is too large. Load diff
@@ -6,6 +6,8 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError
from torba.client.wallet import Wallet
from torba.client.bip32 import PubKey
from lbry.schema.claim import Claim
from lbry.wallet.ledger import MainNetLedger, Account
from lbry.wallet.transaction import Transaction, Output
@@ -67,6 +69,13 @@ def encode_account_doc():
}
def encode_wallet_doc():
return {
'id': 'wallet_id',
'name': 'optional wallet name',
}
def encode_file_doc():
return {
'streaming_url': '(str) url to stream the file using range requests',
@@ -110,6 +119,8 @@ class JSONResponseEncoder(JSONEncoder):
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Account):
return self.encode_account(obj)
if isinstance(obj, Wallet):
return self.encode_wallet(obj)
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, Transaction):
@@ -118,6 +129,8 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_output(obj)
if isinstance(obj, Claim):
return self.encode_claim(obj)
if isinstance(obj, PubKey):
return obj.extended_key_string()
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -219,6 +232,12 @@ class JSONResponseEncoder(JSONEncoder):
result['is_default'] = self.ledger.accounts[0] == account
return result
def encode_wallet(self, wallet):
return {
'id': wallet.id,
'name': wallet.name
}
def encode_file(self, managed_stream):
output_exists = managed_stream.output_file_exists
tx_height = managed_stream.stream_claim_info.height
+11 -4
View File
@@ -3,6 +3,7 @@ from aiohttp.client_exceptions import ClientError
import json
import logging.handlers
import traceback
import aiohttp
from lbry import utils, __version__
@@ -42,6 +43,8 @@ class HTTPSLogglyHandler(logging.Handler):
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
def get_full_message(self, record):
if record.exc_info:
@@ -49,13 +52,17 @@ class HTTPSLogglyHandler(logging.Handler):
else:
return record.getMessage()
async def _emit(self, record):
async def _emit(self, record, retry=True):
data = self.format(record).encode()
try:
async with utils.aiohttp_request('post', self.url, data=self.format(record).encode(),
cookies=self.cookies) as response:
async with self._session.post(self.url, data=data,
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
pass
if self._loop.is_running() and retry:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
asyncio.ensure_future(self._emit(record))
@@ -1,4 +1,5 @@
import os
import sys
import logging
log = logging.getLogger(__name__)
@@ -51,7 +52,6 @@ def migrate_db(conf, start, end):
def run_migration_script():
import sys
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
sys.stdout = open("migrator.out.log", 'w')
@@ -125,10 +125,9 @@ def do_migration(conf):
name_metadata_cursor.execute("select txid, n, sd_hash from name_metadata").fetchall()
}
sd_hash_to_stream_hash = {
sd_hash: stream_hash for (sd_hash, stream_hash) in
sd_hash_to_stream_hash = dict(
lbryfile_cursor.execute("select sd_blob_hash, stream_hash from lbry_file_descriptors").fetchall()
}
)
stream_hash_to_stream_blobs = {}
+67 -67
View File
@@ -101,6 +101,9 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
stream_hashes = tuple(
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
)
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
@@ -108,9 +111,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", [
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
"order by c.rowid desc", stream_hashes):
claim = StoredStreamClaim(stream_hash, *claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -137,7 +138,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
)
for claim_name, claim_id in _batched_select(
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
list(signed_claims.keys())):
tuple(signed_claims.keys())):
for claim in signed_claims[claim_id]:
claim.channel_name = claim_name
return files
@@ -147,35 +148,35 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
# add all blobs, except the last one, which is empty
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
)
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
).fetchall()
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs)
).fetchall()
# ensure should_announce is set regardless if insert was ignored
transaction.execute(
"update blob set should_announce=1 where blob_hash in (?, ?)",
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
)
).fetchall()
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
blob_hashes.append((descriptor.sd_hash, ))
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
@@ -191,7 +192,7 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
)
).fetchall()
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
@@ -293,17 +294,17 @@ class SQLiteStorage(SQLiteMixin):
def _add_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[
(
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
for blob_hash, length in blob_hashes_and_lengths
]
)
)
).fetchall()
if finished:
transaction.executemany(
"update blob set status='finished' where blob.blob_hash=?", [
"update blob set status='finished' where blob.blob_hash=?", (
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
]
)
)
).fetchall()
return await self.db.run(_add_blobs)
def get_blob_status(self, blob_hash: str):
@@ -317,9 +318,9 @@ class SQLiteStorage(SQLiteMixin):
return transaction.executemany(
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
"where blob_hash=?",
[(int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
for blob_hash in blob_hashes]
)
((int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
for blob_hash in blob_hashes)
).fetchall()
return self.db.run(_update_last_announced_blobs)
def should_single_announce_blobs(self, blob_hashes, immediate=False):
@@ -330,11 +331,11 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute(
"update blob set single_announce=1, next_announce_time=? "
"where blob_hash=? and status='finished'", (int(now), blob_hash)
)
).fetchall()
else:
transaction.execute(
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
)
).fetchall()
return self.db.run(set_single_announce)
def get_blobs_to_announce(self):
@@ -347,22 +348,22 @@ class SQLiteStorage(SQLiteMixin):
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
).fetchall()
else:
r = transaction.execute(
"select blob_hash from blob where blob_hash is not null "
"and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
return [b[0] for b in r.fetchall()]
).fetchall()
return [b[0] for b in r]
return self.db.run(get_and_update)
def delete_blobs_from_db(self, blob_hashes):
def delete_blobs(transaction):
transaction.executemany(
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
)
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
).fetchall()
return self.db.run_with_foreign_keys_disabled(delete_blobs)
def get_all_blob_hashes(self):
@@ -370,22 +371,18 @@ class SQLiteStorage(SQLiteMixin):
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
to_update = [
(blob_hash, )
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
if blob_hash not in blob_files
]
finished_blob_hashes = tuple(
blob_hash for (blob_hash, ) in transaction.execute(
"select blob_hash from blob where status='finished'"
).fetchall()
)
finished_blobs_set = set(finished_blob_hashes)
to_update_set = finished_blobs_set.difference(blob_files)
transaction.executemany(
"update blob set status='pending' where blob_hash=?",
to_update
)
return {
blob_hash
for blob_hash, in _batched_select(
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
list(blob_files)
)
}
((blob_hash, ) for blob_hash in to_update_set)
).fetchall()
return blob_files.intersection(finished_blobs_set)
return self.db.run(_sync_blobs)
# # # # # # # # # stream functions # # # # # # # # #
@@ -478,13 +475,12 @@ class SQLiteStorage(SQLiteMixin):
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
else:
removed.append((stream_hash,))
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
)
).fetchall()
return await self.db.run(update_manually_removed_files)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
@@ -492,7 +488,7 @@ class SQLiteStorage(SQLiteMixin):
def change_file_status(self, stream_hash: str, new_status: str):
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
file_name: typing.Optional[str]):
@@ -501,22 +497,22 @@ class SQLiteStorage(SQLiteMixin):
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
encoded_download_dir, encoded_file_name, stream_hash,
))
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
binascii.hexlify(content_fee.raw), stream_hash,
))
async def set_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
stream_hash,
))
async def clear_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
stream_hash,
))
@@ -537,13 +533,13 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
((stream_hash, ) for stream_hash in stream_hashes)
).fetchall()
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
transaction.executemany(
f"update file set download_directory=? where stream_hash=?",
[(download_dir, stream_hash) for stream_hash in stream_hashes]
)
((download_dir, stream_hash) for stream_hash in stream_hashes)
).fetchall()
await self.db.run_with_foreign_keys_disabled(_recover)
def get_all_stream_hashes(self):
@@ -555,14 +551,16 @@ class SQLiteStorage(SQLiteMixin):
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
def _save_support(transaction):
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
transaction.execute(
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
).fetchall()
for claim_id, supports in claim_id_to_supports.items():
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
)
).fetchall()
return self.db.run(_save_support)
def get_supports(self, *claim_ids):
@@ -581,7 +579,7 @@ class SQLiteStorage(SQLiteMixin):
for support_info in _batched_select(
transaction,
"select * from support where claim_id in {}",
tuple(claim_ids)
claim_ids
)
]
@@ -612,7 +610,7 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute(
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
)
).fetchall()
# if this response doesn't have support info don't overwrite the existing
# support info
if 'supports' in claim_info:
@@ -699,7 +697,9 @@ class SQLiteStorage(SQLiteMixin):
)
# update the claim associated to the file
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
transaction.execute(
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
).fetchall()
async def save_content_claim(self, stream_hash, claim_outpoint):
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
@@ -722,11 +722,11 @@ class SQLiteStorage(SQLiteMixin):
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
if success:
return self.db.execute(
return self.db.execute_fetchall(
"insert or replace into reflected_stream values (?, ?, ?)",
(sd_hash, reflector_address, self.time_getter())
)
return self.db.execute(
return self.db.execute_fetchall(
"delete from reflected_stream where sd_hash=? and reflector_address=?",
(sd_hash, reflector_address)
)
+1 -1
View File
@@ -23,7 +23,7 @@ def get_platform() -> dict:
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
import distro
import distro # pylint: disable=import-outside-toplevel
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
+3
View File
@@ -139,12 +139,15 @@ types_map = {
# These are non-standard types, commonly found in the wild.
'.cbr': ('application/vnd.comicbook-rar', 'document'),
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.flac': ('audio/flac', 'audio'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
'.mkv': ('video/x-matroska', 'video'),
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
+5 -2
View File
@@ -4,7 +4,10 @@ from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.com/
# see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
@@ -17,7 +20,7 @@ def _create_url_regex():
def _claim(name, prefix=""):
return _group(
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
_named(name+"_name", prefix + invalid_names_regex) +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
+2 -2
View File
@@ -71,7 +71,7 @@ class StreamDescriptor:
self.sd_hash = sd_hash
@property
def length(self):
def length(self) -> int:
return len(self.as_json())
def get_stream_hash(self) -> str:
@@ -114,7 +114,7 @@ class StreamDescriptor:
])
).encode()
def calculate_old_sort_sd_hash(self):
def calculate_old_sort_sd_hash(self) -> str:
h = get_lbry_hash_obj()
h.update(self.old_sort_json())
return h.hexdigest()
+3 -2
View File
@@ -2,11 +2,12 @@ import asyncio
import typing
import logging
import binascii
from lbry.dht.peer import make_kademlia_peer
from lbry.error import DownloadSDTimeout
from lbry.utils import resolve_host, lru_cache_concurrent
from lbry.stream.descriptor import StreamDescriptor
from lbry.blob_exchange.downloader import BlobDownloader
from lbry.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.dht.node import Node
@@ -50,7 +51,7 @@ class StreamDownloader:
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
make_kademlia_peer(None, address, None, tcp_port=port + 1)
for address, port in addresses
])
+12 -4
View File
@@ -11,6 +11,8 @@ from lbry.stream.downloader import StreamDownloader
from lbry.stream.descriptor import StreamDescriptor
from lbry.stream.reflector.client import StreamReflectorClient
from lbry.extras.daemon.storage import StoredStreamClaim
from lbry.blob import MAX_BLOB_SIZE
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.schema.claim import Claim
@@ -215,6 +217,9 @@ class ManagedStream:
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
"""
Generate a stream from a file and save it to the db
"""
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=blob_manager.blob_completed
@@ -289,13 +294,13 @@ class ManagedStream:
await response.prepare(request)
self.streaming_responses.append((request, response))
self.streaming.set()
wrote = 0
try:
wrote = 0
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
if not wrote:
decrypted = decrypted[first_blob_start_offset:]
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * (MAX_BLOB_SIZE - 1))))
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
len(self.descriptor.blobs) - 1)
await response.write_eof(decrypted)
@@ -308,6 +313,9 @@ class ManagedStream:
if response._eof_sent:
break
return response
except ConnectionResetError:
log.warning("connection was reset after sending browser %i blob bytes", wrote)
raise asyncio.CancelledError("range request transport was reset")
finally:
response.force_close()
if (request, response) in self.streaming_responses:
@@ -483,8 +491,8 @@ class ManagedStream:
if end >= size:
raise HTTPRequestRangeNotSatisfiable()
skip_blobs = start // 2097150
skip = skip_blobs * 2097151
skip_blobs = start // (MAX_BLOB_SIZE - 2) # -2 because ... dont remember
skip = skip_blobs * (MAX_BLOB_SIZE - 1) # -1 because
skip_first_blob = start - skip
start = skip_first_blob + skip
final_size = end - start + 1
+3 -3
View File
@@ -401,8 +401,8 @@ class StreamManager:
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
), 5) if self.config.max_key_fee else None
if max_fee_amount and fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
@@ -442,7 +442,7 @@ class StreamManager:
except asyncio.TimeoutError:
error = DownloadDataTimeout(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, dont delete stream
except Exception as err: # forgive data timeout, don't delete stream
error = err
raise
finally:
+3 -3
View File
@@ -151,11 +151,11 @@ class CommandTestCase(IntegrationTestCase):
wallet_node.manager.old_db = daemon.storage
return daemon
async def confirm_tx(self, txid):
async def confirm_tx(self, txid, ledger=None):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
await self.on_transaction_id(txid)
await self.on_transaction_id(txid, ledger)
await self.generate(1)
await self.on_transaction_id(txid)
await self.on_transaction_id(txid, ledger)
return txid
async def on_transaction_dict(self, tx):
+2 -2
View File
@@ -19,7 +19,6 @@ import collections
from lbry.schema.claim import Claim
from lbry.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
@@ -192,7 +191,8 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
return (await loop.getaddrinfo(
url, port,
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM,
family=socket.AF_INET
))[0][4][0]
+22 -30
View File
@@ -5,7 +5,6 @@ from hashlib import sha256
from string import hexdigits
import ecdsa
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.constants import TXO_TYPES
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
@@ -28,19 +27,17 @@ class Account(BaseAccount):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.channel_keys = {}
self.preferences = {}
@property
def hash(self) -> bytes:
h = sha256(json.dumps(self.to_dict(False)).encode())
h = sha256(json.dumps(self.to_dict(include_channel_keys=False)).encode())
for cert in sorted(self.channel_keys.keys()):
h.update(cert.encode())
return h.digest()
def apply(self, d: dict):
super().apply(d)
def merge(self, d: dict):
super().merge(d)
self.channel_keys.update(d.get('certificates', {}))
self.preferences.update(d.get('preferences', {}))
def add_channel_private_key(self, private_key):
public_key_bytes = private_key.get_verifying_key().to_der()
@@ -80,7 +77,7 @@ class Account(BaseAccount):
constraints.update({'txo_type': 0})
return super().get_balance(confirmations, **constraints)
async def get_granular_balances(self, confirmations=0, reserved_subtotals=False):
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
total = await get_total_balance()
@@ -100,13 +97,13 @@ class Account(BaseAccount):
confirmations=confirmations, include_claims=True, txo_type__gt=0
)
return {
'total': dewies_to_lbc(total),
'available': dewies_to_lbc(total - reserved),
'reserved': dewies_to_lbc(reserved),
'total': total,
'available': total - reserved,
'reserved': reserved,
'reserved_subtotals': {
'claims': dewies_to_lbc(claims_balance),
'supports': dewies_to_lbc(supports_balance),
'tips': dewies_to_lbc(tips_balance)
'claims': claims_balance,
'supports': supports_balance,
'tips': tips_balance
} if reserved_subtotals else None
}
@@ -120,53 +117,48 @@ class Account(BaseAccount):
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
account = super().from_dict(ledger, wallet, d)
account.channel_keys = d.get('certificates', {})
account.preferences = d.get('preferences', {})
return account
def to_dict(self, include_channel_keys=True, include_preferences=True):
d = super().to_dict()
def to_dict(self, encrypt_password: str = None, include_channel_keys: bool = True):
d = super().to_dict(encrypt_password)
if include_channel_keys:
d['certificates'] = self.channel_keys
if include_preferences and self.preferences:
d['preferences'] = self.preferences
return d
async def get_details(self, **kwargs):
details = await super().get_details(**kwargs)
details['certificates'] = len(self.channel_keys)
if self.preferences:
details['preferences'] = self.preferences
return details
def get_transaction_history(self, **constraints):
return self.ledger.get_transaction_history(account=self, **constraints)
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
def get_transaction_history_count(self, **constraints):
return self.ledger.get_transaction_history_count(account=self, **constraints)
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
def get_claims(self, **constraints):
return self.ledger.get_claims(account=self, **constraints)
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
def get_claim_count(self, **constraints):
return self.ledger.get_claim_count(account=self, **constraints)
return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints)
def get_streams(self, **constraints):
return self.ledger.get_streams(account=self, **constraints)
return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints)
def get_stream_count(self, **constraints):
return self.ledger.get_stream_count(account=self, **constraints)
return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints)
def get_channels(self, **constraints):
return self.ledger.get_channels(account=self, **constraints)
return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints)
def get_channel_count(self, **constraints):
return self.ledger.get_channel_count(account=self, **constraints)
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
def get_supports(self, **constraints):
return self.ledger.get_supports(account=self, **constraints)
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
def get_support_count(self, **constraints):
return self.ledger.get_support_count(account=self, **constraints)
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
def get_support_summary(self):
return self.ledger.db.get_supports_summary(account_id=self.id)
+17 -19
View File
@@ -22,18 +22,18 @@ class WalletDatabase(BaseDatabase):
claim_id text,
claim_name text
);
create index if not exists txo_address_idx on txo (address);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_txo_type_idx on txo (txo_type);
"""
CREATE_TABLES_QUERY = (
BaseDatabase.CREATE_TX_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_INDEX +
CREATE_TXO_TABLE +
BaseDatabase.CREATE_TXO_INDEX +
BaseDatabase.CREATE_TXI_TABLE +
BaseDatabase.CREATE_TXI_INDEX
BaseDatabase.PRAGMAS +
BaseDatabase.CREATE_ACCOUNT_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
BaseDatabase.CREATE_TX_TABLE +
CREATE_TXO_TABLE +
BaseDatabase.CREATE_TXI_TABLE
)
def txo_to_row(self, tx, address, txo):
@@ -50,18 +50,16 @@ class WalletDatabase(BaseDatabase):
row['claim_name'] = txo.claim_name
return row
async def get_txos(self, **constraints) -> List[Output]:
my_accounts = constraints.get('my_accounts', constraints.get('accounts', []))
txos = await super().get_txos(**constraints)
async def get_txos(self, wallet=None, no_tx=False, **constraints) -> List[Output]:
txos = await super().get_txos(wallet=wallet, no_tx=no_tx, **constraints)
channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and my_accounts:
for account in my_accounts:
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
@@ -73,7 +71,7 @@ class WalletDatabase(BaseDatabase):
channels = {
txo.claim_id: txo for txo in
(await self.get_claims(
my_accounts=my_accounts,
wallet=wallet,
claim_id__in=channel_ids
))
}
@@ -134,20 +132,20 @@ class WalletDatabase(BaseDatabase):
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account):
await self.db.execute(
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from pubkey_address WHERE account = ?"
" )", [account.public_key.address]
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
def get_supports_summary(self, account_id):
return self.db.execute_fetchall(f"""
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
(txo.txid in
(select txi.txid from txi join pubkey_address a on txi.address = a.address
(select txi.txid from txi join account_address a on txi.address = a.address
where a.account = ?)) as from_me,
(txo.address in (select address from pubkey_address where account=?)) as to_me,
(txo.address in (select address from account_address where account=?)) as to_me,
tx.height
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
""", (account_id, account_id))
+9
View File
@@ -31,3 +31,12 @@ def lbc_to_dewies(lbc: str) -> int:
def dewies_to_lbc(dewies) -> str:
return satoshis_to_coins(dewies)
def dict_values_to_lbc(d):
for key, value in d.items():
if isinstance(value, int):
d[key] = dewies_to_lbc(value)
elif isinstance(value, dict):
dict_values_to_lbc(value)
return d
+20 -11
View File
@@ -1,6 +1,7 @@
import asyncio
import logging
from binascii import unhexlify
from functools import partial
from typing import Tuple, List
from datetime import datetime
@@ -60,7 +61,8 @@ class MainNetLedger(BaseLedger):
return outputs.inflate(txs), outputs.offset, outputs.total
async def resolve(self, urls):
txos = (await self._inflate_outputs(self.network.resolve(urls)))[0]
resolve = partial(self.network.retriable_call, self.network.resolve)
txos = (await self._inflate_outputs(resolve(urls)))[0]
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -121,39 +123,30 @@ class MainNetLedger(BaseLedger):
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_claims(**constraints)
def get_claim_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_claim_count(**constraints)
def get_streams(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_streams(**constraints)
def get_stream_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_stream_count(**constraints)
def get_channels(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_channels(**constraints)
def get_channel_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_channel_count(**constraints)
def get_supports(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_supports(**constraints)
def get_support_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_support_count(**constraints)
async def get_transaction_history(self, **constraints):
self.constraint_account_or_all(constraints)
txs = await self.db.get_transactions(**constraints)
headers = self.headers
history = []
@@ -249,9 +242,25 @@ class MainNetLedger(BaseLedger):
return history
def get_transaction_history_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transaction_count(**constraints)
@staticmethod
async def get_detailed_balance(accounts, confirmations=0, reserved_subtotals=False):
result = {}
for account in accounts:
balance = await account.get_detailed_balance(confirmations, reserved_subtotals)
if result:
for key, value in balance.items():
if key == 'reserved_subtotals':
if value is not None:
for subkey, subvalue in value.items():
result['reserved_subtotals'][subkey] += subvalue
else:
result[key] += value
else:
result = balance
return result
class TestNetLedger(MainNetLedger):
network_name = 'testnet'
+16 -47
View File
@@ -5,6 +5,7 @@ from binascii import unhexlify
from torba.client.basemanager import BaseWalletManager
from torba.client.wallet import ENCRYPT_ON_DISK
from torba.rpc.jsonrpc import CodeMessageError
from lbry.wallet.ledger import MainNetLedger
@@ -26,42 +27,8 @@ class LbryWalletManager(BaseWalletManager):
def db(self) -> WalletDatabase:
return self.ledger.db
@property
def use_encryption(self):
return self.default_account.serialize_encrypted
@property
def is_wallet_unlocked(self):
return not self.default_account.encrypted
def check_locked(self):
return self.default_account.encrypted
def decrypt_account(self, account):
assert account.password is not None, "account is not unlocked"
assert not account.encrypted, "account is not unlocked"
account.serialize_encrypted = False
self.save()
return not account.encrypted and not account.serialize_encrypted
def encrypt_account(self, password, account):
assert not account.encrypted, "account is already encrypted"
account.encrypt(password)
account.serialize_encrypted = True
self.save()
self.unlock_account(password, account)
return account.serialize_encrypted
def unlock_account(self, password, account):
assert account.encrypted, "account is not locked"
account.decrypt(password)
return not account.encrypted
def lock_account(self, account):
assert account.password is not None, "account is already locked"
assert not account.encrypted and account.serialize_encrypted, "account is not encrypted"
account.encrypt(account.password)
return account.encrypted
return self.default_wallet.is_locked
@staticmethod
def migrate_lbryum_to_torba(path):
@@ -138,20 +105,26 @@ class LbryWalletManager(BaseWalletManager):
if not os.path.exists(wallets_directory):
os.mkdir(wallets_directory)
wallet_file_path = os.path.join(wallets_directory, 'default_wallet')
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(wallet_file_path)
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(
os.path.join(wallets_directory, 'default_wallet')
)
manager = cls.from_config({
'ledgers': {ledger_id: ledger_config},
'wallets': [wallet_file_path]
'wallets': [
os.path.join(wallets_directory, wallet_file) for wallet_file in settings.wallets
]
})
ledger = manager.get_or_create_ledger(ledger_id)
ledger.coin_selection_strategy = settings.coin_selection_strategy
if manager.default_account is None:
log.info('Wallet at %s is empty, generating a default account.', wallet_file_path)
manager.default_wallet.generate_account(ledger)
manager.default_wallet.save()
default_wallet = manager.default_wallet
if default_wallet.default_account is None:
log.info('Wallet at %s is empty, generating a default account.', default_wallet.id)
default_wallet.generate_account(ledger)
default_wallet.save()
if default_wallet.is_locked and default_wallet.preferences.get(ENCRYPT_ON_DISK) is None:
default_wallet.preferences[ENCRYPT_ON_DISK] = True
default_wallet.save()
if receiving_addresses or change_addresses:
if not os.path.exists(ledger.path):
os.mkdir(ledger.path)
@@ -207,7 +180,3 @@ class LbryWalletManager(BaseWalletManager):
tx = self.ledger.transaction_class(unhexlify(raw))
await self.ledger.maybe_verify_transaction(tx, height)
return tx
def save(self):
for wallet in self.wallets:
wallet.save()
+1 -1
View File
@@ -79,7 +79,7 @@ class LBC(Coin):
@classmethod
def address_from_script(cls, script):
'''Given a pk_script, return the adddress it pays to, or None.'''
'''Given a pk_script, return the address it pays to, or None.'''
return ScriptPubKey.pay_to(cls.address_handlers, script)
@classmethod
+2 -1
View File
@@ -2,6 +2,7 @@ import time
import struct
import sqlite3
import logging
from operator import itemgetter
from typing import Tuple, List, Dict, Union, Type, Optional
from binascii import unhexlify
from decimal import Decimal
@@ -336,7 +337,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
txo_rows = _search(**constraints)
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(
+12 -8
View File
@@ -1,7 +1,11 @@
from math import sqrt
TRENDING_WINDOW = 650 # number of blocks, ~24hr period
TRENDING_DATA_POINTS = 7 # WINDOW * DATA_POINTS = ~1 week worth of trending data
# TRENDING_WINDOW is the number of blocks in ~6hr period (21600 seconds / 161 seconds per block)
TRENDING_WINDOW = 134
# TRENDING_DATA_POINTS says how many samples to use for the trending algorithm
# i.e. only consider claims from the most recent (TRENDING_WINDOW * TRENDING_DATA_POINTS) blocks
TRENDING_DATA_POINTS = 7
CREATE_TREND_TABLE = """
create table if not exists trend (
@@ -26,7 +30,7 @@ class ZScore:
if self.last is not None:
self.count += 1
self.total += self.last
self.power += self.last**2
self.power += self.last ** 2
self.last = value
@property
@@ -35,7 +39,7 @@ class ZScore:
@property
def standard_deviation(self):
return sqrt((self.power / self.count) - self.mean**2)
return sqrt((self.power / self.count) - self.mean ** 2)
def finalize(self):
if self.count == 0:
@@ -47,19 +51,19 @@ def register_trending_functions(connection):
connection.create_aggregate("zscore", 1, ZScore)
def calculate_trending(db, height, is_first_sync, final_height):
def calculate_trending(db, height, final_height):
# don't start tracking until we're at the end of initial sync
if is_first_sync and height < (final_height - (TRENDING_WINDOW*TRENDING_DATA_POINTS)):
if height < (final_height - (TRENDING_WINDOW * TRENDING_DATA_POINTS)):
return
if height % TRENDING_WINDOW != 0:
return
db.execute(f"""
DELETE FROM trend WHERE height < {height-(TRENDING_WINDOW*TRENDING_DATA_POINTS)}
DELETE FROM trend WHERE height < {height - (TRENDING_WINDOW * TRENDING_DATA_POINTS)}
""")
start = (height-TRENDING_WINDOW)+1
start = (height - TRENDING_WINDOW) + 1
db.execute(f"""
INSERT OR IGNORE INTO trend (claim_hash, height, amount)
SELECT claim_hash, {start}, COALESCE(
+1 -1
View File
@@ -699,7 +699,7 @@ class SQLDB:
update_claims, delete_claim_hashes, affected_channels, forward_timer=True)
r(self.insert_supports, insert_supports)
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
r(calculate_trending, self.db, height, daemon_height)
class LBRYDB(DB):
+5 -5
View File
@@ -203,7 +203,7 @@ class Transaction(BaseTransaction):
@classmethod
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address))
return cls.create([], [output], funding_accounts, change_account)
@@ -211,7 +211,7 @@ class Transaction(BaseTransaction):
def claim_create(
cls, name: str, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
claim_output = Output.pay_claim_name_pubkey_hash(
amount, name, claim, ledger.address_to_hash160(holding_address)
)
@@ -223,7 +223,7 @@ class Transaction(BaseTransaction):
def claim_update(
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
updated_claim = Output.pay_update_claim_pubkey_hash(
amount, previous_claim.claim_name, previous_claim.claim_id,
claim, ledger.address_to_hash160(holding_address)
@@ -239,7 +239,7 @@ class Transaction(BaseTransaction):
@classmethod
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
@@ -248,7 +248,7 @@ class Transaction(BaseTransaction):
@classmethod
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
funding_accounts: List[Account], change_account: Account):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
claim_output = Output.purchase_claim_pubkey_hash(
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
)
+10 -1
View File
@@ -13,13 +13,15 @@ from lbry.extras.daemon.Daemon import (
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
)
from lbry.extras.daemon.json_response_encoder import (
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
encode_wallet_doc
)
from lbry.testcase import CommandTestCase
RETURN_DOCS = {
'Account': encode_account_doc(),
'Wallet': encode_wallet_doc(),
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
@@ -103,6 +105,13 @@ class Examples(CommandTestCase):
'preference', 'get'
)
# wallets
await r(
'List your wallets',
'wallet', 'list'
)
# accounts
await r(
+2 -2
View File
@@ -141,9 +141,9 @@ def release(args):
type_label = get_label(pr, 'type')
if area_labels and type_label:
for area_name in area_labels:
for incompat in get_backwards_incompatible(pr.body):
for incompat in get_backwards_incompatible(pr.body or ""):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body):
for release_text in get_release_text(pr.body or ""):
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
if type_label != 'fixup':
area = areas.setdefault(area_name, [])
+2 -2
View File
@@ -58,7 +58,7 @@ async def main(cmd_args=None):
start = time.perf_counter()
kwargs = {
'page': page,
# 'claim_type': 'stream',
'claim_type': 'stream',
'order_by': ['trending_global', 'trending_mixed'],
'no_totals': True
}
@@ -153,7 +153,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--allow_fees", action='store_true')
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=0, type=int)
parser.add_argument("--stall_download_timeout", default=5, type=int)
parser.add_argument("--keep_files", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
parser.add_argument("--download_pages", type=int, default=10)
@@ -11,14 +11,6 @@ def extract(d, keys):
class AccountManagement(CommandTestCase):
async def test_sqlite_binding_error(self):
tasks = [
self.loop.create_task(self.daemon.jsonrpc_account_create('second account' + str(x))) for x in range(100)
]
await asyncio.wait(tasks)
for result in tasks:
self.assertFalse(isinstance(result.result(), Exception))
async def test_account_list_set_create_remove_add(self):
# check initial account
response = await self.daemon.jsonrpc_account_list()
@@ -93,7 +85,7 @@ class AccountManagement(CommandTestCase):
stream_list = self.daemon.jsonrpc_stream_list
support_list = self.daemon.jsonrpc_support_list
utxo_list = self.daemon.jsonrpc_utxo_list
default_account = self.daemon.default_account
default_account = self.wallet.default_account
second_account = await self.daemon.jsonrpc_account_create('second account')
tx = await self.daemon.jsonrpc_account_send(
@@ -179,3 +171,9 @@ class AccountManagement(CommandTestCase):
})
self.assertEqual(history[6]['value'], '0.0')
self.assertEqual(history[7]['value'], '10.0')
async def test_address_validation(self):
address = await self.daemon.jsonrpc_address_unused()
bad_address = address[0:20] + '9999999' + address[27:]
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
+60 -27
View File
@@ -466,45 +466,63 @@ class ChannelCommands(CommandTestCase):
'title': 'foo', 'email': 'new@email.com'}
)
# send channel to someone else
# move channel to another account
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
# before sending
# before moving
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
other_address = await account2.receiving.get_or_create_usable_address()
tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
# after sending
# after moving
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
# shoud not have private key
txo = (await account2.get_channels())[0]
self.assertIsNone(txo.private_key)
# send the private key too
private_key = self.account.get_channel_private_key(unhexlify(channel['public_key']))
account2.add_channel_private_key(private_key)
# now should have private key
txo = (await account2.get_channels())[0]
self.assertIsNotNone(txo.private_key)
async def test_channel_export_import_into_new_account(self):
async def test_channel_export_import_before_sending_channel(self):
# export
tx = await self.channel_create('@foo', '1.0')
claim_id = self.get_claim_id(tx)
channel_private_key = (await self.account.get_channels())[0].private_key
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
# import
daemon2 = await self.add_daemon()
self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
await daemon2.jsonrpc_channel_import(exported_data)
channels = await daemon2.jsonrpc_channel_list()
self.assertEqual(1, len(channels))
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
# second wallet can't update until channel is sent to it
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
# now send the channel as well
await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused())
# second wallet should be able to update now
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
async def test_channel_update_across_accounts(self):
account2 = await self.daemon.jsonrpc_account_create('second account')
channel = await self.out(self.channel_create('@spam', '1.0', account_id=account2.id))
# channel not in account1
with self.assertRaisesRegex(Exception, "Can't find the channel"):
await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=self.account.id)
# channel is in account2
await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=account2.id)
result = await self.out(self.daemon.jsonrpc_channel_list())
self.assertEqual(result[0]['amount'], '2.0')
# check all accounts for channel
await self.channel_update(self.get_claim_id(channel), bid='3.0')
result = await self.out(self.daemon.jsonrpc_channel_list())
self.assertEqual(result[0]['amount'], '3.0')
await self.channel_abandon(self.get_claim_id(channel))
class StreamCommands(ClaimTestCase):
@@ -562,10 +580,26 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(len(tx['outputs']), 1) # no change
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
async def test_stream_update_and_abandon_across_accounts(self):
account2 = await self.daemon.jsonrpc_account_create('second account')
stream = await self.out(self.stream_create('spam', '1.0', account_id=account2.id))
# stream not in account1
with self.assertRaisesRegex(Exception, "Can't find the stream"):
await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=self.account.id)
# stream is in account2
await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=account2.id)
result = await self.out(self.daemon.jsonrpc_stream_list())
self.assertEqual(result[0]['amount'], '2.0')
# check all accounts for stream
await self.stream_update(self.get_claim_id(stream), bid='3.0')
result = await self.out(self.daemon.jsonrpc_stream_list())
self.assertEqual(result[0]['amount'], '3.0')
await self.stream_abandon(self.get_claim_id(stream))
async def test_publishing_checks_all_accounts_for_channel(self):
account1_id, account1 = self.account.id, self.account
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
await self.out(self.channel_create('@spam', '1.0'))
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
@@ -782,7 +816,7 @@ class StreamCommands(ClaimTestCase):
# send claim to someone else
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
# before sending
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
@@ -1079,13 +1113,12 @@ class StreamCommands(ClaimTestCase):
class SupportCommands(CommandTestCase):
async def test_regular_supports_and_tip_supports(self):
# account2 will be used to send tips and supports to account1
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
account2 = self.daemon.get_account_or_error(account2_id)
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
account2 = wallet2.accounts[0]
# send account2 5 LBC out of the 10 LBC in account1
result = await self.out(self.daemon.jsonrpc_account_send(
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
'5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2')
))
await self.on_transaction_dict(result)
@@ -1103,7 +1136,7 @@ class SupportCommands(CommandTestCase):
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '1.0', True, account2_id, funding_account_ids=[account2_id])
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(tip['txid'])
@@ -1122,7 +1155,7 @@ class SupportCommands(CommandTestCase):
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = await self.out(
self.daemon.jsonrpc_transaction_list(account2_id)
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
@@ -1134,7 +1167,7 @@ class SupportCommands(CommandTestCase):
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '2.0', False, account2_id, funding_account_ids=[account2_id])
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
)
await self.confirm_tx(support['txid'])
@@ -1143,7 +1176,7 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '1.999717')
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
+26
View File
@@ -1,4 +1,6 @@
import contextlib
import asyncio
import logging
from io import StringIO
from torba.testcase import AsyncioTestCase
@@ -37,3 +39,27 @@ class CLIIntegrationTest(AsyncioTestCase):
cli.main(["--api", "localhost:5299", "status"])
actual_output = actual_output.getvalue()
self.assertIn("connection_status", actual_output)
def test_setup_logging(self):
def setup(argv):
parser = cli.get_argument_parser()
args, command_args = parser.parse_known_args(argv)
loop = asyncio.get_event_loop()
conf = Config.create_from_arguments(args)
cli.setup_logging(args, conf, loop)
setup(["start"])
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
setup(["start", "--verbose"])
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
setup(["start", "--verbose", "lbry.extras", "lbry.wallet", "torba.client"])
self.assertTrue(logging.getLogger("lbry.extras").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("lbry.wallet").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("torba.client").isEnabledFor(logging.DEBUG))
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
+5 -5
View File
@@ -3,7 +3,7 @@ from binascii import hexlify
from lbry.dht import constants
from lbry.dht.node import Node
from lbry.dht.peer import PeerManager, KademliaPeer
from lbry.dht.peer import PeerManager, make_kademlia_peer
from torba.testcase import AsyncioTestCase
@@ -39,7 +39,7 @@ class DHTIntegrationTest(AsyncioTestCase):
bad_peers = []
for candidate in self.nodes[1:10]:
address, port, node_id = candidate.protocol.external_ip, candidate.protocol.udp_port, candidate.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
peer = make_kademlia_peer(node_id, address, udp_port=port)
bad_peers.append(peer)
node.protocol.add_peer(peer)
candidate.stop()
@@ -70,7 +70,7 @@ class DHTIntegrationTest(AsyncioTestCase):
await asyncio.sleep(.1)
timeout -= 1
if not timeout:
self.fail("node didnt join back after 2 seconds")
self.fail("node didn't join back after 2 seconds")
async def test_announce_no_peers(self):
await self.setup_network(1)
@@ -100,9 +100,9 @@ class DHTIntegrationTest(AsyncioTestCase):
await self.setup_network(2, seed_nodes=2)
node1, node2 = self.nodes
node2.stop()
# forcefully make it a bad peer but dont remove it from routing table
# forcefully make it a bad peer but don't remove it from routing table
address, port, node_id = node2.protocol.external_ip, node2.protocol.udp_port, node2.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
peer = make_kademlia_peer(node_id, address, udp_port=port)
self.assertTrue(node1.protocol.peer_manager.peer_is_good(peer))
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
+29 -2
View File
@@ -98,7 +98,7 @@ class FileCommands(CommandTestCase):
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didnt create a file")
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didn't create a file")
await self.server.blob_manager.delete_blobs([sd_hash])
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
@@ -118,7 +118,7 @@ class FileCommands(CommandTestCase):
handle.write(b'some other stuff was there instead')
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed
# check that internal state got through up to the file list API
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
file_info = self.sout(self.daemon.jsonrpc_file_list()[0])
@@ -277,6 +277,33 @@ class FileCommands(CommandTestCase):
self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0')
self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address)
async def test_null_max_key_fee(self):
target_address = await self.blockchain.get_raw_change_address()
self.daemon.conf.max_key_fee = None
await self.stream_create(
'somename', '0.5', data=b'Yes, please',
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
)
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
# Assert the fee and bid are subtracted
await self.assertBalance(self.account, '9.483893')
response = await self.daemon.jsonrpc_get('lbry://somename')
await self.ledger.wait(response.content_fee)
await self.assertBalance(self.account, '8.483769')
# Assert the file downloads
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
# Assert the transaction is recorded to the blockchain
starting_balance = await self.blockchain.get_balance()
await self.generate(1)
block_reward_and_claim_fee = 2.0
self.assertEqual(
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
)
async def test_null_fee(self):
target_address = await self.blockchain.get_raw_change_address()
tx = await self.stream_create(
@@ -0,0 +1,12 @@
from lbry.testcase import CommandTestCase
class AddressManagement(CommandTestCase):
async def test_address_list(self):
addresses = await self.out(self.daemon.jsonrpc_address_list())
self.assertEqual(27, len(addresses))
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
self.assertEqual(1, len(single))
self.assertEqual(single[0], addresses[11])
@@ -86,6 +86,12 @@ class ResolveCommand(BaseResolveTestCase):
self.assertEqual(claim['name'], 'gibberish')
self.assertNotIn('value', claim)
# resolve retries
await self.conductor.spv_node.stop()
resolving_future = asyncio.ensure_future(self.resolve('foo'))
await self.conductor.spv_node.start(self.conductor.blockchain_node)
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
async def test_winning_by_effective_amount(self):
# first one remains winner unless something else changes
claim_id1 = self.get_claim_id(
@@ -172,7 +178,7 @@ class ResolveCommand(BaseResolveTestCase):
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim_id = self.get_claim_id(orphan_claim)
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
# Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertEqual(response, {
+1 -1
View File
@@ -413,7 +413,7 @@ class RangeRequestsLRUCache(CommandTestCase):
self.server.stop_server()
# running with cache size 0 gets through without errors without
# this since the server doesnt stop immediately
# this since the server doesn't stop immediately
await asyncio.sleep(1, loop=self.loop)
await self._request_stream()
-114
View File
@@ -1,114 +0,0 @@
from unittest import mock
from torba.orchstr8.node import WalletNode, SPVNode
from torba.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.wallet import LbryWalletManager, RegTestLedger
from lbry.extras.daemon.Daemon import Daemon
from lbry.extras.daemon.Components import WalletComponent
from lbry.extras.daemon.Components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbry.extras.daemon.ComponentManager import ComponentManager
class AccountSynchronization(AsyncioTestCase):
async def asyncSetUp(self):
self.wallet_node = WalletNode(LbryWalletManager, RegTestLedger)
await self.wallet_node.start(
SPVNode(None),
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach absent",
False
)
self.account = self.wallet_node.account
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = False
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('localhost', 50001)]
conf.reflector_servers = []
conf.known_dht_nodes = []
def wallet_maker(component_manager):
self.wallet_component = WalletComponent(component_manager)
self.wallet_component.wallet_manager = self.wallet_node.manager
self.wallet_component._running = True
return self.wallet_component
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
]
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
))
await self.daemon.initialize()
async def asyncTearDown(self):
self.wallet_component._running = False
await self.daemon.stop(shutdown_runner=False)
@mock.patch('time.time', mock.Mock(return_value=12345))
def test_sync(self):
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
self.account.modified_on = 123.456
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
self.assertFalse(self.account.channel_keys)
hash_w_cert = '974721f42dab42657b5911b7caf4af98ce4d3879eea6ac23d50c1d79bc5020ef'
add_cert = (
'czo4MTkyOjE2OjE6qs3JRvS/bhX8p1JD68sqyA2Qhx3EVTqskhqEAwtfAUfUsQqeJ1rtMdRf40vkGKnpt4NT0b'
'XEqb5O+lba4nkLF7vZENhc2zuOrjobCPVbiVHNwOfH56Ayrh1ts5LMcnl5+Mk1BUyGCwXcqEg2KiUkd3YZpiHQ'
'T7WfcODcU6l7IRivb8iawCebZJx9waVyQoqEDKwZUY1i5HA0VLC+s5cV7it1AWbewiyWOQtZdEPzNY44oXLJex'
'SirElQqDqNZyl3Hjy8YqacBbSYoejIRnmXpC9y25keP6hep3f9i1K2HDNwhwns1W1vhuzuO2Gy9+a0JlVm5mwc'
'N2pqO4tCZr6tE3aym2FaSAunOi7QYVFMI6arb9Gvn9P+T+WRiFYfzwDFVR+j5ZPmUDXxHisy5OF163jH61wbBY'
'pPienjlVtDOxoZmA8+AwWXKRdINsRcull9pu7EVCq5yQmrmxoPbLxNh5pRGrBB0JwCCOMIf+KPwS+7Z6dDbiwO'
'2NUpk8USJMTmXmFDCr2B0PJiG6Od2dD2oGN0F7aYZvUuKbqj8eDrJMe/zLbhq47jUjkJFCvtxUioo63ORk1pzH'
'S0/X4/6/95PRSMaXm4DcZ9BdyxR2E/AKc8UN6AL5rrn6quXkC6R3ZhKgN3Si2S9y6EGFsL7dgzX331U08ZviLj'
'NsrG0EKUnf+TGQ42MqnLQBOiO/ZoAwleOzNZnCYOQQ14Mm8y17xUpmdWRDiRKpAOJU22jKnxtqQ='
)
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
self.assertEqual(self.account.channel_keys, {'abcdefg1234:0': '---PRIVATE KEY---'})
# applying the same diff is idempotent
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
self.assertEqual(self.account.channel_keys, {'abcdefg1234:0': '---PRIVATE KEY---'})
@mock.patch('time.time', mock.Mock(return_value=12345))
def test_account_preferences_syncing(self):
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
self.account.modified_on = 123.456
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
self.assertFalse(self.daemon.jsonrpc_preference_get())
hash_w_pref = '2fe43f0b2f8bbf1fbb55537f862d8bcb0823791019a7151c848bd5f5bd32d336'
add_pref = (
'czo4MTkyOjE2OjE6Jgn3nAGrfYP2usMA4KQ/73+YHAwMyiGdSWuxCmgZKlpwSpnfQv8R7R0tum/n2oTSBQxjdL'
'OlTW+tv/G5L2GfQ5op3xaT89gN+F/JJnvf3cdWvYH7Nc+uTUMb7cKhJP7hQvFW5bb1Y3jX3EBBY00Jkqyj9RCR'
'XPtbLVu71KbVRvCAR/oAnMEsgD+ITsC3WkXMwE3BS2LjJDQmeqbH4YXNdcjJN/JzQ6fxOmr3Uk1GqnpuhFsta8'
'H14ViRilq1pLKOSZIN80rrm5cKq45nFO5kFeoqBCEaal4u2/OkX9nOnpQlO3E95wD8hkCmZ3i20aSte6nqwqXx'
'ZKVRZqR2a0TjwVWB8kPXPA2ewKvPILaj190bXPl8EVu+TAnTCQwMgytinYjtcKNZmMz3ENJyI2mCANwpWlX7xl'
'y/J+qLi5b9N+agghTxggs5rVJ/hkaue7GS542dXDrwMrw9nwGqNw3dS/lcU+1wRUQ0fnHwb/85XbbwyO2aDj2i'
'DFNkdyLyUIiIUvB1JfWAnWqX3vQcL1REK1ePgUei7dCHJ3WyWdsRx3cVXzlK8yOPkf0N6d3AKrZQWVebwDC7Nd'
'eL4sDW8AkaXuBIrbuZw6XUHd6WI0NvU/q10j2qMm0YoXSu+dExou1/1THwx5g86MxcX5nwodKUEVCOTzKMyrLz'
'CRsitH/+dAXhZNRp/FbnDCGBMyD3MOYCjZvAFbCZUasoRwqponxILw=='
)
self.daemon.jsonrpc_sync_apply('password', data=add_pref)
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["apple", "orange"]})
self.daemon.jsonrpc_preference_set("fruit", ["peach", "apricot"])
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["peach", "apricot"]})
self.assertNotEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
@@ -38,34 +38,42 @@ class TransactionCommandsTestCase(CommandTestCase):
await self.assertBalance(self.account, '11.0')
async def test_granular_balances(self):
account_balance = self.daemon.jsonrpc_account_balance
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
self.assertEqual(await account_balance(reserved_subtotals=False), {
account_balance = self.daemon.jsonrpc_account_balance
wallet_balance = self.daemon.jsonrpc_wallet_balance
expected = {
'total': '10.0',
'available': '10.0',
'reserved': '0.0',
'reserved_subtotals': None
})
}
self.assertEqual(await account_balance(reserved_subtotals=False), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=False), expected)
self.assertEqual(await account_balance(reserved_subtotals=True), {
expected = {
'total': '10.0',
'available': '10.0',
'reserved': '0.0',
'reserved_subtotals': {'claims': '0.0', 'supports': '0.0', 'tips': '0.0'}
})
}
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
# claim with update + supporting our own claim
stream1 = await self.stream_create('granularity', '3.0')
await self.stream_update(self.get_claim_id(stream1), data=b'news', bid='1.0')
await self.support_create(self.get_claim_id(stream1), '2.0')
self.assertEqual(await account_balance(reserved_subtotals=True), {
expected = {
'total': '9.977534',
'available': '6.977534',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
}
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
address2 = await self.daemon.jsonrpc_address_unused(account2.id)
# send lbc to someone else
@@ -77,6 +85,12 @@ class TransactionCommandsTestCase(CommandTestCase):
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.97741',
'available': '6.97741',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
# tip received
support1 = await self.support_create(
@@ -88,6 +102,12 @@ class TransactionCommandsTestCase(CommandTestCase):
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.977268',
'available': '6.677268',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
# tip claimed
tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0)
@@ -98,6 +118,12 @@ class TransactionCommandsTestCase(CommandTestCase):
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.977161',
'available': '6.977161',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
stream2 = await self.stream_create(
'granularity-is-cool', '0.1', account_id=account2.id, funding_account_ids=[account2.id]
@@ -105,10 +131,17 @@ class TransactionCommandsTestCase(CommandTestCase):
# tip another claim
await self.support_create(
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id])
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id]
)
self.assertEqual(await account_balance(reserved_subtotals=True), {
'total': '9.077157',
'available': '6.077157',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.938908',
'available': '6.638908',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'}
})
@@ -0,0 +1,159 @@
import asyncio
import json
from lbry.testcase import CommandTestCase
from torba.client.wallet import ENCRYPT_ON_DISK
class WalletEncryptionAndSynchronization(CommandTestCase):
SEED = (
"carbon smart garage balance margin twelve chest "
"sword toast envelope bottom stomach absent"
)
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon2 = await self.add_daemon(
seed="chest sword toast envelope bottom stomach absent "
"carbon smart garage balance margin twelve"
)
address = (await self.daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0]
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid, self.daemon2.ledger)
def assertWalletEncrypted(self, wallet_path, encrypted):
wallet = json.load(open(wallet_path))
self.assertEqual(wallet['accounts'][0]['private_key'][1:4] != 'prv', encrypted)
async def test_sync(self):
daemon, daemon2 = self.daemon, self.daemon2
# Preferences
self.assertFalse(daemon.jsonrpc_preference_get())
self.assertFalse(daemon2.jsonrpc_preference_get())
daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]')
daemon.jsonrpc_preference_set("one", "1")
daemon.jsonrpc_preference_set("conflict", "1")
daemon2.jsonrpc_preference_set("another", "A")
await asyncio.sleep(1)
# these preferences will win after merge since they are "newer"
daemon2.jsonrpc_preference_set("two", "2")
daemon2.jsonrpc_preference_set("conflict", "2")
daemon.jsonrpc_preference_set("another", "B")
self.assertDictEqual(daemon.jsonrpc_preference_get(), {
"one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"]
})
self.assertDictEqual(daemon2.jsonrpc_preference_get(), {
"two": "2", "conflict": "2", "another": "A"
})
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 1)
data = await daemon2.jsonrpc_sync_apply('password')
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 2)
self.assertDictEqual(
# "two" key added and "conflict" value changed to "2"
daemon.jsonrpc_preference_get(),
{"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]}
)
# Channel Certificate
channel = await daemon2.jsonrpc_channel_create('@foo', '0.1')
await self.confirm_tx(channel.id, self.daemon2.ledger)
# both daemons will have the channel but only one has the cert so far
self.assertEqual(len(await daemon.jsonrpc_channel_list()), 1)
self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
self.assertEqual(len(await daemon2.jsonrpc_channel_list()), 1)
self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
data = await daemon2.jsonrpc_sync_apply('password')
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
# both daemons have the cert after sync'ing
self.assertEqual(
daemon2.wallet_manager.default_account.channel_keys,
daemon.wallet_manager.default_wallet.accounts[1].channel_keys
)
async def test_encryption_and_locking(self):
daemon = self.daemon
wallet = daemon.wallet_manager.default_wallet
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertIsNone(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK))
self.assertWalletEncrypted(wallet.storage.path, False)
# can't lock an unencrypted account
with self.assertRaisesRegex(AssertionError, "Cannot lock an unencrypted wallet, encrypt first."):
daemon.jsonrpc_wallet_lock()
# safe to call unlock and decrypt, they are no-ops at this point
daemon.jsonrpc_wallet_unlock('password') # already unlocked
daemon.jsonrpc_wallet_decrypt() # already not encrypted
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
daemon.jsonrpc_wallet_lock()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True})
# can't sign transactions with locked wallet
with self.assertRaises(AssertionError):
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_unlock('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_decrypt()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': False})
self.assertWalletEncrypted(wallet.storage.path, False)
async def test_sync_with_encryption_and_password_change(self):
daemon, daemon2 = self.daemon, self.daemon2
wallet, wallet2 = daemon.wallet_manager.default_wallet, daemon2.wallet_manager.default_wallet
self.assertEqual(wallet2.encryption_password, None)
self.assertEqual(wallet2.encryption_password, None)
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(wallet.encryption_password, 'password')
data = await daemon2.jsonrpc_sync_apply('password2')
# sync_apply doesn't save password if encrypt-on-disk is False
self.assertEqual(wallet2.encryption_password, None)
# need to use new password2 in sync_apply
with self.assertRaises(ValueError): # wrong password
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True)
# sync_apply with new password2 also sets it as new local password
self.assertEqual(wallet.encryption_password, 'password2')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
# check new password is active
daemon.jsonrpc_wallet_lock()
self.assertFalse(daemon.jsonrpc_wallet_unlock('password'))
self.assertTrue(daemon.jsonrpc_wallet_unlock('password2'))
# propagate disk encryption to daemon2
data = await daemon.jsonrpc_sync_apply('password3')
# sync_apply (even with no data) on wallet with encrypt-on-disk updates local password
self.assertEqual(wallet.encryption_password, 'password3')
self.assertEqual(wallet2.encryption_password, None)
await daemon2.jsonrpc_sync_apply('password3', data=data['data'], blocking=True)
# the other device got new password and on disk encryption
self.assertEqual(wallet2.encryption_password, 'password3')
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon2.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet2.storage.path, True)
daemon2.jsonrpc_wallet_lock()
self.assertTrue(daemon2.jsonrpc_wallet_unlock('password3'))
@@ -55,7 +55,7 @@ class TestHeadersComponent(CommandTestCase):
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesnt raise
# test is that this doesn't raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -12,7 +12,8 @@ from lbry.extras.daemon.storage import SQLiteStorage
from lbry.blob.blob_manager import BlobManager
from lbry.blob_exchange.server import BlobServer, BlobServerProtocol
from lbry.blob_exchange.client import request_blob
from lbry.dht.peer import KademliaPeer, PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
# import logging
# logging.getLogger("lbry").setLevel(logging.DEBUG)
@@ -43,7 +44,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
self.server_from_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
await self.client_storage.open()
await self.server_storage.open()
@@ -102,7 +103,7 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
await second_client_storage.open()
await second_client_blob_manager.setup()
@@ -193,7 +194,7 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
await second_client_storage.open()
await second_client_blob_manager.setup()
@@ -1,7 +1,7 @@
import asyncio
from unittest import mock, TestCase
from lbry.dht.protocol.data_store import DictDataStore
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
class DataStoreTests(TestCase):
@@ -13,7 +13,7 @@ class DataStoreTests(TestCase):
def _test_add_peer_to_blob(self, blob=b'2' * 48, node_id=b'1' * 48, address='1.2.3.4', tcp_port=3333,
udp_port=4444):
peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
peer = make_kademlia_peer(node_id, address, udp_port)
peer.update_tcp_port(tcp_port)
before = self.data_store.get_peers_for_blob(blob)
self.data_store.add_peer_to_blob(peer, blob)
+7 -7
View File
@@ -2,7 +2,7 @@ import struct
import asyncio
from lbry.utils import generate_id
from lbry.dht.protocol.routing_table import KBucket
from lbry.dht.peer import PeerManager, KademliaPeer
from lbry.dht.peer import PeerManager, make_kademlia_peer
from lbry.dht import constants
from torba.testcase import AsyncioTestCase
@@ -29,8 +29,8 @@ class TestKBucket(AsyncioTestCase):
self.kbucket = KBucket(self.peer_manager, 0, 2**constants.hash_bits, generate_id())
def test_add_peer(self):
peer = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4444)
peer_update2 = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4445)
peer = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4444)
peer_update2 = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4445)
self.assertListEqual([], self.kbucket.peers)
@@ -59,12 +59,12 @@ class TestKBucket(AsyncioTestCase):
# Test if contacts can be added to empty list
# Add k contacts to bucket
for i in range(constants.k):
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
self.assertEqual(peer, self.kbucket.peers[i])
# Test if contact is not added to full list
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
self.assertFalse(self.kbucket.add_peer(peer))
# Test if an existing contact is updated correctly if added again
@@ -125,13 +125,13 @@ class TestKBucket(AsyncioTestCase):
def test_remove_peer(self):
# try remove contact from empty list
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
self.assertRaises(ValueError, self.kbucket.remove_peer, peer)
added = []
# Add couple contacts
for i in range(constants.k-2):
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
self.assertTrue(self.kbucket.add_peer(peer))
added.append(peer)
+10 -10
View File
@@ -5,7 +5,7 @@ from tests import dht_mocks
from lbry.dht.serialization.bencoding import bencode, bdecode
from lbry.dht import constants
from lbry.dht.protocol.protocol import KademliaProtocol
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
class TestProtocol(AsyncioTestCase):
@@ -22,7 +22,7 @@ class TestProtocol(AsyncioTestCase):
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
result = await peer2.get_rpc_peer(peer).ping()
self.assertEqual(result, b'pong')
peer1.stop()
@@ -43,7 +43,7 @@ class TestProtocol(AsyncioTestCase):
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
self.assertEqual(None, peer2.peer_manager.get_node_token(peer.node_id))
await peer2.get_rpc_peer(peer).find_value(b'1' * 48)
self.assertNotEqual(None, peer2.peer_manager.get_node_token(peer.node_id))
@@ -65,12 +65,12 @@ class TestProtocol(AsyncioTestCase):
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
peer2_from_peer1 = make_kademlia_peer(
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
)
peer2_from_peer1.update_tcp_port(3333)
peer3 = peer1.peer_manager.get_kademlia_peer(
peer3 = make_kademlia_peer(
constants.generate_id(), '1.2.3.6', udp_port=4444
)
store_result = await peer2.store_to_peer(b'2' * 48, peer)
@@ -103,7 +103,7 @@ class TestProtocol(AsyncioTestCase):
)
await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444))
proto.start()
return proto, other_peer.peer_manager.get_kademlia_peer(node_id, address, udp_port=udp_port)
return proto, make_kademlia_peer(node_id, address, udp_port=udp_port)
async def test_add_peer_after_handle_request(self):
with dht_mocks.mock_network_loop(self.loop):
@@ -129,7 +129,7 @@ class TestProtocol(AsyncioTestCase):
peer1.routing_table.remove_peer(peer_2_from_peer_1)
# peers not known by be good/bad should be enqueued to maybe-ping
peer1_from_peer3 = peer3.get_rpc_peer(peer3.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1_from_peer3 = peer3.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
pong = await peer1_from_peer3.ping()
self.assertEqual(b'pong', pong)
@@ -137,7 +137,7 @@ class TestProtocol(AsyncioTestCase):
peer1.ping_queue._pending_contacts.clear()
# peers who are already good should be added
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_last_replied('1.2.3.7', 4444)
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
@@ -149,7 +149,7 @@ class TestProtocol(AsyncioTestCase):
peer1.routing_table.buckets[0].peers.clear()
# peers who are known to be bad recently should not be added or maybe-pinged
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
peer1.peer_manager.report_failure('1.2.3.7', 4444)
@@ -3,8 +3,7 @@ from torba.testcase import AsyncioTestCase
from tests import dht_mocks
from lbry.dht import constants
from lbry.dht.node import Node
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
expected_ranges = [
(
@@ -29,6 +28,7 @@ expected_ranges = [
)
]
class TestRouting(AsyncioTestCase):
async def test_fill_one_bucket(self):
loop = asyncio.get_event_loop()
@@ -53,7 +53,7 @@ class TestRouting(AsyncioTestCase):
for i in range(1, len(peer_addresses)):
self.assertEqual(len(node_1.protocol.routing_table.get_peers()), contact_cnt)
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
peer = make_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip,
udp_port=node.protocol.udp_port
)
@@ -66,6 +66,16 @@ class TestRouting(AsyncioTestCase):
for node in nodes.values():
node.protocol.stop()
async def test_cant_add_peer_without_a_node_id_gracefully(self):
loop = asyncio.get_event_loop()
node = Node(loop, PeerManager(loop), constants.generate_id(), 4444, 4444, 3333, '1.2.3.4')
bad_peer = make_kademlia_peer(None, '1.2.3.4', 5555)
with self.assertLogs(level='WARNING') as logged:
self.assertFalse(await node.protocol._add_peer(bad_peer))
self.assertEqual(1, len(logged.output))
self.assertTrue(logged.output[0].endswith('Tried adding a peer with no node id!'))
async def test_split_buckets(self):
loop = asyncio.get_event_loop()
peer_addresses = [
@@ -81,7 +91,7 @@ class TestRouting(AsyncioTestCase):
node_1 = nodes[0]
for i in range(1, len(peer_addresses)):
node = nodes[i]
peer = node_1.protocol.peer_manager.get_kademlia_peer(
peer = make_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip,
udp_port=node.protocol.udp_port
)
+5 -5
View File
@@ -8,7 +8,7 @@ from tests import dht_mocks
from lbry.conf import Config
from lbry.dht import constants
from lbry.dht.node import Node
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.extras.daemon.storage import SQLiteStorage
@@ -35,7 +35,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
self.nodes.update({len(self.nodes): n})
if add_to_routing_table:
self.node.protocol.add_peer(
self.peer_manager.get_kademlia_peer(
make_kademlia_peer(
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
)
)
@@ -67,7 +67,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
await self.add_peer(node_id, address, False)
last_node = self.nodes[len(self.nodes) - 1]
peer = last_node.protocol.get_rpc_peer(
last_node.protocol.peer_manager.get_kademlia_peer(
make_kademlia_peer(
previous_last_node.protocol.node_id, previous_last_node.protocol.external_ip,
previous_last_node.protocol.udp_port
)
@@ -129,12 +129,12 @@ class TestBlobAnnouncer(AsyncioTestCase):
announced_to = self.nodes[0]
for i in range(1, peer_count):
node = self.nodes[i]
kad_peer = announced_to.protocol.peer_manager.get_kademlia_peer(
kad_peer = make_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
)
await announced_to.protocol._add_peer(kad_peer)
peer = node.protocol.get_rpc_peer(
node.protocol.peer_manager.get_kademlia_peer(
make_kademlia_peer(
announced_to.protocol.node_id,
announced_to.protocol.external_ip,
announced_to.protocol.udp_port
+2 -2
View File
@@ -4,7 +4,7 @@ from torba.testcase import AsyncioTestCase
from tests import dht_mocks
from lbry.dht import constants
from lbry.dht.node import Node
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
class TestNodePingQueueDiscover(AsyncioTestCase):
@@ -42,7 +42,7 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
for i in range(1, len(peer_addresses)):
node = nodes[i]
assert node.protocol.node_id != node_1.protocol.node_id
peer = node_1.protocol.peer_manager.get_kademlia_peer(
peer = make_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip, udp_port=node.protocol.udp_port
)
futs.append(node_1.protocol.get_rpc_peer(peer).ping())
+19 -9
View File
@@ -1,7 +1,7 @@
import asyncio
import unittest
from lbry.utils import generate_id
from lbry.dht.peer import PeerManager
from lbry.dht.peer import PeerManager, make_kademlia_peer
from torba.testcase import AsyncioTestCase
@@ -10,20 +10,30 @@ class PeerTest(AsyncioTestCase):
self.loop = asyncio.get_event_loop()
self.peer_manager = PeerManager(self.loop)
self.node_ids = [generate_id(), generate_id(), generate_id()]
self.first_contact = self.peer_manager.get_kademlia_peer(self.node_ids[1], '127.0.0.1', udp_port=1000)
self.second_contact = self.peer_manager.get_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
self.first_contact = make_kademlia_peer(self.node_ids[1], '127.0.0.1', udp_port=1000)
self.second_contact = make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
def test_peer_is_good_unknown_peer(self):
# Scenario: peer replied, but caller doesn't know the node_id.
# Outcome: We can't say it's good or bad.
# (yes, we COULD tell the node id, but not here. It would be
# a side effect and the caller is responsible to discover it)
peer = make_kademlia_peer(None, '1.2.3.4', 4444)
self.peer_manager.report_last_requested('1.2.3.4', 4444)
self.peer_manager.report_last_replied('1.2.3.4', 4444)
self.assertIsNone(self.peer_manager.peer_is_good(peer))
def test_make_contact_error_cases(self):
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20', 100000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20.1', 1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], 'this is not an ip', 1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20', -1000)
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, b'not valid node id', '192.168.1.20', 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', 100000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20.1', 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], 'this is not an ip', 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', -1000)
self.assertRaises(ValueError, make_kademlia_peer, b'not valid node id', '192.168.1.20', 1000)
def test_boolean(self):
self.assertNotEqual(self.first_contact, self.second_contact)
self.assertEqual(
self.second_contact, self.peer_manager.get_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
self.second_contact, make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
)
def test_compact_ip(self):
+31
View File
@@ -55,10 +55,41 @@ class TestURLParsing(unittest.TestCase):
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\u0009')
_url('\u000a')
_url('\u000d')
_url('\u0020')
_url('\uD799')
_url('\uE000')
_url('\uFFFD')
def test_parser_invalid_urls(self):
fail = self._fail_url
fail("lbry://")
fail("lbry://\u0000")
fail("lbry://\u0008")
fail("lbry://\u000b")
fail("lbry://\u000c")
fail("lbry://\u000e")
fail("lbry://\u001f")
fail("lbry://\uD800")
fail("lbry://\uDFFF")
fail("lbry://\uDFFE")
fail("lbry://\uFFFF")
fail("lbry://;")
fail("lbry://\"")
fail("lbry://\\")
fail("lbry:///")
fail("lbry://<") and fail("lbry://>")
fail("lbry://{") and fail("lbry://}")
fail("lbry://[") and fail("lbry://]")
fail("lbry://%")
fail("lbry://|")
fail("lbry://^")
fail("lbry://~")
fail("lbry://`")
fail("lbry://test:3$1")
fail("lbry://test$1:1")
fail("lbry://test#x")
@@ -7,8 +7,7 @@ from lbry.blob.blob_file import MAX_BLOB_SIZE
from lbry.blob_exchange.serialization import BlobResponse
from lbry.blob_exchange.server import BlobServerProtocol
from lbry.dht.node import Node
from lbry.dht.peer import KademliaPeer
from lbry.extras.daemon.storage import StoredStreamClaim
from lbry.dht.peer import make_kademlia_peer
from lbry.stream.managed_stream import ManagedStream
from lbry.stream.descriptor import StreamDescriptor
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
@@ -95,7 +94,7 @@ class TestManagedStream(BlobExchangeTestBase):
mock_node = mock.Mock(spec=Node)
q = asyncio.Queue()
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334)
def _mock_accumulate_peers(q1, q2):
async def _task():
@@ -126,5 +126,5 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
loop, tmp_dir, blob
)
self.assertFalse(blob.file_exists)
# fixme: this is an emergency PR, plase move this to blob_file tests later
# fixme: this is an emergency PR, please move this to blob_file tests later
self.assertIsNone(blob.length)
+1
View File
@@ -51,6 +51,7 @@ class CLITest(AsyncioTestCase):
self.assertEqual('3', normalize_value('3', key="name"))
self.assertEqual('3', normalize_value('3', key="download_directory"))
self.assertEqual('3', normalize_value('3', key="channel_name"))
self.assertEqual('3', normalize_value('3', key="claim_name"))
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
+4
View File
@@ -222,6 +222,10 @@ class ConfigurationTests(unittest.TestCase):
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, None)
args = parser.parse_args(['--max-key-fee', 'null'])
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, None)
# set
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
c = Config.create_from_arguments(args)
+6 -2
View File
@@ -61,13 +61,17 @@ class TestAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
private_key = await self.ledger.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
private_key = await self.ledger.get_private_key_for_address(
account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx'
)
self.assertEqual(
private_key.extended_key_string(),
'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR'
'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT'
)
private_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
private_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
self.assertIsNone(private_key)
def test_load_and_save_account(self):
+1
View File
@@ -11,6 +11,7 @@ ignore_missing_imports = True
[pylint]
ignore=words,server,workbench,rpc
max-parents=10
max-args=10
max-line-length=110
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id
@@ -1,4 +1,5 @@
import logging
import os
import asyncio
from unittest.mock import Mock
@@ -16,6 +17,38 @@ class NetworkTests(IntegrationTestCase):
await self.ledger.network.on_header.first
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
async def test_server_features(self):
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
'hosts': {},
'protocol_max': '1.4',
'protocol_min': '1.1',
'pruning': None,
'description': '',
'payment_address': '',
'daily_fee': 0,
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
address = (await self.account.get_addresses(limit=1))[0]
os.environ.update({
'DESCRIPTION': 'Fastest server in the west.',
'DONATION_ADDRESS': address,
'DAILY_FEE': '42'})
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.network.on_connected.first
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
'hosts': {},
'protocol_max': '1.4',
'protocol_min': '1.1',
'pruning': None,
'description': 'Fastest server in the west.',
'payment_address': address,
'daily_fee': 42,
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
@@ -67,7 +100,7 @@ class ReconnectTests(IntegrationTestCase):
# * goes to pick some water outside... * time passes by and another donation comes in
sendtxid = await self.blockchain.send_to_address(address1, 42)
await self.blockchain.generate(1)
# (this is just so the test doesnt hang forever if it doesnt reconnect)
# (this is just so the test doesn't hang forever if it doesn't reconnect)
if not self.ledger.network.is_connected:
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
# omg, the burned cable still works! torba is fire proof!
@@ -82,6 +115,13 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_online_but_still_unavailable(self):
# Edge case. See issue #2445 for context
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
for session in self.ledger.network.session_pool.sessions:
session.response_time = None
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
class ServerPickingTestCase(AsyncioTestCase):
@@ -122,7 +162,7 @@ class ServerPickingTestCase(AsyncioTestCase):
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
self.assertTrue(network.is_connected)
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
# ensure we are connected to all of them after a while
await asyncio.sleep(1)
@@ -88,7 +88,7 @@ class BasicTransactionTests(IntegrationTestCase):
await self.assertBalance(account1, '0.0')
await self.assertBalance(account2, '0.0')
addresses = await self.account.receiving.get_addresses()
addresses = await account1.receiving.get_addresses()
txids = await asyncio.gather(*(
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
))
@@ -148,7 +148,7 @@ class BasicTransactionTests(IntegrationTestCase):
return summary
self.conductor.spv_node.server.mempool.transaction_summaries = random_summary
# 10 unconfirmed txs, all from blockchain wallet
sends = list(self.blockchain.send_to_address(address, 10) for _ in range(10))
sends = [self.blockchain.send_to_address(address, 10) for _ in range(10)]
# use batching to reduce issues with send_to_address on cli
for batch in range(0, len(sends), 10):
txids = await asyncio.gather(*sends[batch:batch + 10])
+41 -37
View File
@@ -47,7 +47,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
async with self.account.receiving.address_generator_lock:
await self.account.receiving._generate_keys(0, 200)
records = await self.account.receiving.get_address_records()
self.assertEqual(201, len(records))
self.assertEqual(len(records), 201)
async def test_ensure_address_gap(self):
account = self.account
@@ -59,8 +59,8 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
await account.receiving._generate_keys(0, 3)
await account.receiving._generate_keys(8, 11)
records = await account.receiving.get_address_records()
self.assertEqual(
[r['position'] for r in records],
self.assertListEqual(
[r['pubkey'].n for r in records],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
)
@@ -68,8 +68,8 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
new_keys = await account.receiving.ensure_address_gap()
self.assertEqual(len(new_keys), 8)
records = await account.receiving.get_address_records()
self.assertEqual(
[r['position'] for r in records],
self.assertListEqual(
[r['pubkey'].n for r in records],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
)
@@ -125,14 +125,18 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
private_key = await self.ledger.get_private_key_for_address('1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
private_key = await self.ledger.get_private_key_for_address(
account.wallet, '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J'
)
self.assertEqual(
private_key.extended_key_string(),
'xprv9xV7rhbg6M4yWrdTeLorz3Q1GrQb4aQzzGWboP3du7W7UUztzNTUrEYTnDfz7o'
'ptBygDxXYRppyiuenJpoBTgYP2C26E1Ah5FEALM24CsWi'
)
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
invalid_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
self.assertIsNone(invalid_key)
self.assertEqual(
@@ -174,7 +178,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
account_data['ledger'] = 'btc_mainnet'
self.assertDictEqual(account_data, account.to_dict())
def test_apply_diff(self):
def test_merge_diff(self):
account_data = {
'name': 'My Account',
'modified_on': 123.456,
@@ -209,13 +213,13 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
account_data['address_generator']['receiving']['gap'] = 8
account_data['address_generator']['receiving']['maximum_uses_per_address'] = 9
account.apply(account_data)
account.merge(account_data)
# no change because modified_on is not newer
self.assertEqual(account.name, 'My Account')
account_data['modified_on'] = 200.00
account.apply(account_data)
account.merge(account_data)
self.assertEqual(account.name, 'Changed Name')
self.assertEqual(account.change.gap, 6)
self.assertEqual(account.change.maximum_uses_per_address, 7)
@@ -268,19 +272,25 @@ class TestSingleKeyAccount(AsyncioTestCase):
self.assertIsInstance(account.receiving, SingleKey)
addresses = await account.receiving.get_addresses()
self.assertEqual(addresses, [])
self.assertListEqual(addresses, [])
# we have 12, but default gap is 20
new_keys = await account.receiving.ensure_address_gap()
self.assertEqual(len(new_keys), 1)
self.assertEqual(new_keys[0], account.public_key.address)
records = await account.receiving.get_address_records()
self.assertEqual(records, [{
'position': 0, 'chain': 0,
pubkey = records[0].pop('pubkey')
self.assertListEqual(records, [{
'chain': 0,
'account': account.public_key.address,
'address': account.public_key.address,
'history': None,
'used_times': 0
}])
self.assertEqual(
pubkey.extended_key_string(),
account.public_key.extended_key_string()
)
# case #1: no new addresses needed
empty = await account.receiving.ensure_address_gap()
@@ -333,14 +343,18 @@ class TestSingleKeyAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], account.public_key.address)
private_key = await self.ledger.get_private_key_for_address(address[0])
private_key = await self.ledger.get_private_key_for_address(
account.wallet, address[0]
)
self.assertEqual(
private_key.extended_key_string(),
'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp'
'5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna',
)
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
invalid_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
self.assertIsNone(invalid_key)
self.assertEqual(
@@ -421,14 +435,14 @@ class AccountEncryptionTests(AsyncioTestCase):
def test_encrypt_wallet(self):
account = self.ledger.account_class.from_dict(self.ledger, Wallet(), self.unencrypted_account)
account.private_key_encryption_init_vector = self.init_vector
account.seed_encryption_init_vector = self.init_vector
account.init_vectors = {
'seed': self.init_vector,
'private_key': self.init_vector
}
self.assertFalse(account.serialize_encrypted)
self.assertFalse(account.encrypted)
self.assertIsNotNone(account.private_key)
account.encrypt(self.password)
self.assertFalse(account.serialize_encrypted)
self.assertTrue(account.encrypted)
self.assertEqual(account.seed, self.encrypted_account['seed'])
self.assertEqual(account.private_key_string, self.encrypted_account['private_key'])
@@ -437,42 +451,32 @@ class AccountEncryptionTests(AsyncioTestCase):
self.assertEqual(account.to_dict()['seed'], self.encrypted_account['seed'])
self.assertEqual(account.to_dict()['private_key'], self.encrypted_account['private_key'])
account.serialize_encrypted = True
account.decrypt(self.password)
self.assertEqual(account.private_key_encryption_init_vector, self.init_vector)
self.assertEqual(account.seed_encryption_init_vector, self.init_vector)
self.assertEqual(account.init_vectors['private_key'], self.init_vector)
self.assertEqual(account.init_vectors['seed'], self.init_vector)
self.assertEqual(account.seed, self.unencrypted_account['seed'])
self.assertEqual(account.private_key.extended_key_string(), self.unencrypted_account['private_key'])
self.assertEqual(account.to_dict()['seed'], self.encrypted_account['seed'])
self.assertEqual(account.to_dict()['private_key'], self.encrypted_account['private_key'])
self.assertEqual(account.to_dict(encrypt_password=self.password)['seed'], self.encrypted_account['seed'])
self.assertEqual(account.to_dict(encrypt_password=self.password)['private_key'], self.encrypted_account['private_key'])
self.assertFalse(account.encrypted)
self.assertTrue(account.serialize_encrypted)
account.serialize_encrypted = False
self.assertEqual(account.to_dict()['seed'], self.unencrypted_account['seed'])
self.assertEqual(account.to_dict()['private_key'], self.unencrypted_account['private_key'])
def test_decrypt_wallet(self):
account = self.ledger.account_class.from_dict(self.ledger, Wallet(), self.encrypted_account)
self.assertTrue(account.encrypted)
self.assertTrue(account.serialize_encrypted)
account.decrypt(self.password)
self.assertEqual(account.private_key_encryption_init_vector, self.init_vector)
self.assertEqual(account.seed_encryption_init_vector, self.init_vector)
self.assertEqual(account.init_vectors['private_key'], self.init_vector)
self.assertEqual(account.init_vectors['seed'], self.init_vector)
self.assertFalse(account.encrypted)
self.assertTrue(account.serialize_encrypted)
self.assertEqual(account.seed, self.unencrypted_account['seed'])
self.assertEqual(account.private_key.extended_key_string(), self.unencrypted_account['private_key'])
self.assertEqual(account.to_dict()['seed'], self.encrypted_account['seed'])
self.assertEqual(account.to_dict()['private_key'], self.encrypted_account['private_key'])
account.serialize_encrypted = False
self.assertEqual(account.to_dict(encrypt_password=self.password)['seed'], self.encrypted_account['seed'])
self.assertEqual(account.to_dict(encrypt_password=self.password)['private_key'], self.encrypted_account['private_key'])
self.assertEqual(account.to_dict()['seed'], self.unencrypted_account['seed'])
self.assertEqual(account.to_dict()['private_key'], self.unencrypted_account['private_key'])
@@ -37,13 +37,13 @@ class BaseSelectionTestCase(AsyncioTestCase):
class TestCoinSelectionTests(BaseSelectionTestCase):
def test_empty_coins(self):
self.assertEqual(CoinSelector(0, 0).select([]), [])
self.assertListEqual(CoinSelector(0, 0).select([]), [])
def test_skip_binary_search_if_total_not_enough(self):
fee = utxo(CENT).get_estimator(self.ledger).fee
big_pool = self.estimates(utxo(CENT+fee) for _ in range(100))
selector = CoinSelector(101 * CENT, 0)
self.assertEqual(selector.select(big_pool), [])
self.assertListEqual(selector.select(big_pool), [])
self.assertEqual(selector.tries, 0) # Never tried.
# check happy path
selector = CoinSelector(100 * CENT, 0)
@@ -59,7 +59,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
)
selector = CoinSelector(CENT, 0)
match = selector.select(utxo_pool)
self.assertEqual([CENT + fee], [c.txo.amount for c in match])
self.assertListEqual([CENT + fee], [c.txo.amount for c in match])
self.assertTrue(selector.exact_match)
def test_random_draw(self):
@@ -70,7 +70,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
)
selector = CoinSelector(CENT, 0, '\x00')
match = selector.select(utxo_pool)
self.assertEqual([2 * CENT], [c.txo.amount for c in match])
self.assertListEqual([2 * CENT], [c.txo.amount for c in match])
self.assertFalse(selector.exact_match)
def test_pick(self):
@@ -83,7 +83,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
)
selector = CoinSelector(3*CENT, 0)
match = selector.select(utxo_pool)
self.assertEqual([5*CENT], [c.txo.amount for c in match])
self.assertListEqual([5*CENT], [c.txo.amount for c in match])
def test_confirmed_strategies(self):
utxo_pool = self.estimates(
@@ -94,14 +94,14 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
)
match = CoinSelector(20*CENT, 0).select(utxo_pool, "only_confirmed")
self.assertEqual([5, 5], [c.txo.tx_ref.height for c in match])
self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match])
match = CoinSelector(25*CENT, 0).select(utxo_pool, "only_confirmed")
self.assertEqual([], [c.txo.tx_ref.height for c in match])
self.assertListEqual([], [c.txo.tx_ref.height for c in match])
match = CoinSelector(20*CENT, 0).select(utxo_pool, "prefer_confirmed")
self.assertEqual([5, 5], [c.txo.tx_ref.height for c in match])
self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match])
match = CoinSelector(25*CENT, 0, '\x00').select(utxo_pool, "prefer_confirmed")
self.assertEqual([5, 0, -2], [c.txo.tx_ref.height for c in match])
self.assertListEqual([5, 0, -2], [c.txo.tx_ref.height for c in match])
class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
@@ -136,20 +136,20 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
)
# Select 1 Cent
self.assertEqual([1 * CENT], search(utxo_pool, 1 * CENT, 0.5 * CENT))
self.assertListEqual([1 * CENT], search(utxo_pool, 1 * CENT, 0.5 * CENT))
# Select 2 Cent
self.assertEqual([2 * CENT], search(utxo_pool, 2 * CENT, 0.5 * CENT))
self.assertListEqual([2 * CENT], search(utxo_pool, 2 * CENT, 0.5 * CENT))
# Select 5 Cent
self.assertEqual([3 * CENT, 2 * CENT], search(utxo_pool, 5 * CENT, 0.5 * CENT))
self.assertListEqual([3 * CENT, 2 * CENT], search(utxo_pool, 5 * CENT, 0.5 * CENT))
# Select 11 Cent, not possible
self.assertEqual([], search(utxo_pool, 11 * CENT, 0.5 * CENT))
self.assertListEqual([], search(utxo_pool, 11 * CENT, 0.5 * CENT))
# Select 10 Cent
utxo_pool += self.estimates(utxo(5 * CENT))
self.assertEqual(
self.assertListEqual(
[4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT],
search(utxo_pool, 10 * CENT, 0.5 * CENT)
)
@@ -157,18 +157,18 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
# Negative effective value
# Select 10 Cent but have 1 Cent not be possible because too small
# TODO: bitcoin has [5, 3, 2]
self.assertEqual(
self.assertListEqual(
[4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT],
search(utxo_pool, 10 * CENT, 5000)
)
# Select 0.25 Cent, not possible
self.assertEqual(search(utxo_pool, 0.25 * CENT, 0.5 * CENT), [])
self.assertListEqual(search(utxo_pool, 0.25 * CENT, 0.5 * CENT), [])
# Iteration exhaustion test
utxo_pool, target = self.make_hard_case(17)
selector = CoinSelector(target, 0)
self.assertEqual(selector.select(utxo_pool, 'branch_and_bound'), [])
self.assertListEqual(selector.select(utxo_pool, 'branch_and_bound'), [])
self.assertEqual(selector.tries, MAXIMUM_TRIES) # Should exhaust
utxo_pool, target = self.make_hard_case(14)
self.assertIsNotNone(search(utxo_pool, target, 0)) # Should not exhaust
@@ -181,7 +181,7 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
utxo(7 * CENT),
utxo(2 * CENT)
] + [utxo(5 * CENT)]*50000)
self.assertEqual(
self.assertListEqual(
[7 * CENT, 7 * CENT, 7 * CENT, 7 * CENT, 2 * CENT],
search(utxo_pool, 30 * CENT, 5000)
)
@@ -189,4 +189,4 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
# Select 1 Cent with pool of only greater than 5 Cent
utxo_pool = self.estimates(utxo(i * CENT) for i in range(5, 21))
for _ in range(100):
self.assertEqual(search(utxo_pool, 1 * CENT, 2 * CENT), [])
self.assertListEqual(search(utxo_pool, 1 * CENT, 2 * CENT), [])
+172 -58
View File
@@ -1,7 +1,10 @@
import sys
import os
import unittest
import sqlite3
import tempfile
import os
import asyncio
from concurrent.futures.thread import ThreadPoolExecutor
from torba.client.wallet import Wallet
from torba.client.constants import COIN
@@ -57,17 +60,17 @@ class TestAIOSQLite(AsyncioTestCase):
class TestQueryBuilder(unittest.TestCase):
def test_dot(self):
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.position': 18}),
('txo.position = :txo_position0', {'txo_position0': 18})
)
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.position#6': 18}),
('txo.position = :txo_position6', {'txo_position6': 18})
)
def test_any(self):
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({
'ages__any': {
'txo.age__gt': 18,
@@ -81,41 +84,41 @@ class TestQueryBuilder(unittest.TestCase):
)
def test_in(self):
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.age__in#2': [18, 38]}),
('txo.age IN (:txo_age__in2_0, :txo_age__in2_1)', {
'txo_age__in2_0': 18,
'txo_age__in2_1': 38
})
)
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.name__in': ('abc123', 'def456')}),
('txo.name IN (:txo_name__in0_0, :txo_name__in0_1)', {
'txo_name__in0_0': 'abc123',
'txo_name__in0_1': 'def456'
})
)
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
('txo.age IN (SELECT age from ages_table)', {})
)
def test_not_in(self):
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.age__not_in': [18, 38]}),
('txo.age NOT IN (:txo_age__not_in0_0, :txo_age__not_in0_1)', {
'txo_age__not_in0_0': 18,
'txo_age__not_in0_1': 38
})
)
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.name__not_in': ('abc123', 'def456')}),
('txo.name NOT IN (:txo_name__not_in0_0, :txo_name__not_in0_1)', {
'txo_name__not_in0_0': 'abc123',
'txo_name__not_in0_1': 'def456'
})
)
self.assertEqual(
self.assertTupleEqual(
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
('txo.age NOT IN (SELECT age from ages_table)', {})
)
@@ -125,11 +128,11 @@ class TestQueryBuilder(unittest.TestCase):
constraints_to_sql({'ages__in': 9})
def test_query(self):
self.assertEqual(
self.assertTupleEqual(
query("select * from foo"),
("select * from foo", {})
)
self.assertEqual(
self.assertTupleEqual(
query(
"select * from foo",
a__not='b', b__in='select * from blah where c=:$c',
@@ -143,11 +146,11 @@ class TestQueryBuilder(unittest.TestCase):
)
def test_query_order_by(self):
self.assertEqual(
self.assertTupleEqual(
query("select * from foo", order_by='foo'),
("select * from foo ORDER BY foo", {})
)
self.assertEqual(
self.assertTupleEqual(
query("select * from foo", order_by=['foo', 'bar']),
("select * from foo ORDER BY foo, bar", {})
)
@@ -155,15 +158,15 @@ class TestQueryBuilder(unittest.TestCase):
query("select * from foo", order_by={'foo': 'bar'})
def test_query_limit_offset(self):
self.assertEqual(
self.assertTupleEqual(
query("select * from foo", limit=10),
("select * from foo LIMIT 10", {})
)
self.assertEqual(
self.assertTupleEqual(
query("select * from foo", offset=10),
("select * from foo OFFSET 10", {})
)
self.assertEqual(
self.assertTupleEqual(
query("select * from foo", limit=20, offset=10),
("select * from foo LIMIT 20 OFFSET 10", {})
)
@@ -185,7 +188,7 @@ class TestQueryBuilder(unittest.TestCase):
"(one LIKE 'o' OR two = 2) AND "
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
"ORDER BY b LIMIT 10",
"ORDER BY b LIMIT 10"
)
@@ -196,13 +199,14 @@ class TestQueries(AsyncioTestCase):
'db': ledger_class.database_class(':memory:'),
'headers': ledger_class.headers_class(':memory:'),
})
self.wallet = Wallet()
await self.ledger.db.open()
async def asyncTearDown(self):
await self.ledger.db.close()
async def create_account(self):
account = self.ledger.account_class.generate(self.ledger, Wallet())
async def create_account(self, wallet=None):
account = self.ledger.account_class.generate(self.ledger, wallet or self.wallet)
await account.ensure_address_gap()
return account
@@ -260,8 +264,9 @@ class TestQueries(AsyncioTestCase):
for height in range(1, 1200):
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
for limit in range(variable_limit-2, variable_limit+2):
txs = await self.ledger.get_transactions(limit=limit, order_by='height asc')
for limit in range(variable_limit - 2, variable_limit + 2):
txs = await self.ledger.get_transactions(
accounts=self.wallet.accounts, limit=limit, order_by='height asc')
self.assertEqual(len(txs), limit)
inputs, outputs, last_tx = set(), set(), txs[0]
for tx in txs[1:]:
@@ -271,19 +276,23 @@ class TestQueries(AsyncioTestCase):
last_tx = tx
async def test_queries(self):
self.assertEqual(0, await self.ledger.db.get_address_count())
account1 = await self.create_account()
self.assertEqual(26, await self.ledger.db.get_address_count())
account2 = await self.create_account()
self.assertEqual(52, await self.ledger.db.get_address_count())
wallet1 = Wallet()
account1 = await self.create_account(wallet1)
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1]))
wallet2 = Wallet()
account2 = await self.create_account(wallet2)
account3 = await self.create_account(wallet2)
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2]))
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3]))
self.assertEqual(0, await self.ledger.db.get_utxo_count())
self.assertEqual([], await self.ledger.db.get_utxos())
self.assertListEqual([], await self.ledger.db.get_utxos())
self.assertEqual(0, await self.ledger.db.get_txo_count())
self.assertEqual(0, await self.ledger.db.get_balance())
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
tx1 = await self.create_tx_from_nothing(account1, 1)
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1]))
@@ -291,20 +300,28 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance())
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2)
tx2b = await self.create_tx_from_nothing(account3, 2)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3]))
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance())
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3]))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
@@ -313,23 +330,25 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance())
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual([3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertListEqual([3, 2, 1], [tx.height for tx in txs])
txs = await self.ledger.db.get_transactions(accounts=[account1])
self.assertEqual([tx2.id, tx1.id], [tx.id for tx in txs])
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
txs = await self.ledger.db.get_transactions(accounts=[account2])
self.assertEqual([tx3.id, tx2.id], [tx.id for tx in txs])
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
@@ -340,21 +359,21 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(tx.id, tx2.id)
self.assertEqual(tx.inputs[0].is_my_account, False)
self.assertEqual(tx.outputs[0].is_my_account, False)
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account1])
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
self.assertEqual(tx.inputs[0].is_my_account, True)
self.assertEqual(tx.outputs[0].is_my_account, False)
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account2])
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
self.assertEqual(tx.inputs[0].is_my_account, False)
self.assertEqual(tx.outputs[0].is_my_account, True)
# height 0 sorted to the top with the rest in descending order
tx4 = await self.create_tx_from_nothing(account1, 0)
txos = await self.ledger.db.get_txos()
self.assertEqual([0, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertEqual([tx4.id, tx2.id, tx1.id], [txo.tx_ref.id for txo in txos])
self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
class TestUpgrade(AsyncioTestCase):
@@ -379,13 +398,13 @@ class TestUpgrade(AsyncioTestCase):
def add_address(self, address):
with sqlite3.connect(self.path) as conn:
conn.execute("""
INSERT INTO pubkey_address (address, account, chain, position, pubkey)
VALUES (?, 'account1', 0, 0, 'pubkey blob')
INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth)
VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0)
""", (address,))
def get_addresses(self):
with sqlite3.connect(self.path) as conn:
sql = "SELECT address FROM pubkey_address ORDER BY address;"
sql = "SELECT address FROM account_address ORDER BY address;"
return [col[0] for col in conn.execute(sql).fetchall()]
async def test_reset_on_version_change(self):
@@ -396,10 +415,10 @@ class TestUpgrade(AsyncioTestCase):
# initial open, pre-version enabled db
self.ledger.db.SCHEMA_VERSION = None
self.assertEqual(self.get_tables(), [])
self.assertListEqual(self.get_tables(), [])
await self.ledger.db.open()
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo'])
self.assertEqual(self.get_addresses(), [])
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo'])
self.assertListEqual(self.get_addresses(), [])
self.add_address('address1')
await self.ledger.db.close()
@@ -407,18 +426,18 @@ class TestUpgrade(AsyncioTestCase):
self.ledger.db.SCHEMA_VERSION = '1.0'
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertListEqual(self.get_addresses(), []) # address1 deleted during version upgrade
self.add_address('address2')
await self.ledger.db.close()
# nothing changes
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), ['address2'])
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertListEqual(self.get_addresses(), ['address2'])
await self.ledger.db.close()
# upgrade version, database reset
@@ -428,6 +447,101 @@ class TestUpgrade(AsyncioTestCase):
"""
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.1')
self.assertEqual(self.get_tables(), ['foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # all tables got reset
self.assertListEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertListEqual(self.get_addresses(), []) # all tables got reset
await self.ledger.db.close()
class TestSQLiteRace(AsyncioTestCase):
max_misuse_attempts = 40000
def setup_db(self):
self.db = sqlite3.connect(":memory:", isolation_level=None)
self.db.executescript(
"create table test1 (id text primary key not null, val text);\n" +
"create table test2 (id text primary key not null, val text);\n" +
"\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000))
)
async def asyncSetUp(self):
self.executor = ThreadPoolExecutor(1)
await self.loop.run_in_executor(self.executor, self.setup_db)
async def asyncTearDown(self):
await self.loop.run_in_executor(self.executor, self.db.close)
self.executor.shutdown()
async def test_binding_param_0_error(self):
# test real param 0 binding errors
for supported_type in [str, int, bytes]:
await self.loop.run_in_executor(
self.executor, self.db.executemany, "insert into test2 values (?, NULL)",
[(supported_type(1), ), (supported_type(2), )]
)
await self.loop.run_in_executor(
self.executor, self.db.execute, "delete from test2 where id in (1, 2)"
)
for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]:
try:
await self.loop.run_in_executor(
self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)",
[(unsupported_type(1), ), (unsupported_type(2), )]
)
self.assertTrue(False)
except sqlite3.InterfaceError as err:
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
async def test_unhandled_sqlite_misuse(self):
# test SQLITE_MISUSE being incorrectly raised as a param 0 binding error
attempts = 0
python_version = sys.version.split('\n')[0].rstrip(' ')
try:
while attempts < self.max_misuse_attempts:
f1 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, self.db.executemany, "update test1 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
f2 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, self.db.executemany, "update test2 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
attempts += 1
await asyncio.gather(f1, f2)
print(f"\nsqlite3 {sqlite3.version}/python {python_version} "
f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition")
self.assertTrue(False, 'this test failing means either the sqlite race conditions '
'have been fixed in cpython or the test max_attempts needs to be increased')
except sqlite3.InterfaceError as err:
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE "
f"after {attempts} attempts of the race condition")
@unittest.SkipTest
async def test_fetchall_prevents_sqlite_misuse(self):
# test that calling fetchall sufficiently avoids the race
attempts = 0
def executemany_fetchall(query, params):
self.db.executemany(query, params).fetchall()
while attempts < self.max_misuse_attempts:
f1 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, executemany_fetchall, "update test1 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
f2 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, executemany_fetchall, "update test2 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
attempts += 1
await asyncio.gather(f1, f2)
+1 -1
View File
@@ -23,7 +23,7 @@ class TestAESEncryptDecrypt(TestCase):
'ZmZmZmZmZmZmZmZmZmZmZjlrKptoKD+MFwDxcg3XtCD9qz8UWhEhq/TVJT5+Mtp2a8sE'
'CaO6WQj7fYsWGu2Hvbc0qYqxdN0HeTsiO+cZRo3eJISgr3F+rXFYi5oSBlD2'
)
self.assertEqual(
self.assertTupleEqual(
aes_decrypt(self.password, self.expected),
(self.message, b'f' * 16)
)
@@ -35,7 +35,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
async def test_serialization(self):
h = await self.get_headers()
self.assertEqual(h[0], {
self.assertDictEqual(h[0], {
'bits': 486604799,
'block_height': 0,
'merkle_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
@@ -44,7 +44,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
'timestamp': 1231006505,
'version': 1
})
self.assertEqual(h[self.RETARGET_BLOCK-1], {
self.assertDictEqual(h[self.RETARGET_BLOCK-1], {
'bits': 486604799,
'block_height': 32255,
'merkle_root': b'89b4f223789e40b5b475af6483bb05bceda54059e17d2053334b358f6bb310ac',
@@ -53,7 +53,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
'timestamp': 1262152739,
'version': 1
})
self.assertEqual(h[self.RETARGET_BLOCK], {
self.assertDictEqual(h[self.RETARGET_BLOCK], {
'bits': 486594666,
'block_height': 32256,
'merkle_root': b'64b5e5f5a262f47af443a0120609206a3305877693edfe03e994f20a024ab627',
@@ -62,7 +62,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
'timestamp': 1262153464,
'version': 1
})
self.assertEqual(h[self.RETARGET_BLOCK+1], {
self.assertDictEqual(h[self.RETARGET_BLOCK+1], {
'bits': 486594666,
'block_height': 32257,
'merkle_root': b'4d1488981f08b3037878193297dbac701a2054e0f803d4424fe6a4d763d62334',
+6 -6
View File
@@ -90,8 +90,8 @@ class TestSynchronization(LedgerTestCase):
'abcd03': hexlify(get_transaction(get_output(3)).raw),
})
await self.ledger.update_history(address, '')
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
@@ -104,16 +104,16 @@ class TestSynchronization(LedgerTestCase):
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
await self.ledger.update_history(address, '')
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, [])
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])
self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 3})
self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
await self.ledger.update_history(address, '')
self.assertEqual(self.ledger.network.get_history_called, [address])
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd04'])
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd04'])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
address_details['history'],
+12 -12
View File
@@ -24,14 +24,14 @@ def parse(opcodes, source):
class TestScriptTemplates(unittest.TestCase):
def test_push_data(self):
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_SINGLE('script_hash'),),
(b'abcdef',)
), {
'script_hash': b'abcdef'
}
)
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_SINGLE('first'), PUSH_INTEGER('rating')),
(b'Satoshi', (1000).to_bytes(2, 'little'))
), {
@@ -39,7 +39,7 @@ class TestScriptTemplates(unittest.TestCase):
'rating': 1000,
}
)
self.assertEqual(parse(
self.assertDictEqual(parse(
(OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL),
(OP_HASH160, b'abcdef', OP_EQUAL)
), {
@@ -48,21 +48,21 @@ class TestScriptTemplates(unittest.TestCase):
)
def test_push_data_many(self):
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_MANY('names'),),
(b'amit',)
), {
'names': [b'amit']
}
)
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_MANY('names'),),
(b'jeremy', b'amit', b'victor')
), {
'names': [b'jeremy', b'amit', b'victor']
}
)
self.assertEqual(parse(
self.assertDictEqual(parse(
(OP_HASH160, PUSH_MANY('names'), OP_EQUAL),
(OP_HASH160, b'grin', b'jack', OP_EQUAL)
), {
@@ -71,7 +71,7 @@ class TestScriptTemplates(unittest.TestCase):
)
def test_push_data_mixed(self):
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_SINGLE('CEO'), PUSH_MANY('Devs'), PUSH_SINGLE('CTO'), PUSH_SINGLE('State')),
(b'jeremy', b'lex', b'amit', b'victor', b'jack', b'grin', b'NH')
), {
@@ -83,7 +83,7 @@ class TestScriptTemplates(unittest.TestCase):
)
def test_push_data_many_separated(self):
self.assertEqual(parse(
self.assertDictEqual(parse(
(PUSH_MANY('Chiefs'), OP_HASH160, PUSH_MANY('Devs')),
(b'jeremy', b'grin', OP_HASH160, b'lex', b'jack')
), {
@@ -135,16 +135,16 @@ class TestRedeemScriptHash(unittest.TestCase):
)
subscript1 = src1.values['script']
self.assertEqual(src1.template.name, 'script_hash')
self.assertEqual([hexlify(v) for v in src1.values['signatures']], sigs)
self.assertEqual([hexlify(p) for p in subscript1.values['pubkeys']], pubkeys)
self.assertListEqual([hexlify(v) for v in src1.values['signatures']], sigs)
self.assertListEqual([hexlify(p) for p in subscript1.values['pubkeys']], pubkeys)
self.assertEqual(subscript1.values['signatures_count'], len(sigs))
self.assertEqual(subscript1.values['pubkeys_count'], len(pubkeys))
# now we test that it will round trip
src2 = BaseInputScript(src1.source)
subscript2 = src2.values['script']
self.assertEqual(src2.template.name, 'script_hash')
self.assertEqual([hexlify(v) for v in src2.values['signatures']], sigs)
self.assertEqual([hexlify(p) for p in subscript2.values['pubkeys']], pubkeys)
self.assertListEqual([hexlify(v) for v in src2.values['signatures']], sigs)
self.assertListEqual([hexlify(p) for p in subscript2.values['pubkeys']], pubkeys)
self.assertEqual(subscript2.values['signatures_count'], len(sigs))
self.assertEqual(subscript2.values['pubkeys_count'], len(pubkeys))
return hexlify(src1.source)
@@ -9,7 +9,7 @@ class StreamControllerTestCase(AsyncioTestCase):
controller.stream.listen(on_data=events.append)
controller.add("yo")
controller.add("yo")
self.assertEqual(events, ["yo", "yo"])
self.assertListEqual(events, ["yo", "yo"])
def test_unique_events(self):
events = []
@@ -17,4 +17,4 @@ class StreamControllerTestCase(AsyncioTestCase):
controller.stream.listen(on_data=events.append)
controller.add("yo")
controller.add("yo")
self.assertEqual(events, ["yo"])
self.assertListEqual(events, ["yo"])
@@ -294,9 +294,9 @@ class TransactionIOBalancing(AsyncioTestCase):
[self.txo(3)] # outputs
)
# best UTXO match is 5 (as UTXO 3 will be short 0.02 to cover fees)
self.assertEqual(self.inputs(tx), [5])
self.assertListEqual(self.inputs(tx), [5])
# a change of 1.98 is added to reach balance
self.assertEqual(self.outputs(tx), [3, 1.98])
self.assertListEqual(self.outputs(tx), [3, 1.98])
await self.ledger.release_outputs(utxos)
@@ -306,8 +306,8 @@ class TransactionIOBalancing(AsyncioTestCase):
[self.txo(2.98)] # outputs
)
# best UTXO match is 3 and no change is needed
self.assertEqual(self.inputs(tx), [3])
self.assertEqual(self.outputs(tx), [2.98])
self.assertListEqual(self.inputs(tx), [3])
self.assertListEqual(self.outputs(tx), [2.98])
await self.ledger.release_outputs(utxos)
@@ -317,9 +317,9 @@ class TransactionIOBalancing(AsyncioTestCase):
[self.txo(11)] # outputs
)
# additional input is chosen (UTXO 3)
self.assertEqual([10, 3], self.inputs(tx))
self.assertListEqual([10, 3], self.inputs(tx))
# change is now needed to consume extra input
self.assertEqual([11, 1.96], self.outputs(tx))
self.assertListEqual([11, 1.96], self.outputs(tx))
await self.ledger.release_outputs(utxos)
@@ -328,9 +328,9 @@ class TransactionIOBalancing(AsyncioTestCase):
[self.txi(self.txo(10))], # inputs
[] # outputs
)
self.assertEqual([10], self.inputs(tx))
self.assertListEqual([10], self.inputs(tx))
# missing change added to consume the amount
self.assertEqual([9.98], self.outputs(tx))
self.assertListEqual([9.98], self.outputs(tx))
await self.ledger.release_outputs(utxos)
@@ -340,6 +340,6 @@ class TransactionIOBalancing(AsyncioTestCase):
[] # outputs
)
# UTXO 1 is added to cover some of the fee
self.assertEqual([0.01, 1], self.inputs(tx))
self.assertListEqual([0.01, 1], self.inputs(tx))
# change is now needed to consume extra input
self.assertEqual([0.97], self.outputs(tx))
self.assertListEqual([0.97], self.outputs(tx))
+64 -3
View File
@@ -1,12 +1,13 @@
import tempfile
from binascii import hexlify
from unittest import TestCase, mock
from torba.testcase import AsyncioTestCase
from torba.coin.bitcoinsegwit import MainNetLedger as BTCLedger
from torba.coin.bitcoincash import MainNetLedger as BCHLedger
from torba.client.basemanager import BaseWalletManager
from torba.client.wallet import Wallet, WalletStorage
from torba.client.wallet import Wallet, WalletStorage, TimestampedPreferences
class TestWalletCreation(AsyncioTestCase):
@@ -20,7 +21,7 @@ class TestWalletCreation(AsyncioTestCase):
def test_create_wallet_and_accounts(self):
wallet = Wallet()
self.assertEqual(wallet.name, 'Wallet')
self.assertEqual(wallet.accounts, [])
self.assertListEqual(wallet.accounts, [])
account1 = wallet.generate_account(self.btc_ledger)
wallet.generate_account(self.btc_ledger)
@@ -32,6 +33,7 @@ class TestWalletCreation(AsyncioTestCase):
wallet_dict = {
'version': 1,
'name': 'Main Wallet',
'preferences': {},
'accounts': [
{
'name': 'An Account',
@@ -60,7 +62,7 @@ class TestWalletCreation(AsyncioTestCase):
wallet = Wallet.from_storage(storage, self.manager)
self.assertEqual(wallet.name, 'Main Wallet')
self.assertEqual(
hexlify(wallet.hash), b'9f462b8dd802eb8c913e54f09a09827ebc14abbc13f33baa90d8aec5ae920fc7'
hexlify(wallet.hash), b'1bd61fbe18875cb7828c466022af576104ed861c8a1fdb1dadf5e39417a68483'
)
self.assertEqual(len(wallet.accounts), 1)
account = wallet.default_account
@@ -91,3 +93,62 @@ class TestWalletCreation(AsyncioTestCase):
wallet = Wallet.from_storage(wallet_storage, manager)
self.assertEqual(account.public_key.address, wallet.default_account.public_key.address)
def test_merge(self):
wallet1 = Wallet()
wallet1.preferences['one'] = 1
wallet1.preferences['conflict'] = 1
wallet1.generate_account(self.btc_ledger)
wallet2 = Wallet()
wallet2.preferences['two'] = 2
wallet2.preferences['conflict'] = 2 # will be more recent
wallet2.generate_account(self.btc_ledger)
self.assertEqual(len(wallet1.accounts), 1)
self.assertEqual(wallet1.preferences, {'one': 1, 'conflict': 1})
added = wallet1.merge(self.manager, 'password', wallet2.pack('password'))
self.assertEqual(added[0].id, wallet2.default_account.id)
self.assertEqual(len(wallet1.accounts), 2)
self.assertEqual(wallet1.accounts[1].id, wallet2.default_account.id)
self.assertEqual(wallet1.preferences, {'one': 1, 'two': 2, 'conflict': 2})
class TestTimestampedPreferences(TestCase):
def test_init(self):
p = TimestampedPreferences()
p['one'] = 1
p2 = TimestampedPreferences(p.data)
self.assertEqual(p2['one'], 1)
def test_hash(self):
p = TimestampedPreferences()
self.assertEqual(
hexlify(p.hash), b'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a'
)
with mock.patch('time.time', mock.Mock(return_value=12345)):
p['one'] = 1
self.assertEqual(
hexlify(p.hash), b'c9e82bf4cb099dd0125f78fa381b21a8131af601917eb531e1f5f980f8f3da66'
)
def test_merge(self):
p1 = TimestampedPreferences()
p2 = TimestampedPreferences()
with mock.patch('time.time', mock.Mock(return_value=10)):
p1['one'] = 1
p1['conflict'] = 1
with mock.patch('time.time', mock.Mock(return_value=20)):
p2['two'] = 2
p2['conflict'] = 2
# conflict in p2 overrides conflict in p1
p1.merge(p2.data)
self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 2})
# have a newer conflict in p1 so it is not overridden this time
with mock.patch('time.time', mock.Mock(return_value=21)):
p1['conflict'] = 1
p1.merge(p2.data)
self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 1})
+42 -42
View File
@@ -1,3 +1,4 @@
import os
import json
import time
import asyncio
@@ -42,7 +43,7 @@ class AddressManager:
d['change'] = change_dict
return d
def apply(self, d: dict):
def merge(self, d: dict):
pass
def to_dict_instance(self) -> Optional[dict]:
@@ -85,7 +86,7 @@ class AddressManager:
class HierarchicalDeterministic(AddressManager):
""" Implements simple version of Bitcoin Hierarchical Deterministic key management. """
name = "deterministic-chain"
name: str = "deterministic-chain"
__slots__ = 'gap', 'maximum_uses_per_address'
@@ -101,7 +102,7 @@ class HierarchicalDeterministic(AddressManager):
cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
)
def apply(self, d: dict):
def merge(self, d: dict):
self.gap = d.get('gap', self.gap)
self.maximum_uses_per_address = d.get('maximum_uses_per_address', self.maximum_uses_per_address)
@@ -115,7 +116,7 @@ class HierarchicalDeterministic(AddressManager):
return self.account.public_key.child(self.chain_number).child(index)
async def get_max_gap(self) -> int:
addresses = await self._query_addresses(order_by="position ASC")
addresses = await self._query_addresses(order_by="n asc")
max_gap = 0
current_gap = 0
for address in addresses:
@@ -128,7 +129,7 @@ class HierarchicalDeterministic(AddressManager):
async def ensure_address_gap(self) -> List[str]:
async with self.address_generator_lock:
addresses = await self._query_addresses(limit=self.gap, order_by="position DESC")
addresses = await self._query_addresses(limit=self.gap, order_by="n desc")
existing_gap = 0
for address in addresses:
@@ -140,7 +141,7 @@ class HierarchicalDeterministic(AddressManager):
if existing_gap == self.gap:
return []
start = addresses[0]['position']+1 if addresses else 0
start = addresses[0]['pubkey'].n+1 if addresses else 0
end = start + (self.gap - existing_gap)
new_keys = await self._generate_keys(start, end-1)
await self.account.ledger.announce_addresses(self, new_keys)
@@ -149,22 +150,22 @@ class HierarchicalDeterministic(AddressManager):
async def _generate_keys(self, start: int, end: int) -> List[str]:
if not self.address_generator_lock.locked():
raise RuntimeError('Should not be called outside of address_generator_lock.')
keys = [(index, self.public_key.child(index)) for index in range(start, end+1)]
keys = [self.public_key.child(index) for index in range(start, end+1)]
await self.account.ledger.db.add_keys(self.account, self.chain_number, keys)
return [key[1].address for key in keys]
return [key.address for key in keys]
def get_address_records(self, only_usable: bool = False, **constraints):
if only_usable:
constraints['used_times__lt'] = self.maximum_uses_per_address
if 'order_by' not in constraints:
constraints['order_by'] = "used_times ASC, position ASC"
constraints['order_by'] = "used_times asc, n asc"
return self._query_addresses(**constraints)
class SingleKey(AddressManager):
""" Single Key address manager always returns the same address for all operations. """
name = "single-address"
name: str = "single-address"
__slots__ = ()
@@ -190,9 +191,7 @@ class SingleKey(AddressManager):
async with self.address_generator_lock:
exists = await self.get_address_records()
if not exists:
await self.account.ledger.db.add_keys(
self.account, self.chain_number, [(0, self.public_key)]
)
await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key])
new_keys = [self.public_key.address]
await self.account.ledger.announce_addresses(self, new_keys)
return new_keys
@@ -223,12 +222,8 @@ class BaseAccount:
self.seed = seed
self.modified_on = modified_on
self.private_key_string = private_key_string
self.password: Optional[str] = None
self.private_key_encryption_init_vector: Optional[bytes] = None
self.seed_encryption_init_vector: Optional[bytes] = None
self.init_vectors: Dict[str, bytes] = {}
self.encrypted = encrypted
self.serialize_encrypted = encrypted
self.private_key = private_key
self.public_key = public_key
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
@@ -238,6 +233,12 @@ class BaseAccount:
ledger.add_account(self)
wallet.add_account(self)
def get_init_vector(self, key) -> Optional[bytes]:
init_vector = self.init_vectors.get(key, None)
if init_vector is None:
init_vector = self.init_vectors[key] = os.urandom(16)
return init_vector
@classmethod
def generate(cls, ledger: 'baseledger.BaseLedger', wallet: 'basewallet.Wallet',
name: str = None, address_generator: dict = None):
@@ -291,28 +292,27 @@ class BaseAccount:
modified_on=d.get('modified_on', time.time())
)
def to_dict(self):
def to_dict(self, encrypt_password: str = None):
private_key_string, seed = self.private_key_string, self.seed
if not self.encrypted and self.private_key:
private_key_string = self.private_key.extended_key_string()
if not self.encrypted and self.serialize_encrypted:
assert None not in [self.seed_encryption_init_vector, self.private_key_encryption_init_vector]
if not self.encrypted and encrypt_password:
private_key_string = aes_encrypt(
self.password, private_key_string, self.private_key_encryption_init_vector
encrypt_password, private_key_string, self.get_init_vector('private_key')
)
seed = aes_encrypt(self.password, self.seed, self.seed_encryption_init_vector)
seed = aes_encrypt(encrypt_password, self.seed, self.get_init_vector('seed'))
return {
'ledger': self.ledger.get_id(),
'name': self.name,
'seed': seed,
'encrypted': self.serialize_encrypted,
'encrypted': bool(self.encrypted or encrypt_password),
'private_key': private_key_string,
'public_key': self.public_key.extended_key_string(),
'address_generator': self.address_generator.to_dict(self.receiving, self.change),
'modified_on': self.modified_on
}
def apply(self, d: dict):
def merge(self, d: dict):
if d.get('modified_on', 0) > self.modified_on:
self.name = d['name']
self.modified_on = d.get('modified_on', time.time())
@@ -320,10 +320,11 @@ class BaseAccount:
for chain_name in ('change', 'receiving'):
if chain_name in d['address_generator']:
chain_object = getattr(self, chain_name)
chain_object.apply(d['address_generator'][chain_name])
chain_object.merge(d['address_generator'][chain_name])
@property
def hash(self) -> bytes:
assert not self.encrypted, "Cannot hash an encrypted account."
return sha256(json.dumps(self.to_dict()).encode())
async def get_details(self, show_seed=False, **kwargs):
@@ -341,42 +342,41 @@ class BaseAccount:
details['seed'] = self.seed
return details
def decrypt(self, password: str) -> None:
def decrypt(self, password: str) -> bool:
assert self.encrypted, "Key is not encrypted."
try:
seed, seed_iv = aes_decrypt(password, self.seed)
pk_string, pk_iv = aes_decrypt(password, self.private_key_string)
except ValueError: # failed to remove padding, password is wrong
return
return False
try:
Mnemonic().mnemonic_decode(seed)
except IndexError: # failed to decode the seed, this either means it decrypted and is invalid
# or that we hit an edge case where an incorrect password gave valid padding
return
return False
try:
private_key = from_extended_key_string(
self.ledger, pk_string
)
except (TypeError, ValueError):
return
return False
self.seed = seed
self.seed_encryption_init_vector = seed_iv
self.private_key = private_key
self.private_key_encryption_init_vector = pk_iv
self.password = password
self.init_vectors['seed'] = seed_iv
self.init_vectors['private_key'] = pk_iv
self.encrypted = False
return True
def encrypt(self, password: str) -> None:
def encrypt(self, password: str) -> bool:
assert not self.encrypted, "Key is already encrypted."
assert isinstance(self.private_key, PrivateKey)
self.seed = aes_encrypt(password, self.seed, self.seed_encryption_init_vector)
self.seed = aes_encrypt(password, self.seed, self.get_init_vector('seed'))
self.private_key_string = aes_encrypt(
password, self.private_key.extended_key_string(), self.private_key_encryption_init_vector
password, self.private_key.extended_key_string(), self.get_init_vector('private_key')
)
self.private_key = None
self.password = None
self.encrypted = True
return True
async def ensure_address_gap(self):
addresses = []
@@ -417,16 +417,16 @@ class BaseAccount:
}
def get_utxos(self, **constraints):
return self.ledger.get_utxos(account=self, **constraints)
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
def get_utxo_count(self, **constraints):
return self.ledger.get_utxo_count(account=self, **constraints)
return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints)
def get_transactions(self, **constraints):
return self.ledger.get_transactions(account=self, **constraints)
return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints)
def get_transaction_count(self, **constraints):
return self.ledger.get_transaction_count(account=self, **constraints)
return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints)
async def fund(self, to_account, amount=None, everything=False,
outputs=1, broadcast=False, **constraints):
+104 -85
View File
@@ -3,12 +3,12 @@ import asyncio
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Optional
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
import sqlite3
from torba.client.basetransaction import BaseTransaction, TXRefImmutable
from torba.client.baseaccount import BaseAccount
from torba.client.bip32 import PubKey
log = logging.getLogger(__name__)
@@ -19,6 +19,7 @@ class AIOSQLite:
# has to be single threaded as there is no mapping of thread:connection
self.executor = ThreadPoolExecutor(max_workers=1)
self.connection: sqlite3.Connection = None
self._closing = False
@classmethod
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
@@ -29,14 +30,12 @@ class AIOSQLite:
return db
async def close(self):
def __close(conn):
self.executor.submit(conn.close)
self.executor.shutdown(wait=True)
conn = self.connection
if not conn:
if self._closing:
return
self._closing = True
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
self.executor.shutdown(wait=True)
self.connection = None
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
def executemany(self, sql: str, params: Iterable):
params = params if params is not None else []
@@ -87,10 +86,10 @@ class AIOSQLite:
if not foreign_keys_enabled:
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
try:
self.connection.execute('pragma foreign_keys=off')
self.connection.execute('pragma foreign_keys=off').fetchone()
return self.__run_transaction(fun, *args, **kwargs)
finally:
self.connection.execute('pragma foreign_keys=on')
self.connection.execute('pragma foreign_keys=on').fetchone()
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
@@ -103,20 +102,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
if not key:
sql.append(constraint)
continue
elif key.startswith('$'):
if key.startswith('$'):
values[key] = constraint
continue
elif key.endswith('__not'):
if key.endswith('__not'):
col, op = col[:-len('__not')], '!='
elif key.endswith('__is_null'):
col = col[:-len('__is_null')]
sql.append(f'{col} IS NULL')
continue
elif key.endswith('__is_not_null'):
if key.endswith('__is_not_null'):
col = col[:-len('__is_not_null')]
sql.append(f'{col} IS NOT NULL')
continue
elif key.endswith('__lt'):
if key.endswith('__lt'):
col, op = col[:-len('__lt')], '<'
elif key.endswith('__lte'):
col, op = col[:-len('__lte')], '<='
@@ -150,7 +149,7 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
sql.append(f'({where})')
values.update(subvalues)
continue
elif key.endswith('__and'):
if key.endswith('__and'):
where, subvalues = constraints_to_sql(constraint, ' AND ', key+tag+'_')
sql.append(f'({where})')
values.update(subvalues)
@@ -160,15 +159,14 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
return joiner.join(sql) if sql else '', values
def query(select, **constraints):
def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
sql = [select]
limit = constraints.pop('limit', None)
offset = constraints.pop('offset', None)
order_by = constraints.pop('order_by', None)
constraints.pop('my_accounts', None)
accounts = constraints.pop('accounts', None)
if accounts is not None:
accounts = constraints.pop('accounts', [])
if accounts:
constraints['account__in'] = [a.public_key.address for a in accounts]
where, values = constraints_to_sql(constraints)
@@ -286,26 +284,33 @@ class SQLiteMixin:
class BaseDatabase(SQLiteMixin):
SCHEMA_VERSION = "1.0"
SCHEMA_VERSION = "1.1"
PRAGMAS = """
pragma journal_mode=WAL;
"""
CREATE_ACCOUNT_TABLE = """
create table if not exists account_address (
account text not null,
address text not null,
chain integer not null,
pubkey blob not null,
chain_code blob not null,
n integer not null,
depth integer not null,
primary key (account, address)
);
create index if not exists address_account_idx on account_address (address, account);
"""
CREATE_PUBKEY_ADDRESS_TABLE = """
create table if not exists pubkey_address (
address text primary key,
account text not null,
chain integer not null,
position integer not null,
pubkey blob not null,
history text,
used_times integer not null default 0
);
"""
CREATE_PUBKEY_ADDRESS_INDEX = """
create index if not exists pubkey_address_account_idx on pubkey_address (account);
"""
CREATE_TX_TABLE = """
create table if not exists tx (
@@ -327,8 +332,6 @@ class BaseDatabase(SQLiteMixin):
script blob not null,
is_reserved boolean not null default 0
);
"""
CREATE_TXO_INDEX = """
create index if not exists txo_address_idx on txo (address);
"""
@@ -338,21 +341,17 @@ class BaseDatabase(SQLiteMixin):
txoid text references txo,
address text references pubkey_address
);
"""
CREATE_TXI_INDEX = """
create index if not exists txi_address_idx on txi (address);
create index if not exists txi_txoid_idx on txi (txoid);
"""
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_TX_TABLE +
CREATE_ACCOUNT_TABLE +
CREATE_PUBKEY_ADDRESS_TABLE +
CREATE_PUBKEY_ADDRESS_INDEX +
CREATE_TX_TABLE +
CREATE_TXO_TABLE +
CREATE_TXO_INDEX +
CREATE_TXI_TABLE +
CREATE_TXI_INDEX
CREATE_TXI_TABLE
)
@staticmethod
@@ -377,10 +376,10 @@ class BaseDatabase(SQLiteMixin):
}
async def insert_transaction(self, tx):
await self.db.execute(*self._insert_sql('tx', self.tx_to_row(tx)))
await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx)))
async def update_transaction(self, tx):
await self.db.execute(*self._update_sql("tx", {
await self.db.execute_fetchall(*self._update_sql("tx", {
'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
}, 'txid = ?', (tx.id,)))
@@ -391,7 +390,7 @@ class BaseDatabase(SQLiteMixin):
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
conn.execute(*self._insert_sql(
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
))
)).fetchall()
elif txo.script.is_pay_script_hash:
# TODO: implement script hash payments
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
@@ -404,7 +403,7 @@ class BaseDatabase(SQLiteMixin):
'txid': tx.id,
'txoid': txo.id,
'address': address,
}, ignore_duplicate=True))
}, ignore_duplicate=True)).fetchall()
conn.execute(
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
@@ -435,25 +434,22 @@ class BaseDatabase(SQLiteMixin):
async def select_transactions(self, cols, accounts=None, **constraints):
if not set(constraints) & {'txid', 'txid__in'}:
assert accounts is not None, "'accounts' argument required when no 'txid' constraint"
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
constraints.update({
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
})
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
where = f" WHERE account_address.account IN ({account_values})"
constraints['txid__in'] = f"""
SELECT txo.txid FROM txo
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
UNION
SELECT txi.txid FROM txi
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
"""
return await self.db.execute_fetchall(
*query("SELECT {} FROM tx".format(cols), **constraints)
)
async def get_transactions(self, **constraints):
accounts = constraints.get('accounts', None)
async def get_transactions(self, wallet=None, **constraints):
tx_rows = await self.select_transactions(
'txid, raw, height, position, is_verified',
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
@@ -478,7 +474,7 @@ class BaseDatabase(SQLiteMixin):
annotated_txos.update({
txo.id: txo for txo in
(await self.get_txos(
my_accounts=accounts,
wallet=wallet,
txid__in=txids[offset:offset+step],
))
})
@@ -488,7 +484,7 @@ class BaseDatabase(SQLiteMixin):
referenced_txos.update({
txo.id: txo for txo in
(await self.get_txos(
my_accounts=accounts,
wallet=wallet,
txoid__in=txi_txoids[offset:offset+step],
))
})
@@ -508,6 +504,7 @@ class BaseDatabase(SQLiteMixin):
return txs
async def get_transaction_count(self, **constraints):
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -520,23 +517,24 @@ class BaseDatabase(SQLiteMixin):
return txs[0]
async def select_txos(self, cols, **constraints):
return await self.db.execute_fetchall(*query(
"SELECT {} FROM txo"
" JOIN pubkey_address USING (address)"
" JOIN tx USING (txid)".format(cols), **constraints
))
sql = "SELECT {} FROM txo JOIN tx USING (txid)"
if 'accounts' in constraints:
sql += " JOIN account_address USING (address)"
return await self.db.execute_fetchall(*query(sql.format(cols), **constraints))
async def get_txos(self, my_accounts=None, no_tx=False, **constraints):
my_accounts = [
(a.public_key.address if isinstance(a, BaseAccount) else a)
for a in (my_accounts or constraints.get('accounts', []))
]
async def get_txos(self, wallet=None, no_tx=False, **constraints):
my_accounts = set(a.public_key.address for a in wallet.accounts) if wallet else set()
if 'order_by' not in constraints:
constraints['order_by'] = [
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"]
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
]
rows = await self.select_txos(
"tx.txid, raw, tx.height, tx.position, tx.is_verified, "
"txo.position, chain, account, amount, script",
"""
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
select group_concat(account||"|"||chain) from account_address
where account_address.address=txo.address
)
""",
**constraints
)
txos = []
@@ -545,8 +543,8 @@ class BaseDatabase(SQLiteMixin):
for row in rows:
if no_tx:
txo = output_class(
amount=row[8],
script=output_class.script_class(row[9]),
amount=row[6],
script=output_class.script_class(row[7]),
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
position=row[5]
)
@@ -556,12 +554,18 @@ class BaseDatabase(SQLiteMixin):
row[1], height=row[2], position=row[3], is_verified=row[4]
)
txo = txs[row[0]].outputs[row[5]]
txo.is_change = row[6] == 1
txo.is_my_account = row[7] in my_accounts
row_accounts = dict(a.split('|') for a in row[8].split(','))
account_match = set(row_accounts) & my_accounts
if account_match:
txo.is_my_account = True
txo.is_change = row_accounts[account_match.pop()] == '1'
else:
txo.is_change = txo.is_my_account = False
txos.append(txo)
return txos
async def get_txo_count(self, **constraints):
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -581,45 +585,60 @@ class BaseDatabase(SQLiteMixin):
self.constrain_utxo(constraints)
return self.get_txo_count(**constraints)
async def get_balance(self, **constraints):
async def get_balance(self, wallet=None, accounts=None, **constraints):
assert wallet or accounts, \
"'wallet' or 'accounts' constraints required to calculate balance"
constraints['accounts'] = accounts or wallet.accounts
self.constrain_utxo(constraints)
balance = await self.select_txos('SUM(amount)', **constraints)
return balance[0][0] or 0
async def select_addresses(self, cols, **constraints):
return await self.db.execute_fetchall(*query(
"SELECT {} FROM pubkey_address".format(cols), **constraints
"SELECT {} FROM pubkey_address JOIN account_address USING (address)".format(cols),
**constraints
))
async def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'),
**constraints):
addresses = await self.select_addresses(', '.join(cols), **constraints)
return rows_to_dict(addresses, cols)
async def get_addresses(self, cols=None, **constraints):
cols = cols or (
'address', 'account', 'chain', 'history', 'used_times',
'pubkey', 'chain_code', 'n', 'depth'
)
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
for address in addresses:
address['pubkey'] = PubKey(
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
address.pop('n'), address.pop('depth')
)
return addresses
async def get_address_count(self, **constraints):
count = await self.select_addresses('count(*)', **constraints)
return count[0][0]
async def get_address(self, **constraints):
addresses = await self.get_addresses(
cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
limit=1, **constraints
)
addresses = await self.get_addresses(limit=1, **constraints)
if addresses:
return addresses[0]
async def add_keys(self, account, chain, keys):
async def add_keys(self, account, chain, pubkeys):
await self.db.executemany(
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
(
(pubkey.address, account.public_key.address, chain, position,
sqlite3.Binary(pubkey.pubkey_bytes))
for position, pubkey in keys
)
"insert or ignore into account_address "
"(account, address, chain, pubkey, chain_code, n, depth) values "
"(?, ?, ?, ?, ?, ?, ?)", ((
account.id, k.address, chain,
sqlite3.Binary(k.pubkey_bytes),
sqlite3.Binary(k.chain_code),
k.n, k.depth
) for k in pubkeys)
)
await self.db.executemany(
"insert or ignore into pubkey_address (address) values (?)",
((pubkey.address,) for pubkey in pubkeys)
)
async def _set_address_history(self, address, history):
await self.db.execute(
await self.db.execute_fetchall(
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
(history, history.count(':')//2, address)
)
+24 -28
View File
@@ -19,6 +19,7 @@ from torba.client.coinselection import CoinSelector
from torba.client.constants import COIN, NULL_HASH32
from torba.stream import StreamController
from torba.client.hash import hash160, double_sha256, sha256, Base58
from torba.client.bip32 import PubKey, PrivateKey
log = logging.getLogger(__name__)
@@ -160,7 +161,7 @@ class BaseLedger(metaclass=LedgerRegistry):
@classmethod
def is_valid_address(cls, address):
decoded = Base58.decode(address)
decoded = Base58.decode_check(address)
return decoded[0] == cls.pubkey_address_prefix[0]
@classmethod
@@ -178,27 +179,29 @@ class BaseLedger(metaclass=LedgerRegistry):
def add_account(self, account: baseaccount.BaseAccount):
self.accounts.append(account)
async def _get_account_and_address_info_for_address(self, address):
match = await self.db.get_address(address=address)
async def _get_account_and_address_info_for_address(self, wallet, address):
match = await self.db.get_address(accounts=wallet.accounts, address=address)
if match:
for account in self.accounts:
for account in wallet.accounts:
if match['account'] == account.public_key.address:
return account, match
async def get_private_key_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]:
match = await self._get_account_and_address_info_for_address(wallet, address)
if match:
account, address_info = match
return account.get_private_key(address_info['chain'], address_info['position'])
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
return None
async def get_public_key_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
match = await self._get_account_and_address_info_for_address(wallet, address)
if match:
account, address_info = match
return account.get_public_key(address_info['chain'], address_info['position'])
_, address_info = match
return address_info['pubkey']
return None
async def get_account_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
async def get_account_for_address(self, wallet, address):
match = await self._get_account_and_address_info_for_address(wallet, address)
if match:
return match[0]
@@ -210,6 +213,12 @@ class BaseLedger(metaclass=LedgerRegistry):
estimators.append(utxo.get_estimator(self))
return estimators
async def get_addresses(self, **constraints):
return await self.db.get_addresses(**constraints)
def get_address_count(self, **constraints):
return self.db.get_address_count(**constraints)
async def get_spendable_utxos(self, amount: int, funding_accounts):
async with self._utxo_reservation_lock:
txos = await self.get_effective_amount_estimators(funding_accounts)
@@ -229,29 +238,16 @@ class BaseLedger(metaclass=LedgerRegistry):
def release_tx(self, tx):
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
def constraint_account_or_all(self, constraints):
if 'accounts' in constraints:
return
account = constraints.pop('account', None)
if account:
constraints['accounts'] = [account]
else:
constraints['accounts'] = self.accounts
def get_utxos(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_utxo_count(**constraints)
def get_transactions(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transactions(**constraints)
def get_transaction_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transaction_count(**constraints)
async def get_local_status_and_history(self, address, history=None):
@@ -548,7 +544,7 @@ class BaseLedger(metaclass=LedgerRegistry):
return None
def broadcast(self, tx):
# broadcast cant be a retriable call yet
# broadcast can't be a retriable call yet
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
@@ -562,7 +558,7 @@ class BaseLedger(metaclass=LedgerRegistry):
addresses.add(
self.hash160_to_address(txo.script.values['pubkey_hash'])
)
records = await self.db.get_addresses(cols=('address',), address__in=addresses)
records = await self.db.get_addresses(address__in=addresses)
_, pending = await asyncio.wait([
self.on_transaction.where(partial(
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
+12 -11
View File
@@ -1,6 +1,6 @@
import asyncio
import logging
from typing import Type, MutableSequence, MutableMapping
from typing import Type, MutableSequence, MutableMapping, Optional
from torba.client.baseledger import BaseLedger, LedgerRegistry
from torba.client.wallet import Wallet, WalletStorage
@@ -41,16 +41,6 @@ class BaseWalletManager:
self.wallets.append(wallet)
return wallet
async def get_detailed_accounts(self, **kwargs):
ledgers = {}
for i, account in enumerate(self.accounts):
details = await account.get_details(**kwargs)
details['is_default'] = i == 0
ledger_id = account.ledger.get_id()
ledgers.setdefault(ledger_id, [])
ledgers[ledger_id].append(details)
return ledgers
@property
def default_wallet(self):
for wallet in self.wallets:
@@ -78,3 +68,14 @@ class BaseWalletManager:
l.stop() for l in self.ledgers.values()
))
self.running = False
def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet:
if wallet_id is None:
return self.default_wallet
return self.get_wallet_or_error(wallet_id)
def get_wallet_or_error(self, wallet_id: str) -> Wallet:
for wallet in self.wallets:
if wallet.id == wallet_id:
return wallet
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
+9 -3
View File
@@ -73,7 +73,7 @@ class ClientSession(BaseClientSession):
return reply
except (RPCError, ProtocolError) as e:
if str(e).find('.*no such .*transaction.*'):
# shouldnt the server return none instead?
# shouldn't the server return none instead?
return None
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
@@ -185,6 +185,9 @@ class BaseNetwork:
async def start(self):
self.running = True
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
@@ -253,11 +256,14 @@ class BaseNetwork:
try:
return await self.rpc('blockchain.address.subscribe', [address], True)
except asyncio.TimeoutError:
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
if self.client:
self.client.abort()
raise asyncio.CancelledError()
def get_server_features(self):
return self.rpc('server.features', (), restricted=True)
class SessionPool:
@@ -281,7 +287,7 @@ class SessionPool:
return None
return min(
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions],
for session in self.available_sessions] or [(0, None)],
key=itemgetter(0)
)[1]
+34 -23
View File
@@ -1,6 +1,6 @@
import logging
import typing
from typing import List, Iterable, Optional
from typing import List, Iterable, Optional, Tuple
from binascii import hexlify
from torba.client.basescript import BaseInputScript, BaseOutputScript
@@ -12,7 +12,7 @@ from torba.client.util import ReadOnlyList
from torba.client.errors import InsufficientFundsError
if typing.TYPE_CHECKING:
from torba.client import baseledger
from torba.client import baseledger, wallet as basewallet
log = logging.getLogger()
@@ -431,21 +431,32 @@ class BaseTransaction:
self.locktime = stream.read_uint32()
@classmethod
def ensure_all_have_same_ledger(cls, funding_accounts: Iterable[BaseAccount],
change_account: BaseAccount = None) -> 'baseledger.BaseLedger':
ledger = None
def ensure_all_have_same_ledger_and_wallet(
cls, funding_accounts: Iterable[BaseAccount],
change_account: BaseAccount = None) -> Tuple['baseledger.BaseLedger', 'basewallet.Wallet']:
ledger = wallet = None
for account in funding_accounts:
if ledger is None:
ledger = account.ledger
wallet = account.wallet
if ledger != account.ledger:
raise ValueError(
'All funding accounts used to create a transaction must be on the same ledger.'
)
if change_account is not None and change_account.ledger != ledger:
raise ValueError('Change account must use same ledger as funding accounts.')
if wallet != account.wallet:
raise ValueError(
'All funding accounts used to create a transaction must be from the same wallet.'
)
if change_account is not None:
if change_account.ledger != ledger:
raise ValueError('Change account must use same ledger as funding accounts.')
if change_account.wallet != wallet:
raise ValueError('Change account must use same wallet as funding accounts.')
if ledger is None:
raise ValueError('No ledger found.')
return ledger
if wallet is None:
raise ValueError('No wallet found.')
return ledger, wallet
@classmethod
async def create(cls, inputs: Iterable[BaseInput], outputs: Iterable[BaseOutput],
@@ -458,7 +469,7 @@ class BaseTransaction:
.add_inputs(inputs) \
.add_outputs(outputs)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
# value of the outputs plus associated fees
cost = (
@@ -496,18 +507,17 @@ class BaseTransaction:
if tx._outputs:
break
else:
# this condition and the outer range(5) loop cover an edge case
# whereby a single input is just enough to cover the fee and
# has some change left over, but the change left over is less
# than the cost_of_change: thus the input is completely
# consumed and no output is added, which is an invalid tx.
# to be able to spend this input we must increase the cost
# of the TX and run through the balance algorithm a second time
# adding an extra input and change output, making tx valid.
# we do this 5 times in case the other UTXOs added are also
# less than the fee, after 5 attempts we give up and go home
cost += cost_of_change + 1
# this condition and the outer range(5) loop cover an edge case
# whereby a single input is just enough to cover the fee and
# has some change left over, but the change left over is less
# than the cost_of_change: thus the input is completely
# consumed and no output is added, which is an invalid tx.
# to be able to spend this input we must increase the cost
# of the TX and run through the balance algorithm a second time
# adding an extra input and change output, making tx valid.
# we do this 5 times in case the other UTXOs added are also
# less than the fee, after 5 attempts we give up and go home
cost += cost_of_change + 1
if sign:
await tx.sign(funding_accounts)
@@ -524,14 +534,15 @@ class BaseTransaction:
return hash_type
async def sign(self, funding_accounts: Iterable[BaseAccount]):
ledger = self.ensure_all_have_same_ledger(funding_accounts)
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
for i, txi in enumerate(self._inputs):
assert txi.script is not None
assert txi.txo_ref.txo is not None
txo_script = txi.txo_ref.txo.script
if txo_script.is_pay_pubkey_hash:
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
private_key = await ledger.get_private_key_for_address(address)
private_key = await ledger.get_private_key_for_address(wallet, address)
assert private_key is not None, 'Cannot find private key for signing output.'
tx = self._serialize_for_signature(i)
txi.script.values['signature'] = \
private_key.sign(tx) + bytes((self.signature_hash_type(1),))
+171 -10
View File
@@ -1,9 +1,12 @@
import os
import time
import stat
import json
import zlib
import typing
from typing import Sequence, MutableSequence
import logging
from typing import List, Sequence, MutableSequence, Optional
from collections import UserDict
from hashlib import sha256
from operator import attrgetter
from torba.client.hash import better_aes_encrypt, better_aes_decrypt
@@ -12,6 +15,46 @@ if typing.TYPE_CHECKING:
from torba.client import basemanager, baseaccount, baseledger
log = logging.getLogger(__name__)
ENCRYPT_ON_DISK = 'encrypt-on-disk'
class TimestampedPreferences(UserDict):
def __init__(self, d: dict = None):
super().__init__()
if d is not None:
self.data = d.copy()
def __getitem__(self, key):
return self.data[key]['value']
def __setitem__(self, key, value):
self.data[key] = {
'value': value,
'ts': time.time()
}
def __repr__(self):
return repr(self.to_dict_without_ts())
def to_dict_without_ts(self):
return {
key: value['value'] for key, value in self.data.items()
}
@property
def hash(self):
return sha256(json.dumps(self.data).encode()).digest()
def merge(self, other: dict):
for key, value in other.items():
if key in self.data and value['ts'] < self.data[key]['ts']:
continue
self.data[key] = value
class Wallet:
""" The primary role of Wallet is to encapsulate a collection
of accounts (seed/private keys) and the spending rules / settings
@@ -19,23 +62,68 @@ class Wallet:
by physical files on the filesystem.
"""
preferences: TimestampedPreferences
encryption_password: Optional[str]
def __init__(self, name: str = 'Wallet', accounts: MutableSequence['baseaccount.BaseAccount'] = None,
storage: 'WalletStorage' = None) -> None:
storage: 'WalletStorage' = None, preferences: dict = None) -> None:
self.name = name
self.accounts = accounts or []
self.storage = storage or WalletStorage()
self.preferences = TimestampedPreferences(preferences or {})
self.encryption_password = None
def add_account(self, account):
@property
def id(self):
if self.storage.path:
return os.path.basename(self.storage.path)
return self.name
def add_account(self, account: 'baseaccount.BaseAccount'):
self.accounts.append(account)
def generate_account(self, ledger: 'baseledger.BaseLedger') -> 'baseaccount.BaseAccount':
return ledger.account_class.generate(ledger, self)
@property
def default_account(self) -> Optional['baseaccount.BaseAccount']:
for account in self.accounts:
return account
return None
def get_account_or_default(self, account_id: str) -> Optional['baseaccount.BaseAccount']:
if account_id is None:
return self.default_account
return self.get_account_or_error(account_id)
def get_account_or_error(self, account_id: str) -> 'baseaccount.BaseAccount':
for account in self.accounts:
if account.id == account_id:
return account
raise ValueError(f"Couldn't find account: {account_id}.")
def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['baseaccount.BaseAccount']:
return [
self.get_account_or_error(account_id)
for account_id in account_ids
] if account_ids else self.accounts
async def get_detailed_accounts(self, **kwargs):
ledgers = {}
for i, account in enumerate(self.accounts):
details = await account.get_details(**kwargs)
details['is_default'] = i == 0
ledger_id = account.ledger.get_id()
ledgers.setdefault(ledger_id, [])
ledgers[ledger_id].append(details)
return ledgers
@classmethod
def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':
json_dict = storage.read()
wallet = cls(
name=json_dict.get('name', 'Wallet'),
preferences=json_dict.get('preferences', {}),
storage=storage
)
account_dicts: Sequence[dict] = json_dict.get('accounts', [])
@@ -44,29 +132,40 @@ class Wallet:
ledger.account_class.from_dict(ledger, wallet, account_dict)
return wallet
def to_dict(self):
def to_dict(self, encrypt_password: str = None):
return {
'version': WalletStorage.LATEST_VERSION,
'name': self.name,
'accounts': [a.to_dict() for a in self.accounts]
'preferences': self.preferences.data,
'accounts': [a.to_dict(encrypt_password) for a in self.accounts]
}
def save(self):
if self.preferences.get(ENCRYPT_ON_DISK, False):
if self.encryption_password:
self.storage.write(self.to_dict(encrypt_password=self.encryption_password))
return
else:
log.warning(
"Disk encryption requested but no password available for encryption. "
"Saving wallet in an unencrypted state."
)
self.storage.write(self.to_dict())
@property
def default_account(self):
for account in self.accounts:
return account
@property
def hash(self) -> bytes:
h = sha256()
if self.preferences.get(ENCRYPT_ON_DISK, False):
assert self.encryption_password, \
"Encryption is enabled but no password is available, cannot generate hash."
h.update(self.encryption_password.encode())
h.update(self.preferences.hash)
for account in sorted(self.accounts, key=attrgetter('id')):
h.update(account.hash)
return h.digest()
def pack(self, password):
assert not self.is_locked, "Cannot pack a wallet with locked/encrypted accounts."
new_data = json.dumps(self.to_dict())
new_data_compressed = zlib.compress(new_data.encode())
return better_aes_encrypt(password, new_data_compressed)
@@ -77,6 +176,67 @@ class Wallet:
decompressed = zlib.decompress(decrypted)
return json.loads(decompressed)
def merge(self, manager: 'basemanager.BaseWalletManager',
password: str, data: str) -> List['baseaccount.BaseAccount']:
assert not self.is_locked, "Cannot sync apply on a locked wallet."
added_accounts = []
decrypted_data = self.unpack(password, data)
self.preferences.merge(decrypted_data.get('preferences', {}))
for account_dict in decrypted_data['accounts']:
ledger = manager.get_or_create_ledger(account_dict['ledger'])
_, _, pubkey = ledger.account_class.keys_from_dict(ledger, account_dict)
account_id = pubkey.address
local_match = None
for local_account in self.accounts:
if account_id == local_account.id:
local_match = local_account
break
if local_match is not None:
local_match.merge(account_dict)
else:
new_account = ledger.account_class.from_dict(ledger, self, account_dict)
added_accounts.append(new_account)
return added_accounts
@property
def is_locked(self) -> bool:
for account in self.accounts:
if account.encrypted:
return True
return False
def unlock(self, password):
self.encryption_password = password
for account in self.accounts:
if account.encrypted:
if not account.decrypt(password):
return False
return True
def lock(self):
assert self.encryption_password is not None, "Cannot lock an unencrypted wallet, encrypt first."
for account in self.accounts:
if not account.encrypted:
account.encrypt(self.encryption_password)
return True
@property
def is_encrypted(self) -> bool:
return self.is_locked or self.preferences.get(ENCRYPT_ON_DISK, False)
def decrypt(self):
assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first."
self.preferences[ENCRYPT_ON_DISK] = False
self.save()
return True
def encrypt(self, password):
assert not self.is_locked, "Cannot re-encrypt a locked wallet, unlock first."
self.encryption_password = password
self.preferences[ENCRYPT_ON_DISK] = True
self.save()
return True
class WalletStorage:
@@ -87,6 +247,7 @@ class WalletStorage:
self._default = default or {
'version': self.LATEST_VERSION,
'name': 'My Wallet',
'preferences': {},
'accounts': []
}
+4 -4
View File
@@ -14,11 +14,11 @@ from torba.client.baseheader import BaseHeaders, ArithUint256
class MainHeaders(BaseHeaders):
header_size = 80
chunk_size = 2016
max_target = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
header_size: int = 80
chunk_size: int = 2016
max_target: int = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash: Optional[bytes] = b'000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
target_timespan = 14 * 24 * 60 * 60
target_timespan: int = 14 * 24 * 60 * 60
@staticmethod
def serialize(header: dict) -> bytes:
+14 -9
View File
@@ -68,7 +68,7 @@ def set_logging(ledger_module, level, handler=None):
class Conductor:
def __init__(self, ledger_module=None, manager_module=None, verbosity=logging.WARNING,
enable_segwit=False):
enable_segwit=False, seed=None):
self.ledger_module = ledger_module or get_ledger_from_environment()
self.manager_module = manager_module or get_manager_from_environment()
self.spv_module = get_spvserver_from_ledger(self.ledger_module)
@@ -76,7 +76,9 @@ class Conductor:
self.blockchain_node = get_blockchain_node_from_ledger(self.ledger_module)
self.blockchain_node.segwit_enabled = enable_segwit
self.spv_node = SPVNode(self.spv_module)
self.wallet_node = WalletNode(self.manager_module, self.ledger_module.RegTestLedger)
self.wallet_node = WalletNode(
self.manager_module, self.ledger_module.RegTestLedger, default_seed=seed
)
set_logging(self.ledger_module, verbosity)
@@ -138,7 +140,7 @@ class Conductor:
class WalletNode:
def __init__(self, manager_class: Type[BaseWalletManager], ledger_class: Type[BaseLedger],
verbose: bool = False, port: int = 5280) -> None:
verbose: bool = False, port: int = 5280, default_seed: str = None) -> None:
self.manager_class = manager_class
self.ledger_class = ledger_class
self.verbose = verbose
@@ -148,10 +150,13 @@ class WalletNode:
self.account: Optional[BaseAccount] = None
self.data_path: Optional[str] = None
self.port = port
self.default_seed = default_seed
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
self.data_path = tempfile.mkdtemp()
wallet_file_name = os.path.join(self.data_path, 'my_wallet.json')
wallets_dir = os.path.join(self.data_path, 'wallets')
os.mkdir(wallets_dir)
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
with open(wallet_file_name, 'w') as wallet_file:
wallet_file.write('{"version": 1, "accounts": []}\n')
self.manager = self.manager_class.from_config({
@@ -166,14 +171,14 @@ class WalletNode:
})
self.ledger = self.manager.ledgers[self.ledger_class]
self.wallet = self.manager.default_wallet
if seed is None and self.wallet is not None:
self.wallet.generate_account(self.ledger)
elif self.wallet is not None:
if not self.wallet:
raise ValueError('Wallet is required.')
if seed or self.default_seed:
self.ledger.account_class.from_dict(
self.ledger, self.wallet, {'seed': seed}
self.ledger, self.wallet, {'seed': seed or self.default_seed}
)
else:
raise ValueError('Wallet is required.')
self.wallet.generate_account(self.ledger)
self.account = self.wallet.default_account
if connect:
await self.manager.start()
+1 -1
View File
@@ -37,7 +37,7 @@ class FramerBase(object):
"""Abstract base class for a framer.
A framer breaks an incoming byte stream into protocol messages,
buffering if necesary. It also frames outgoing messages into
buffering if necessary. It also frames outgoing messages into
a byte stream.
"""
+1 -1
View File
@@ -513,7 +513,7 @@ class JSONRPCv2(JSONRPC):
class JSONRPCLoose(JSONRPC):
"""A relaxed versin of JSON RPC."""
"""A relaxed version of JSON RPC."""
# Don't be so loose we accept any old message ID
_message_id = JSONRPCv2._message_id
+1 -1
View File
@@ -93,7 +93,7 @@ class SessionBase(asyncio.Protocol):
# Set when a connection is made
self._address = None
self._proxy_address = None
# For logger.debug messsages
# For logger.debug messages
self.verbosity = 0
# Cleared when the send socket is full
self._can_send = Event()
+1 -1
View File
@@ -117,7 +117,7 @@ class Daemon:
async def _send(self, payload, processor):
"""Send a payload to be converted to JSON.
Handles temporary connection issues. Daemon reponse errors
Handles temporary connection issues. Daemon response errors
are raise through DaemonError.
"""
def log_error(error):
+2
View File
@@ -80,6 +80,8 @@ class Env:
self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
self.description = self.default('DESCRIPTION', '')
self.daily_fee = self.integer('DAILY_FEE', 0)
# Identities
clearnet_identity = self.clearnet_identity()
+1 -1
View File
@@ -228,7 +228,7 @@ class History:
def _compact_hashX(self, hashX, hist_map, hist_list,
write_items, keys_to_delete):
"""Compres history for a hashX. hist_list is an ordered list of
"""Compress history for a hashX. hist_list is an ordered list of
the histories to be compressed."""
# History entries (tx numbers) are 4 bytes each. Distribute
# over rows of up to 50KB in size. A fixed row size means
+6 -3
View File
@@ -385,7 +385,7 @@ class SessionManager:
return "peer '{}' added".format(real_name)
async def rpc_disconnect(self, session_ids):
"""Disconnect sesssions.
"""Disconnect sessions.
session_ids: array of session IDs
"""
@@ -397,7 +397,7 @@ class SessionManager:
return await self._for_each_session(session_ids, close)
async def rpc_log(self, session_ids):
"""Toggle logging of sesssions.
"""Toggle logging of sessions.
session_ids: array of session IDs
"""
@@ -414,7 +414,7 @@ class SessionManager:
try:
self.daemon.set_url(daemon_url)
except Exception as e:
raise RPCError(BAD_REQUEST, f'an error occured: {e!r}')
raise RPCError(BAD_REQUEST, f'an error occurred: {e!r}')
return f'now using daemon at {self.daemon.logged_url()}'
async def rpc_stop(self):
@@ -767,6 +767,9 @@ class ElectrumX(SessionBase):
'protocol_min': min_str,
'protocol_max': max_str,
'genesis_hash': env.coin.GENESIS_HASH,
'description': env.description,
'payment_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
}
+1 -1
View File
@@ -457,7 +457,7 @@ class TxInputTokenPay(TxInput):
self.script[1] == self.OP_ANON_MARKER)
def is_generation(self):
# Transactions comming in from stealth addresses are seen by
# Transactions coming in from stealth addresses are seen by
# the blockchain as newly minted coins. The reverse, where coins
# are sent TO a stealth address, are seen by the blockchain as
# a coin burn.
+2 -1
View File
@@ -183,6 +183,7 @@ class AdvanceTimeTestCase(AsyncioTestCase):
class IntegrationTestCase(AsyncioTestCase):
SEED = None
LEDGER = None
MANAGER = None
ENABLE_SEGWIT = False
@@ -201,7 +202,7 @@ class IntegrationTestCase(AsyncioTestCase):
async def asyncSetUp(self):
self.conductor = Conductor(
ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY,
enable_segwit=self.ENABLE_SEGWIT
enable_segwit=self.ENABLE_SEGWIT, seed=self.SEED
)
await self.conductor.start_blockchain()
self.addCleanup(self.conductor.stop_blockchain)
Loaded 100 of 101 files, more files were not shown because too many files have changed in this diff. Show more