Compare commits

..
1 Commits
Author SHA1 Message Date
Victor Shyba 5718be1c61 log top memory usage on status if tracemalloc enabled 2019-09-16 15:00:35 -03:00
39 changed files with 713 additions and 1552 deletions

No files matched your search

+107 -447
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.42.0"
__version__ = "0.41.0"
version = tuple(__version__.split('.'))
+3 -13
View File
@@ -148,12 +148,7 @@ class MaxKeyFee(Setting[dict]):
@staticmethod
def _parse_list(l):
if l == ['null']:
return None
assert len(l) == 2, (
'Max key fee is made up of either two values: '
'"AMOUNT CURRENCY", or "null" (to set no limit)'
)
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
try:
amount = float(l[0])
except ValueError:
@@ -181,7 +176,7 @@ class MaxKeyFee(Setting[dict]):
parser.add_argument(
self.cli_name,
help=self.doc,
nargs='+',
nargs=2,
metavar=('AMOUNT', 'CURRENCY'),
default=NOT_SET
)
@@ -485,10 +480,6 @@ class Config(CLIConfig):
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
wallets = Strings(
"Wallet files in 'wallet_dir' to load at startup.",
['default_wallet']
)
# network
use_upnp = Toggle(
@@ -536,8 +527,7 @@ class Config(CLIConfig):
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
)
# reflector settings
+1 -1
View File
@@ -45,7 +45,7 @@ async def execute_command(conf, method, params, callback=display):
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
File diff suppressed because it is too large. Load diff
@@ -6,8 +6,6 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError
from torba.client.wallet import Wallet
from torba.client.bip32 import PubKey
from lbry.schema.claim import Claim
from lbry.wallet.ledger import MainNetLedger, Account
from lbry.wallet.transaction import Transaction, Output
@@ -69,13 +67,6 @@ def encode_account_doc():
}
def encode_wallet_doc():
return {
'id': 'wallet_id',
'name': 'optional wallet name',
}
def encode_file_doc():
return {
'streaming_url': '(str) url to stream the file using range requests',
@@ -119,8 +110,6 @@ class JSONResponseEncoder(JSONEncoder):
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Account):
return self.encode_account(obj)
if isinstance(obj, Wallet):
return self.encode_wallet(obj)
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, Transaction):
@@ -129,8 +118,6 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_output(obj)
if isinstance(obj, Claim):
return self.encode_claim(obj)
if isinstance(obj, PubKey):
return obj.extended_key_string()
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -232,12 +219,6 @@ class JSONResponseEncoder(JSONEncoder):
result['is_default'] = self.ledger.accounts[0] == account
return result
def encode_wallet(self, wallet):
return {
'id': wallet.id,
'name': wallet.name
}
def encode_file(self, managed_stream):
output_exists = managed_stream.output_file_exists
tx_height = managed_stream.stream_claim_info.height
+65 -66
View File
@@ -101,9 +101,6 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
stream_hashes = tuple(
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
)
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
@@ -111,7 +108,9 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", stream_hashes):
"order by c.rowid desc", [
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
claim = StoredStreamClaim(stream_hash, *claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -138,7 +137,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
)
for claim_name, claim_id in _batched_select(
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
tuple(signed_claims.keys())):
list(signed_claims.keys())):
for claim in signed_claims[claim_id]:
claim.channel_name = claim_name
return files
@@ -148,35 +147,35 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
# add all blobs, except the last one, which is empty
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
).fetchall()
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs)
).fetchall()
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
# ensure should_announce is set regardless if insert was ignored
transaction.execute(
"update blob set should_announce=1 where blob_hash in (?, ?)",
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
).fetchall()
)
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
blob_hashes.append((descriptor.sd_hash, ))
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
@@ -192,7 +191,7 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
).fetchall()
)
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
@@ -294,17 +293,17 @@ class SQLiteStorage(SQLiteMixin):
def _add_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
(
[
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
for blob_hash, length in blob_hashes_and_lengths
)
).fetchall()
]
)
if finished:
transaction.executemany(
"update blob set status='finished' where blob.blob_hash=?", (
"update blob set status='finished' where blob.blob_hash=?", [
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
)
).fetchall()
]
)
return await self.db.run(_add_blobs)
def get_blob_status(self, blob_hash: str):
@@ -318,9 +317,9 @@ class SQLiteStorage(SQLiteMixin):
return transaction.executemany(
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
"where blob_hash=?",
((int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
for blob_hash in blob_hashes)
).fetchall()
[(int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
for blob_hash in blob_hashes]
)
return self.db.run(_update_last_announced_blobs)
def should_single_announce_blobs(self, blob_hashes, immediate=False):
@@ -331,11 +330,11 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute(
"update blob set single_announce=1, next_announce_time=? "
"where blob_hash=? and status='finished'", (int(now), blob_hash)
).fetchall()
)
else:
transaction.execute(
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
).fetchall()
)
return self.db.run(set_single_announce)
def get_blobs_to_announce(self):
@@ -348,22 +347,22 @@ class SQLiteStorage(SQLiteMixin):
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
).fetchall()
)
else:
r = transaction.execute(
"select blob_hash from blob where blob_hash is not null "
"and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
).fetchall()
return [b[0] for b in r]
)
return [b[0] for b in r.fetchall()]
return self.db.run(get_and_update)
def delete_blobs_from_db(self, blob_hashes):
def delete_blobs(transaction):
transaction.executemany(
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
).fetchall()
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
)
return self.db.run_with_foreign_keys_disabled(delete_blobs)
def get_all_blob_hashes(self):
@@ -371,18 +370,22 @@ class SQLiteStorage(SQLiteMixin):
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
finished_blob_hashes = tuple(
blob_hash for (blob_hash, ) in transaction.execute(
"select blob_hash from blob where status='finished'"
).fetchall()
)
finished_blobs_set = set(finished_blob_hashes)
to_update_set = finished_blobs_set.difference(blob_files)
to_update = [
(blob_hash, )
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
if blob_hash not in blob_files
]
transaction.executemany(
"update blob set status='pending' where blob_hash=?",
((blob_hash, ) for blob_hash in to_update_set)
).fetchall()
return blob_files.intersection(finished_blobs_set)
to_update
)
return {
blob_hash
for blob_hash, in _batched_select(
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
list(blob_files)
)
}
return self.db.run(_sync_blobs)
# # # # # # # # # stream functions # # # # # # # # #
@@ -481,7 +484,7 @@ class SQLiteStorage(SQLiteMixin):
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
).fetchall()
)
return await self.db.run(update_manually_removed_files)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
@@ -489,7 +492,7 @@ class SQLiteStorage(SQLiteMixin):
def change_file_status(self, stream_hash: str, new_status: str):
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
file_name: typing.Optional[str]):
@@ -498,22 +501,22 @@ class SQLiteStorage(SQLiteMixin):
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
encoded_download_dir, encoded_file_name, stream_hash,
))
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
binascii.hexlify(content_fee.raw), stream_hash,
))
async def set_saved_file(self, stream_hash: str):
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
stream_hash,
))
async def clear_saved_file(self, stream_hash: str):
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
stream_hash,
))
@@ -534,13 +537,13 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
((stream_hash, ) for stream_hash in stream_hashes)
).fetchall()
[(stream_hash, ) for stream_hash in stream_hashes]
)
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
transaction.executemany(
f"update file set download_directory=? where stream_hash=?",
((download_dir, stream_hash) for stream_hash in stream_hashes)
).fetchall()
[(download_dir, stream_hash) for stream_hash in stream_hashes]
)
await self.db.run_with_foreign_keys_disabled(_recover)
def get_all_stream_hashes(self):
@@ -552,16 +555,14 @@ class SQLiteStorage(SQLiteMixin):
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
def _save_support(transaction):
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
transaction.execute(
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
).fetchall()
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
for claim_id, supports in claim_id_to_supports.items():
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
).fetchall()
)
return self.db.run(_save_support)
def get_supports(self, *claim_ids):
@@ -580,7 +581,7 @@ class SQLiteStorage(SQLiteMixin):
for support_info in _batched_select(
transaction,
"select * from support where claim_id in {}",
claim_ids
tuple(claim_ids)
)
]
@@ -611,7 +612,7 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute(
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
).fetchall()
)
# if this response doesn't have support info don't overwrite the existing
# support info
if 'supports' in claim_info:
@@ -698,9 +699,7 @@ class SQLiteStorage(SQLiteMixin):
)
# update the claim associated to the file
transaction.execute(
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
).fetchall()
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
async def save_content_claim(self, stream_hash, claim_outpoint):
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
@@ -723,11 +722,11 @@ class SQLiteStorage(SQLiteMixin):
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
if success:
return self.db.execute_fetchall(
return self.db.execute(
"insert or replace into reflected_stream values (?, ?, ?)",
(sd_hash, reflector_address, self.time_getter())
)
return self.db.execute_fetchall(
return self.db.execute(
"delete from reflected_stream where sd_hash=? and reflector_address=?",
(sd_hash, reflector_address)
)
-1
View File
@@ -139,7 +139,6 @@ types_map = {
# These are non-standard types, commonly found in the wild.
'.cbr': ('application/vnd.comicbook-rar', 'document'),
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.flac': ('audio/flac', 'audio'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
+2 -5
View File
@@ -4,10 +4,7 @@ from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
# see https://spec.lbry.com/
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
@@ -20,7 +17,7 @@ def _create_url_regex():
def _claim(name, prefix=""):
return _group(
_named(name+"_name", prefix + invalid_names_regex) +
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
+2 -2
View File
@@ -401,8 +401,8 @@ class StreamManager:
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5) if self.config.max_key_fee else None
if max_fee_amount and fee_amount > max_fee_amount:
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
+10 -10
View File
@@ -139,34 +139,34 @@ class Account(BaseAccount):
return details
def get_transaction_history(self, **constraints):
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_transaction_history(account=self, **constraints)
def get_transaction_history_count(self, **constraints):
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_transaction_history_count(account=self, **constraints)
def get_claims(self, **constraints):
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_claims(account=self, **constraints)
def get_claim_count(self, **constraints):
return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_claim_count(account=self, **constraints)
def get_streams(self, **constraints):
return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_streams(account=self, **constraints)
def get_stream_count(self, **constraints):
return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_stream_count(account=self, **constraints)
def get_channels(self, **constraints):
return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_channels(account=self, **constraints)
def get_channel_count(self, **constraints):
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_channel_count(account=self, **constraints)
def get_supports(self, **constraints):
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_supports(account=self, **constraints)
def get_support_count(self, **constraints):
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_support_count(account=self, **constraints)
def get_support_summary(self):
return self.ledger.db.get_supports_summary(account_id=self.id)
+19 -17
View File
@@ -22,18 +22,18 @@ class WalletDatabase(BaseDatabase):
claim_id text,
claim_name text
);
create index if not exists txo_address_idx on txo (address);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_txo_type_idx on txo (txo_type);
"""
CREATE_TABLES_QUERY = (
BaseDatabase.PRAGMAS +
BaseDatabase.CREATE_ACCOUNT_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
BaseDatabase.CREATE_TX_TABLE +
CREATE_TXO_TABLE +
BaseDatabase.CREATE_TXI_TABLE
BaseDatabase.CREATE_TX_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
BaseDatabase.CREATE_PUBKEY_ADDRESS_INDEX +
CREATE_TXO_TABLE +
BaseDatabase.CREATE_TXO_INDEX +
BaseDatabase.CREATE_TXI_TABLE +
BaseDatabase.CREATE_TXI_INDEX
)
def txo_to_row(self, tx, address, txo):
@@ -50,16 +50,18 @@ class WalletDatabase(BaseDatabase):
row['claim_name'] = txo.claim_name
return row
async def get_txos(self, wallet=None, no_tx=False, **constraints) -> List[Output]:
txos = await super().get_txos(wallet=wallet, no_tx=no_tx, **constraints)
async def get_txos(self, **constraints) -> List[Output]:
my_accounts = constraints.get('my_accounts', constraints.get('accounts', []))
txos = await super().get_txos(**constraints)
channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
if txo.claim.is_channel and my_accounts:
for account in my_accounts:
private_key = account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
@@ -71,7 +73,7 @@ class WalletDatabase(BaseDatabase):
channels = {
txo.claim_id: txo for txo in
(await self.get_claims(
wallet=wallet,
my_accounts=my_accounts,
claim_id__in=channel_ids
))
}
@@ -132,20 +134,20 @@ class WalletDatabase(BaseDatabase):
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account):
await self.db.execute_fetchall(
await self.db.execute(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
" SELECT address from pubkey_address WHERE account = ?"
" )", [account.public_key.address]
)
def get_supports_summary(self, account_id):
return self.db.execute_fetchall(f"""
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
(txo.txid in
(select txi.txid from txi join account_address a on txi.address = a.address
(select txi.txid from txi join pubkey_address a on txi.address = a.address
where a.account = ?)) as from_me,
(txo.address in (select address from account_address where account=?)) as to_me,
(txo.address in (select address from pubkey_address where account=?)) as to_me,
tx.height
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
""", (account_id, account_id))
+10
View File
@@ -121,30 +121,39 @@ class MainNetLedger(BaseLedger):
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_claims(**constraints)
def get_claim_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_claim_count(**constraints)
def get_streams(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_streams(**constraints)
def get_stream_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_stream_count(**constraints)
def get_channels(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_channels(**constraints)
def get_channel_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_channel_count(**constraints)
def get_supports(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_supports(**constraints)
def get_support_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_support_count(**constraints)
async def get_transaction_history(self, **constraints):
self.constraint_account_or_all(constraints)
txs = await self.db.get_transactions(**constraints)
headers = self.headers
history = []
@@ -240,6 +249,7 @@ class MainNetLedger(BaseLedger):
return history
def get_transaction_history_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transaction_count(**constraints)
+8 -11
View File
@@ -138,23 +138,20 @@ class LbryWalletManager(BaseWalletManager):
if not os.path.exists(wallets_directory):
os.mkdir(wallets_directory)
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(
os.path.join(wallets_directory, 'default_wallet')
)
wallet_file_path = os.path.join(wallets_directory, 'default_wallet')
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(wallet_file_path)
manager = cls.from_config({
'ledgers': {ledger_id: ledger_config},
'wallets': [
os.path.join(wallets_directory, wallet_file) for wallet_file in settings.wallets
]
'wallets': [wallet_file_path]
})
ledger = manager.get_or_create_ledger(ledger_id)
ledger.coin_selection_strategy = settings.coin_selection_strategy
default_wallet = manager.default_wallet
if default_wallet.default_account is None:
log.info('Wallet at %s is empty, generating a default account.', default_wallet.id)
default_wallet.generate_account(ledger)
default_wallet.save()
if manager.default_account is None:
log.info('Wallet at %s is empty, generating a default account.', wallet_file_path)
manager.default_wallet.generate_account(ledger)
manager.default_wallet.save()
if receiving_addresses or change_addresses:
if not os.path.exists(ledger.path):
os.mkdir(ledger.path)
+5 -5
View File
@@ -203,7 +203,7 @@ class Transaction(BaseTransaction):
@classmethod
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address))
return cls.create([], [output], funding_accounts, change_account)
@@ -211,7 +211,7 @@ class Transaction(BaseTransaction):
def claim_create(
cls, name: str, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
claim_output = Output.pay_claim_name_pubkey_hash(
amount, name, claim, ledger.address_to_hash160(holding_address)
)
@@ -223,7 +223,7 @@ class Transaction(BaseTransaction):
def claim_update(
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
updated_claim = Output.pay_update_claim_pubkey_hash(
amount, previous_claim.claim_name, previous_claim.claim_id,
claim, ledger.address_to_hash160(holding_address)
@@ -239,7 +239,7 @@ class Transaction(BaseTransaction):
@classmethod
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account):
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
support_output = Output.pay_support_pubkey_hash(
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
)
@@ -248,7 +248,7 @@ class Transaction(BaseTransaction):
@classmethod
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
funding_accounts: List[Account], change_account: Account):
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
claim_output = Output.purchase_claim_pubkey_hash(
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
)
+1 -10
View File
@@ -13,15 +13,13 @@ from lbry.extras.daemon.Daemon import (
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
)
from lbry.extras.daemon.json_response_encoder import (
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
encode_wallet_doc
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
)
from lbry.testcase import CommandTestCase
RETURN_DOCS = {
'Account': encode_account_doc(),
'Wallet': encode_wallet_doc(),
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
@@ -105,13 +103,6 @@ class Examples(CommandTestCase):
'preference', 'get'
)
# wallets
await r(
'List your wallets',
'wallet', 'list'
)
# accounts
await r(
@@ -11,6 +11,14 @@ def extract(d, keys):
class AccountManagement(CommandTestCase):
async def test_sqlite_binding_error(self):
tasks = [
self.loop.create_task(self.daemon.jsonrpc_account_create('second account' + str(x))) for x in range(100)
]
await asyncio.wait(tasks)
for result in tasks:
self.assertFalse(isinstance(result.result(), Exception))
async def test_account_list_set_create_remove_add(self):
# check initial account
response = await self.daemon.jsonrpc_account_list()
@@ -85,7 +93,7 @@ class AccountManagement(CommandTestCase):
stream_list = self.daemon.jsonrpc_stream_list
support_list = self.daemon.jsonrpc_support_list
utxo_list = self.daemon.jsonrpc_utxo_list
default_account = self.wallet.default_account
default_account = self.daemon.default_account
second_account = await self.daemon.jsonrpc_account_create('second account')
tx = await self.daemon.jsonrpc_account_send(
@@ -171,9 +179,3 @@ class AccountManagement(CommandTestCase):
})
self.assertEqual(history[6]['value'], '0.0')
self.assertEqual(history[7]['value'], '10.0')
async def test_address_validation(self):
address = await self.daemon.jsonrpc_address_unused()
bad_address = address[0:20] + '9999999' + address[27:]
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
+27 -28
View File
@@ -466,47 +466,45 @@ class ChannelCommands(CommandTestCase):
'title': 'foo', 'email': 'new@email.com'}
)
# move channel to another account
# send channel to someone else
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
# before moving
# before sending
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
other_address = await account2.receiving.get_or_create_usable_address()
tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
# after moving
# after sending
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
async def test_channel_export_import_before_sending_channel(self):
# export
# shoud not have private key
txo = (await account2.get_channels())[0]
self.assertIsNone(txo.private_key)
# send the private key too
private_key = self.account.get_channel_private_key(unhexlify(channel['public_key']))
account2.add_channel_private_key(private_key)
# now should have private key
txo = (await account2.get_channels())[0]
self.assertIsNotNone(txo.private_key)
async def test_channel_export_import_into_new_account(self):
tx = await self.channel_create('@foo', '1.0')
claim_id = self.get_claim_id(tx)
channel_private_key = (await self.account.get_channels())[0].private_key
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
# import
daemon2 = await self.add_daemon()
self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
await daemon2.jsonrpc_channel_import(exported_data)
channels = await daemon2.jsonrpc_channel_list()
self.assertEqual(1, len(channels))
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
# second wallet can't update until channel is sent to it
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
# now send the channel as well
await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused())
# second wallet should be able to update now
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
class StreamCommands(ClaimTestCase):
@@ -567,7 +565,7 @@ class StreamCommands(ClaimTestCase):
async def test_publishing_checks_all_accounts_for_channel(self):
account1_id, account1 = self.account.id, self.account
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
await self.out(self.channel_create('@spam', '1.0'))
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
@@ -784,7 +782,7 @@ class StreamCommands(ClaimTestCase):
# send claim to someone else
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
# before sending
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
@@ -1081,12 +1079,13 @@ class StreamCommands(ClaimTestCase):
class SupportCommands(CommandTestCase):
async def test_regular_supports_and_tip_supports(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
account2 = wallet2.accounts[0]
# account2 will be used to send tips and supports to account1
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
account2 = self.daemon.get_account_or_error(account2_id)
# send account2 5 LBC out of the 10 LBC in account1
result = await self.out(self.daemon.jsonrpc_account_send(
'5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2')
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
))
await self.on_transaction_dict(result)
@@ -1104,7 +1103,7 @@ class SupportCommands(CommandTestCase):
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
claim_id, '1.0', True, account2_id, funding_account_ids=[account2_id])
)
await self.confirm_tx(tip['txid'])
@@ -1123,7 +1122,7 @@ class SupportCommands(CommandTestCase):
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = await self.out(
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
self.daemon.jsonrpc_transaction_list(account2_id)
)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
@@ -1135,7 +1134,7 @@ class SupportCommands(CommandTestCase):
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_support_create(
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
claim_id, '2.0', False, account2_id, funding_account_ids=[account2_id])
)
await self.confirm_tx(support['txid'])
@@ -1144,7 +1143,7 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '1.999717')
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
@@ -277,33 +277,6 @@ class FileCommands(CommandTestCase):
self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0')
self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address)
async def test_null_max_key_fee(self):
target_address = await self.blockchain.get_raw_change_address()
self.daemon.conf.max_key_fee = None
await self.stream_create(
'somename', '0.5', data=b'Yes, please',
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
)
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
# Assert the fee and bid are subtracted
await self.assertBalance(self.account, '9.483893')
response = await self.daemon.jsonrpc_get('lbry://somename')
await self.ledger.wait(response.content_fee)
await self.assertBalance(self.account, '8.483769')
# Assert the file downloads
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
# Assert the transaction is recorded to the blockchain
starting_balance = await self.blockchain.get_balance()
await self.generate(1)
block_reward_and_claim_fee = 2.0
self.assertEqual(
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
)
async def test_null_fee(self):
target_address = await self.blockchain.get_raw_change_address()
tx = await self.stream_create(
@@ -1,12 +0,0 @@
from lbry.testcase import CommandTestCase
class AddressManagement(CommandTestCase):
async def test_address_list(self):
addresses = await self.out(self.daemon.jsonrpc_address_list())
self.assertEqual(27, len(addresses))
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
self.assertEqual(1, len(single))
self.assertEqual(single[0], addresses[11])
-31
View File
@@ -55,41 +55,10 @@ class TestURLParsing(unittest.TestCase):
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\u0009')
_url('\u000a')
_url('\u000d')
_url('\u0020')
_url('\uD799')
_url('\uE000')
_url('\uFFFD')
def test_parser_invalid_urls(self):
fail = self._fail_url
fail("lbry://")
fail("lbry://\u0000")
fail("lbry://\u0008")
fail("lbry://\u000b")
fail("lbry://\u000c")
fail("lbry://\u000e")
fail("lbry://\u001f")
fail("lbry://\uD800")
fail("lbry://\uDFFF")
fail("lbry://\uDFFE")
fail("lbry://\uFFFF")
fail("lbry://;")
fail("lbry://\"")
fail("lbry://\\")
fail("lbry:///")
fail("lbry://<") and fail("lbry://>")
fail("lbry://{") and fail("lbry://}")
fail("lbry://[") and fail("lbry://]")
fail("lbry://%")
fail("lbry://|")
fail("lbry://^")
fail("lbry://~")
fail("lbry://`")
fail("lbry://test:3$1")
fail("lbry://test$1:1")
fail("lbry://test#x")
-1
View File
@@ -51,7 +51,6 @@ class CLITest(AsyncioTestCase):
self.assertEqual('3', normalize_value('3', key="name"))
self.assertEqual('3', normalize_value('3', key="download_directory"))
self.assertEqual('3', normalize_value('3', key="channel_name"))
self.assertEqual('3', normalize_value('3', key="claim_name"))
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
-4
View File
@@ -222,10 +222,6 @@ class ConfigurationTests(unittest.TestCase):
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, None)
args = parser.parse_args(['--max-key-fee', 'null'])
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, None)
# set
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
c = Config.create_from_arguments(args)
+2 -6
View File
@@ -61,17 +61,13 @@ class TestAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
private_key = await self.ledger.get_private_key_for_address(
account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx'
)
private_key = await self.ledger.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
self.assertEqual(
private_key.extended_key_string(),
'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR'
'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT'
)
private_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
private_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
self.assertIsNone(private_key)
def test_load_and_save_account(self):
@@ -1,5 +1,4 @@
import logging
import os
import asyncio
from unittest.mock import Mock
@@ -17,38 +16,6 @@ class NetworkTests(IntegrationTestCase):
await self.ledger.network.on_header.first
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
async def test_server_features(self):
self.assertEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
'hosts': {},
'protocol_max': '1.4',
'protocol_min': '1.1',
'pruning': None,
'description': '',
'payment_address': '',
'daily_fee': 0,
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
address = (await self.account.get_addresses(limit=1))[0]
os.environ.update({
'DESCRIPTION': 'Fastest server in the west.',
'DONATION_ADDRESS': address,
'DAILY_FEE': '42'})
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.network.on_connected.first
self.assertEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
'hosts': {},
'protocol_max': '1.4',
'protocol_min': '1.1',
'pruning': None,
'description': 'Fastest server in the west.',
'payment_address': address,
'daily_fee': 42,
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
class ReconnectTests(IntegrationTestCase):
@@ -88,7 +88,7 @@ class BasicTransactionTests(IntegrationTestCase):
await self.assertBalance(account1, '0.0')
await self.assertBalance(account2, '0.0')
addresses = await account1.receiving.get_addresses()
addresses = await self.account.receiving.get_addresses()
txids = await asyncio.gather(*(
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
))
+7 -21
View File
@@ -60,7 +60,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
await account.receiving._generate_keys(8, 11)
records = await account.receiving.get_address_records()
self.assertEqual(
[r['pubkey'].n for r in records],
[r['position'] for r in records],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
)
@@ -69,7 +69,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
self.assertEqual(len(new_keys), 8)
records = await account.receiving.get_address_records()
self.assertEqual(
[r['pubkey'].n for r in records],
[r['position'] for r in records],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
)
@@ -125,18 +125,14 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
private_key = await self.ledger.get_private_key_for_address(
account.wallet, '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J'
)
private_key = await self.ledger.get_private_key_for_address('1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
self.assertEqual(
private_key.extended_key_string(),
'xprv9xV7rhbg6M4yWrdTeLorz3Q1GrQb4aQzzGWboP3du7W7UUztzNTUrEYTnDfz7o'
'ptBygDxXYRppyiuenJpoBTgYP2C26E1Ah5FEALM24CsWi'
)
invalid_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
self.assertIsNone(invalid_key)
self.assertEqual(
@@ -279,18 +275,12 @@ class TestSingleKeyAccount(AsyncioTestCase):
self.assertEqual(len(new_keys), 1)
self.assertEqual(new_keys[0], account.public_key.address)
records = await account.receiving.get_address_records()
pubkey = records[0].pop('pubkey')
self.assertEqual(records, [{
'chain': 0,
'position': 0, 'chain': 0,
'account': account.public_key.address,
'address': account.public_key.address,
'history': None,
'used_times': 0
}])
self.assertEqual(
pubkey.extended_key_string(),
account.public_key.extended_key_string()
)
# case #1: no new addresses needed
empty = await account.receiving.ensure_address_gap()
@@ -343,18 +333,14 @@ class TestSingleKeyAccount(AsyncioTestCase):
address = await account.receiving.ensure_address_gap()
self.assertEqual(address[0], account.public_key.address)
private_key = await self.ledger.get_private_key_for_address(
account.wallet, address[0]
)
private_key = await self.ledger.get_private_key_for_address(address[0])
self.assertEqual(
private_key.extended_key_string(),
'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp'
'5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna',
)
invalid_key = await self.ledger.get_private_key_for_address(
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
)
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
self.assertIsNone(invalid_key)
self.assertEqual(
+28 -142
View File
@@ -1,10 +1,7 @@
import sys
import os
import unittest
import sqlite3
import tempfile
import asyncio
from concurrent.futures.thread import ThreadPoolExecutor
import os
from torba.client.wallet import Wallet
from torba.client.constants import COIN
@@ -199,14 +196,13 @@ class TestQueries(AsyncioTestCase):
'db': ledger_class.database_class(':memory:'),
'headers': ledger_class.headers_class(':memory:'),
})
self.wallet = Wallet()
await self.ledger.db.open()
async def asyncTearDown(self):
await self.ledger.db.close()
async def create_account(self, wallet=None):
account = self.ledger.account_class.generate(self.ledger, wallet or self.wallet)
async def create_account(self):
account = self.ledger.account_class.generate(self.ledger, Wallet())
await account.ensure_address_gap()
return account
@@ -265,8 +261,7 @@ class TestQueries(AsyncioTestCase):
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
for limit in range(variable_limit-2, variable_limit+2):
txs = await self.ledger.get_transactions(
accounts=self.wallet.accounts, limit=limit, order_by='height asc')
txs = await self.ledger.get_transactions(limit=limit, order_by='height asc')
self.assertEqual(len(txs), limit)
inputs, outputs, last_tx = set(), set(), txs[0]
for tx in txs[1:]:
@@ -276,23 +271,19 @@ class TestQueries(AsyncioTestCase):
last_tx = tx
async def test_queries(self):
wallet1 = Wallet()
account1 = await self.create_account(wallet1)
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1]))
wallet2 = Wallet()
account2 = await self.create_account(wallet2)
account3 = await self.create_account(wallet2)
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_address_count())
account1 = await self.create_account()
self.assertEqual(26, await self.ledger.db.get_address_count())
account2 = await self.create_account()
self.assertEqual(52, await self.ledger.db.get_address_count())
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3]))
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2]))
self.assertEqual(0, await self.ledger.db.get_utxo_count())
self.assertEqual([], await self.ledger.db.get_utxos())
self.assertEqual(0, await self.ledger.db.get_txo_count())
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(0, await self.ledger.db.get_balance())
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
tx1 = await self.create_tx_from_nothing(account1, 1)
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1]))
@@ -300,28 +291,20 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(10**8, await self.ledger.db.get_balance())
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2)
tx2b = await self.create_tx_from_nothing(account3, 2)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3]))
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3]))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(10**8, await self.ledger.db.get_balance())
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
@@ -330,24 +313,22 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2]))
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2))
self.assertEqual(0, await self.ledger.db.get_balance())
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual([3, 2, 1], [tx.height for tx in txs])
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
txs = await self.ledger.db.get_transactions(accounts=[account1])
self.assertEqual([tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
txs = await self.ledger.db.get_transactions(accounts=[account2])
self.assertEqual([tx3.id, tx2.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
@@ -359,18 +340,18 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(tx.id, tx2.id)
self.assertEqual(tx.inputs[0].is_my_account, False)
self.assertEqual(tx.outputs[0].is_my_account, False)
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account1])
self.assertEqual(tx.inputs[0].is_my_account, True)
self.assertEqual(tx.outputs[0].is_my_account, False)
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account2])
self.assertEqual(tx.inputs[0].is_my_account, False)
self.assertEqual(tx.outputs[0].is_my_account, True)
# height 0 sorted to the top with the rest in descending order
tx4 = await self.create_tx_from_nothing(account1, 0)
txos = await self.ledger.db.get_txos()
self.assertEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
self.assertEqual([0, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertEqual([tx4.id, tx2.id, tx1.id], [txo.tx_ref.id for txo in txos])
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
@@ -398,13 +379,13 @@ class TestUpgrade(AsyncioTestCase):
def add_address(self, address):
with sqlite3.connect(self.path) as conn:
conn.execute("""
INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth)
VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0)
INSERT INTO pubkey_address (address, account, chain, position, pubkey)
VALUES (?, 'account1', 0, 0, 'pubkey blob')
""", (address,))
def get_addresses(self):
with sqlite3.connect(self.path) as conn:
sql = "SELECT address FROM account_address ORDER BY address;"
sql = "SELECT address FROM pubkey_address ORDER BY address;"
return [col[0] for col in conn.execute(sql).fetchall()]
async def test_reset_on_version_change(self):
@@ -417,7 +398,7 @@ class TestUpgrade(AsyncioTestCase):
self.ledger.db.SCHEMA_VERSION = None
self.assertEqual(self.get_tables(), [])
await self.ledger.db.open()
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo'])
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo'])
self.assertEqual(self.get_addresses(), [])
self.add_address('address1')
await self.ledger.db.close()
@@ -426,17 +407,17 @@ class TestUpgrade(AsyncioTestCase):
self.ledger.db.SCHEMA_VERSION = '1.0'
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
self.add_address('address2')
await self.ledger.db.close()
# nothing changes
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), ['address2'])
await self.ledger.db.close()
@@ -447,101 +428,6 @@ class TestUpgrade(AsyncioTestCase):
"""
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.1')
self.assertEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_tables(), ['foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # all tables got reset
await self.ledger.db.close()
class TestSQLiteRace(AsyncioTestCase):
max_misuse_attempts = 40000
def setup_db(self):
self.db = sqlite3.connect(":memory:", isolation_level=None)
self.db.executescript(
"create table test1 (id text primary key not null, val text);\n" +
"create table test2 (id text primary key not null, val text);\n" +
"\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000))
)
async def asyncSetUp(self):
self.executor = ThreadPoolExecutor(1)
await self.loop.run_in_executor(self.executor, self.setup_db)
async def asyncTearDown(self):
await self.loop.run_in_executor(self.executor, self.db.close)
self.executor.shutdown()
async def test_binding_param_0_error(self):
# test real param 0 binding errors
for supported_type in [str, int, bytes]:
await self.loop.run_in_executor(
self.executor, self.db.executemany, "insert into test2 values (?, NULL)",
[(supported_type(1), ), (supported_type(2), )]
)
await self.loop.run_in_executor(
self.executor, self.db.execute, "delete from test2 where id in (1, 2)"
)
for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]:
try:
await self.loop.run_in_executor(
self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)",
[(unsupported_type(1), ), (unsupported_type(2), )]
)
self.assertTrue(False)
except sqlite3.InterfaceError as err:
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
async def test_unhandled_sqlite_misuse(self):
# test SQLITE_MISUSE being incorrectly raised as a param 0 binding error
attempts = 0
python_version = sys.version.split('\n')[0].rstrip(' ')
try:
while attempts < self.max_misuse_attempts:
f1 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, self.db.executemany, "update test1 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
f2 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, self.db.executemany, "update test2 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
attempts += 1
await asyncio.gather(f1, f2)
print(f"\nsqlite3 {sqlite3.version}/python {python_version} "
f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition")
self.assertTrue(False, 'this test failing means either the sqlite race conditions '
'have been fixed in cpython or the test max_attempts needs to be increased')
except sqlite3.InterfaceError as err:
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE "
f"after {attempts} attempts of the race condition")
@unittest.SkipTest
async def test_fetchall_prevents_sqlite_misuse(self):
# test that calling fetchall sufficiently avoids the race
attempts = 0
def executemany_fetchall(query, params):
self.db.executemany(query, params).fetchall()
while attempts < self.max_misuse_attempts:
f1 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, executemany_fetchall, "update test1 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
f2 = asyncio.wrap_future(
self.loop.run_in_executor(
self.executor, executemany_fetchall, "update test2 set val='derp' where id=?",
((str(i),) for i in range(2))
)
)
attempts += 1
await asyncio.gather(f1, f2)
+13 -11
View File
@@ -115,7 +115,7 @@ class HierarchicalDeterministic(AddressManager):
return self.account.public_key.child(self.chain_number).child(index)
async def get_max_gap(self) -> int:
addresses = await self._query_addresses(order_by="n asc")
addresses = await self._query_addresses(order_by="position ASC")
max_gap = 0
current_gap = 0
for address in addresses:
@@ -128,7 +128,7 @@ class HierarchicalDeterministic(AddressManager):
async def ensure_address_gap(self) -> List[str]:
async with self.address_generator_lock:
addresses = await self._query_addresses(limit=self.gap, order_by="n desc")
addresses = await self._query_addresses(limit=self.gap, order_by="position DESC")
existing_gap = 0
for address in addresses:
@@ -140,7 +140,7 @@ class HierarchicalDeterministic(AddressManager):
if existing_gap == self.gap:
return []
start = addresses[0]['pubkey'].n+1 if addresses else 0
start = addresses[0]['position']+1 if addresses else 0
end = start + (self.gap - existing_gap)
new_keys = await self._generate_keys(start, end-1)
await self.account.ledger.announce_addresses(self, new_keys)
@@ -149,15 +149,15 @@ class HierarchicalDeterministic(AddressManager):
async def _generate_keys(self, start: int, end: int) -> List[str]:
if not self.address_generator_lock.locked():
raise RuntimeError('Should not be called outside of address_generator_lock.')
keys = [self.public_key.child(index) for index in range(start, end+1)]
keys = [(index, self.public_key.child(index)) for index in range(start, end+1)]
await self.account.ledger.db.add_keys(self.account, self.chain_number, keys)
return [key.address for key in keys]
return [key[1].address for key in keys]
def get_address_records(self, only_usable: bool = False, **constraints):
if only_usable:
constraints['used_times__lt'] = self.maximum_uses_per_address
if 'order_by' not in constraints:
constraints['order_by'] = "used_times asc, n asc"
constraints['order_by'] = "used_times ASC, position ASC"
return self._query_addresses(**constraints)
@@ -190,7 +190,9 @@ class SingleKey(AddressManager):
async with self.address_generator_lock:
exists = await self.get_address_records()
if not exists:
await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key])
await self.account.ledger.db.add_keys(
self.account, self.chain_number, [(0, self.public_key)]
)
new_keys = [self.public_key.address]
await self.account.ledger.announce_addresses(self, new_keys)
return new_keys
@@ -415,16 +417,16 @@ class BaseAccount:
}
def get_utxos(self, **constraints):
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_utxos(account=self, **constraints)
def get_utxo_count(self, **constraints):
return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_utxo_count(account=self, **constraints)
def get_transactions(self, **constraints):
return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_transactions(account=self, **constraints)
def get_transaction_count(self, **constraints):
return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints)
return self.ledger.get_transaction_count(account=self, **constraints)
async def fund(self, to_account, amount=None, everything=False,
outputs=1, broadcast=False, **constraints):
+80 -99
View File
@@ -3,12 +3,12 @@ import asyncio
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Optional
import sqlite3
from torba.client.basetransaction import BaseTransaction, TXRefImmutable
from torba.client.bip32 import PubKey
from torba.client.baseaccount import BaseAccount
log = logging.getLogger(__name__)
@@ -19,7 +19,6 @@ class AIOSQLite:
# has to be single threaded as there is no mapping of thread:connection
self.executor = ThreadPoolExecutor(max_workers=1)
self.connection: sqlite3.Connection = None
self._closing = False
@classmethod
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
@@ -30,12 +29,14 @@ class AIOSQLite:
return db
async def close(self):
if self._closing:
def __close(conn):
self.executor.submit(conn.close)
self.executor.shutdown(wait=True)
conn = self.connection
if not conn:
return
self._closing = True
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
self.executor.shutdown(wait=True)
self.connection = None
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
def executemany(self, sql: str, params: Iterable):
params = params if params is not None else []
@@ -86,10 +87,10 @@ class AIOSQLite:
if not foreign_keys_enabled:
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
try:
self.connection.execute('pragma foreign_keys=off').fetchone()
self.connection.execute('pragma foreign_keys=off')
return self.__run_transaction(fun, *args, **kwargs)
finally:
self.connection.execute('pragma foreign_keys=on').fetchone()
self.connection.execute('pragma foreign_keys=on')
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
@@ -159,14 +160,15 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
return joiner.join(sql) if sql else '', values
def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
def query(select, **constraints):
sql = [select]
limit = constraints.pop('limit', None)
offset = constraints.pop('offset', None)
order_by = constraints.pop('order_by', None)
accounts = constraints.pop('accounts', [])
if accounts:
constraints.pop('my_accounts', None)
accounts = constraints.pop('accounts', None)
if accounts is not None:
constraints['account__in'] = [a.public_key.address for a in accounts]
where, values = constraints_to_sql(constraints)
@@ -284,33 +286,26 @@ class SQLiteMixin:
class BaseDatabase(SQLiteMixin):
SCHEMA_VERSION = "1.1"
SCHEMA_VERSION = "1.0"
PRAGMAS = """
pragma journal_mode=WAL;
"""
CREATE_ACCOUNT_TABLE = """
create table if not exists account_address (
account text not null,
address text not null,
chain integer not null,
pubkey blob not null,
chain_code blob not null,
n integer not null,
depth integer not null,
primary key (account, address)
);
create index if not exists address_account_idx on account_address (address, account);
"""
CREATE_PUBKEY_ADDRESS_TABLE = """
create table if not exists pubkey_address (
address text primary key,
account text not null,
chain integer not null,
position integer not null,
pubkey blob not null,
history text,
used_times integer not null default 0
);
"""
CREATE_PUBKEY_ADDRESS_INDEX = """
create index if not exists pubkey_address_account_idx on pubkey_address (account);
"""
CREATE_TX_TABLE = """
create table if not exists tx (
@@ -332,6 +327,8 @@ class BaseDatabase(SQLiteMixin):
script blob not null,
is_reserved boolean not null default 0
);
"""
CREATE_TXO_INDEX = """
create index if not exists txo_address_idx on txo (address);
"""
@@ -341,17 +338,21 @@ class BaseDatabase(SQLiteMixin):
txoid text references txo,
address text references pubkey_address
);
"""
CREATE_TXI_INDEX = """
create index if not exists txi_address_idx on txi (address);
create index if not exists txi_txoid_idx on txi (txoid);
"""
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_ACCOUNT_TABLE +
CREATE_PUBKEY_ADDRESS_TABLE +
CREATE_TX_TABLE +
CREATE_PUBKEY_ADDRESS_TABLE +
CREATE_PUBKEY_ADDRESS_INDEX +
CREATE_TXO_TABLE +
CREATE_TXI_TABLE
CREATE_TXO_INDEX +
CREATE_TXI_TABLE +
CREATE_TXI_INDEX
)
@staticmethod
@@ -376,10 +377,10 @@ class BaseDatabase(SQLiteMixin):
}
async def insert_transaction(self, tx):
await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx)))
await self.db.execute(*self._insert_sql('tx', self.tx_to_row(tx)))
async def update_transaction(self, tx):
await self.db.execute_fetchall(*self._update_sql("tx", {
await self.db.execute(*self._update_sql("tx", {
'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
}, 'txid = ?', (tx.id,)))
@@ -390,7 +391,7 @@ class BaseDatabase(SQLiteMixin):
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
conn.execute(*self._insert_sql(
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
)).fetchall()
))
elif txo.script.is_pay_script_hash:
# TODO: implement script hash payments
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
@@ -403,7 +404,7 @@ class BaseDatabase(SQLiteMixin):
'txid': tx.id,
'txoid': txo.id,
'address': address,
}, ignore_duplicate=True)).fetchall()
}, ignore_duplicate=True))
conn.execute(
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
@@ -434,22 +435,25 @@ class BaseDatabase(SQLiteMixin):
async def select_transactions(self, cols, accounts=None, **constraints):
if not set(constraints) & {'txid', 'txid__in'}:
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
assert accounts is not None, "'accounts' argument required when no 'txid' constraint"
constraints.update({
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
})
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
where = f" WHERE account_address.account IN ({account_values})"
constraints['txid__in'] = f"""
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
SELECT txo.txid FROM txo
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
UNION
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
SELECT txi.txid FROM txi
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
"""
return await self.db.execute_fetchall(
*query("SELECT {} FROM tx".format(cols), **constraints)
)
async def get_transactions(self, wallet=None, **constraints):
async def get_transactions(self, **constraints):
accounts = constraints.get('accounts', None)
tx_rows = await self.select_transactions(
'txid, raw, height, position, is_verified',
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
@@ -474,7 +478,7 @@ class BaseDatabase(SQLiteMixin):
annotated_txos.update({
txo.id: txo for txo in
(await self.get_txos(
wallet=wallet,
my_accounts=accounts,
txid__in=txids[offset:offset+step],
))
})
@@ -484,7 +488,7 @@ class BaseDatabase(SQLiteMixin):
referenced_txos.update({
txo.id: txo for txo in
(await self.get_txos(
wallet=wallet,
my_accounts=accounts,
txoid__in=txi_txoids[offset:offset+step],
))
})
@@ -504,7 +508,6 @@ class BaseDatabase(SQLiteMixin):
return txs
async def get_transaction_count(self, **constraints):
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -517,24 +520,23 @@ class BaseDatabase(SQLiteMixin):
return txs[0]
async def select_txos(self, cols, **constraints):
sql = "SELECT {} FROM txo JOIN tx USING (txid)"
if 'accounts' in constraints:
sql += " JOIN account_address USING (address)"
return await self.db.execute_fetchall(*query(sql.format(cols), **constraints))
return await self.db.execute_fetchall(*query(
"SELECT {} FROM txo"
" JOIN pubkey_address USING (address)"
" JOIN tx USING (txid)".format(cols), **constraints
))
async def get_txos(self, wallet=None, no_tx=False, **constraints):
my_accounts = set(a.public_key.address for a in wallet.accounts) if wallet else set()
async def get_txos(self, my_accounts=None, no_tx=False, **constraints):
my_accounts = [
(a.public_key.address if isinstance(a, BaseAccount) else a)
for a in (my_accounts or constraints.get('accounts', []))
]
if 'order_by' not in constraints:
constraints['order_by'] = [
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
]
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"]
rows = await self.select_txos(
"""
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
select group_concat(account||"|"||chain) from account_address
where account_address.address=txo.address
)
""",
"tx.txid, raw, tx.height, tx.position, tx.is_verified, "
"txo.position, chain, account, amount, script",
**constraints
)
txos = []
@@ -543,8 +545,8 @@ class BaseDatabase(SQLiteMixin):
for row in rows:
if no_tx:
txo = output_class(
amount=row[6],
script=output_class.script_class(row[7]),
amount=row[8],
script=output_class.script_class(row[9]),
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
position=row[5]
)
@@ -554,18 +556,12 @@ class BaseDatabase(SQLiteMixin):
row[1], height=row[2], position=row[3], is_verified=row[4]
)
txo = txs[row[0]].outputs[row[5]]
row_accounts = {k: v for k, v in (a.split('|') for a in row[8].split(','))}
account_match = set(row_accounts) & my_accounts
if account_match:
txo.is_my_account = True
txo.is_change = row_accounts[account_match.pop()] == '1'
else:
txo.is_change = txo.is_my_account = False
txo.is_change = row[6] == 1
txo.is_my_account = row[7] in my_accounts
txos.append(txo)
return txos
async def get_txo_count(self, **constraints):
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -585,60 +581,45 @@ class BaseDatabase(SQLiteMixin):
self.constrain_utxo(constraints)
return self.get_txo_count(**constraints)
async def get_balance(self, wallet=None, accounts=None, **constraints):
assert wallet or accounts, \
"'wallet' or 'accounts' constraints required to calculate balance"
constraints['accounts'] = accounts or wallet.accounts
async def get_balance(self, **constraints):
self.constrain_utxo(constraints)
balance = await self.select_txos('SUM(amount)', **constraints)
return balance[0][0] or 0
async def select_addresses(self, cols, **constraints):
return await self.db.execute_fetchall(*query(
"SELECT {} FROM pubkey_address JOIN account_address USING (address)".format(cols),
**constraints
"SELECT {} FROM pubkey_address".format(cols), **constraints
))
async def get_addresses(self, cols=None, **constraints):
cols = cols or (
'address', 'account', 'chain', 'history', 'used_times',
'pubkey', 'chain_code', 'n', 'depth'
)
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
for address in addresses:
address['pubkey'] = PubKey(
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
address.pop('n'), address.pop('depth')
)
return addresses
async def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'),
**constraints):
addresses = await self.select_addresses(', '.join(cols), **constraints)
return rows_to_dict(addresses, cols)
async def get_address_count(self, **constraints):
count = await self.select_addresses('count(*)', **constraints)
return count[0][0]
async def get_address(self, **constraints):
addresses = await self.get_addresses(limit=1, **constraints)
addresses = await self.get_addresses(
cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
limit=1, **constraints
)
if addresses:
return addresses[0]
async def add_keys(self, account, chain, pubkeys):
async def add_keys(self, account, chain, keys):
await self.db.executemany(
"insert or ignore into account_address "
"(account, address, chain, pubkey, chain_code, n, depth) values "
"(?, ?, ?, ?, ?, ?, ?)", ((
account.id, k.address, chain,
sqlite3.Binary(k.pubkey_bytes),
sqlite3.Binary(k.chain_code),
k.n, k.depth
) for k in pubkeys)
)
await self.db.executemany(
"insert or ignore into pubkey_address (address) values (?)",
((pubkey.address,) for pubkey in pubkeys)
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
(
(pubkey.address, account.public_key.address, chain, position,
sqlite3.Binary(pubkey.pubkey_bytes))
for position, pubkey in keys
)
)
async def _set_address_history(self, address, history):
await self.db.execute_fetchall(
await self.db.execute(
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
(history, history.count(':')//2, address)
)
+27 -23
View File
@@ -19,7 +19,6 @@ from torba.client.coinselection import CoinSelector
from torba.client.constants import COIN, NULL_HASH32
from torba.stream import StreamController
from torba.client.hash import hash160, double_sha256, sha256, Base58
from torba.client.bip32 import PubKey, PrivateKey
log = logging.getLogger(__name__)
@@ -161,7 +160,7 @@ class BaseLedger(metaclass=LedgerRegistry):
@classmethod
def is_valid_address(cls, address):
decoded = Base58.decode_check(address)
decoded = Base58.decode(address)
return decoded[0] == cls.pubkey_address_prefix[0]
@classmethod
@@ -179,29 +178,27 @@ class BaseLedger(metaclass=LedgerRegistry):
def add_account(self, account: baseaccount.BaseAccount):
self.accounts.append(account)
async def _get_account_and_address_info_for_address(self, wallet, address):
match = await self.db.get_address(accounts=wallet.accounts, address=address)
async def _get_account_and_address_info_for_address(self, address):
match = await self.db.get_address(address=address)
if match:
for account in wallet.accounts:
for account in self.accounts:
if match['account'] == account.public_key.address:
return account, match
async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]:
match = await self._get_account_and_address_info_for_address(wallet, address)
async def get_private_key_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
if match:
account, address_info = match
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
return None
return account.get_private_key(address_info['chain'], address_info['position'])
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
match = await self._get_account_and_address_info_for_address(wallet, address)
async def get_public_key_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
if match:
_, address_info = match
return address_info['pubkey']
return None
account, address_info = match
return account.get_public_key(address_info['chain'], address_info['position'])
async def get_account_for_address(self, wallet, address):
match = await self._get_account_and_address_info_for_address(wallet, address)
async def get_account_for_address(self, address):
match = await self._get_account_and_address_info_for_address(address)
if match:
return match[0]
@@ -213,12 +210,6 @@ class BaseLedger(metaclass=LedgerRegistry):
estimators.append(utxo.get_estimator(self))
return estimators
async def get_addresses(self, **constraints):
return await self.db.get_addresses(**constraints)
def get_address_count(self, **constraints):
return self.db.get_address_count(**constraints)
async def get_spendable_utxos(self, amount: int, funding_accounts):
async with self._utxo_reservation_lock:
txos = await self.get_effective_amount_estimators(funding_accounts)
@@ -238,16 +229,29 @@ class BaseLedger(metaclass=LedgerRegistry):
def release_tx(self, tx):
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
def constraint_account_or_all(self, constraints):
if 'accounts' in constraints:
return
account = constraints.pop('account', None)
if account:
constraints['accounts'] = [account]
else:
constraints['accounts'] = self.accounts
def get_utxos(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_utxo_count(**constraints)
def get_transactions(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transactions(**constraints)
def get_transaction_count(self, **constraints):
self.constraint_account_or_all(constraints)
return self.db.get_transaction_count(**constraints)
async def get_local_status_and_history(self, address, history=None):
@@ -558,7 +562,7 @@ class BaseLedger(metaclass=LedgerRegistry):
addresses.add(
self.hash160_to_address(txo.script.values['pubkey_hash'])
)
records = await self.db.get_addresses(address__in=addresses)
records = await self.db.get_addresses(cols=('address',), address__in=addresses)
_, pending = await asyncio.wait([
self.on_transaction.where(partial(
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
+11 -12
View File
@@ -1,6 +1,6 @@
import asyncio
import logging
from typing import Type, MutableSequence, MutableMapping, Optional
from typing import Type, MutableSequence, MutableMapping
from torba.client.baseledger import BaseLedger, LedgerRegistry
from torba.client.wallet import Wallet, WalletStorage
@@ -41,6 +41,16 @@ class BaseWalletManager:
self.wallets.append(wallet)
return wallet
async def get_detailed_accounts(self, **kwargs):
ledgers = {}
for i, account in enumerate(self.accounts):
details = await account.get_details(**kwargs)
details['is_default'] = i == 0
ledger_id = account.ledger.get_id()
ledgers.setdefault(ledger_id, [])
ledgers[ledger_id].append(details)
return ledgers
@property
def default_wallet(self):
for wallet in self.wallets:
@@ -68,14 +78,3 @@ class BaseWalletManager:
l.stop() for l in self.ledgers.values()
))
self.running = False
def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet:
if wallet_id is None:
return self.default_wallet
return self.get_wallet_or_error(wallet_id)
def get_wallet_or_error(self, wallet_id: str) -> Wallet:
for wallet in self.wallets:
if wallet.id == wallet_id:
return wallet
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
-3
View File
@@ -258,9 +258,6 @@ class BaseNetwork:
self.client.abort()
raise asyncio.CancelledError()
def get_server_features(self):
return self.rpc('server.features', (), restricted=True)
class SessionPool:
+11 -23
View File
@@ -1,6 +1,6 @@
import logging
import typing
from typing import List, Iterable, Optional, Tuple
from typing import List, Iterable, Optional
from binascii import hexlify
from torba.client.basescript import BaseInputScript, BaseOutputScript
@@ -12,7 +12,7 @@ from torba.client.util import ReadOnlyList
from torba.client.errors import InsufficientFundsError
if typing.TYPE_CHECKING:
from torba.client import baseledger, wallet as basewallet
from torba.client import baseledger
log = logging.getLogger()
@@ -431,32 +431,21 @@ class BaseTransaction:
self.locktime = stream.read_uint32()
@classmethod
def ensure_all_have_same_ledger_and_wallet(
cls, funding_accounts: Iterable[BaseAccount],
change_account: BaseAccount = None) -> Tuple['baseledger.BaseLedger', 'basewallet.Wallet']:
ledger = wallet = None
def ensure_all_have_same_ledger(cls, funding_accounts: Iterable[BaseAccount],
change_account: BaseAccount = None) -> 'baseledger.BaseLedger':
ledger = None
for account in funding_accounts:
if ledger is None:
ledger = account.ledger
wallet = account.wallet
if ledger != account.ledger:
raise ValueError(
'All funding accounts used to create a transaction must be on the same ledger.'
)
if wallet != account.wallet:
raise ValueError(
'All funding accounts used to create a transaction must be from the same wallet.'
)
if change_account is not None:
if change_account.ledger != ledger:
raise ValueError('Change account must use same ledger as funding accounts.')
if change_account.wallet != wallet:
raise ValueError('Change account must use same wallet as funding accounts.')
if change_account is not None and change_account.ledger != ledger:
raise ValueError('Change account must use same ledger as funding accounts.')
if ledger is None:
raise ValueError('No ledger found.')
if wallet is None:
raise ValueError('No wallet found.')
return ledger, wallet
return ledger
@classmethod
async def create(cls, inputs: Iterable[BaseInput], outputs: Iterable[BaseOutput],
@@ -469,7 +458,7 @@ class BaseTransaction:
.add_inputs(inputs) \
.add_outputs(outputs)
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
# value of the outputs plus associated fees
cost = (
@@ -535,15 +524,14 @@ class BaseTransaction:
return hash_type
async def sign(self, funding_accounts: Iterable[BaseAccount]):
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
ledger = self.ensure_all_have_same_ledger(funding_accounts)
for i, txi in enumerate(self._inputs):
assert txi.script is not None
assert txi.txo_ref.txo is not None
txo_script = txi.txo_ref.txo.script
if txo_script.is_pay_pubkey_hash:
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
private_key = await ledger.get_private_key_for_address(wallet, address)
assert private_key is not None, 'Cannot find private key for signing output.'
private_key = await ledger.get_private_key_for_address(address)
tx = self._serialize_for_signature(i)
txi.script.values['signature'] = \
private_key.sign(tx) + bytes((self.signature_hash_type(1),))
+7 -41
View File
@@ -3,7 +3,7 @@ import stat
import json
import zlib
import typing
from typing import List, Sequence, MutableSequence, Optional
from typing import Sequence, MutableSequence
from hashlib import sha256
from operator import attrgetter
from torba.client.hash import better_aes_encrypt, better_aes_decrypt
@@ -25,51 +25,12 @@ class Wallet:
self.accounts = accounts or []
self.storage = storage or WalletStorage()
@property
def id(self):
if self.storage.path:
return os.path.basename(self.storage.path)
return self.name
def add_account(self, account: 'baseaccount.BaseAccount'):
def add_account(self, account):
self.accounts.append(account)
def generate_account(self, ledger: 'baseledger.BaseLedger') -> 'baseaccount.BaseAccount':
return ledger.account_class.generate(ledger, self)
@property
def default_account(self) -> Optional['baseaccount.BaseAccount']:
for account in self.accounts:
return account
return None
def get_account_or_default(self, account_id: str) -> Optional['baseaccount.BaseAccount']:
if account_id is None:
return self.default_account
return self.get_account_or_error(account_id)
def get_account_or_error(self, account_id: str) -> 'baseaccount.BaseAccount':
for account in self.accounts:
if account.id == account_id:
return account
raise ValueError(f"Couldn't find account: {account_id}.")
def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['baseaccount.BaseAccount']:
return [
self.get_account_or_error(account_id)
for account_id in account_ids
] if account_ids else self.accounts
async def get_detailed_accounts(self, **kwargs):
ledgers = {}
for i, account in enumerate(self.accounts):
details = await account.get_details(**kwargs)
details['is_default'] = i == 0
ledger_id = account.ledger.get_id()
ledgers.setdefault(ledger_id, [])
ledgers[ledger_id].append(details)
return ledgers
@classmethod
def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':
json_dict = storage.read()
@@ -93,6 +54,11 @@ class Wallet:
def save(self):
self.storage.write(self.to_dict())
@property
def default_account(self):
for account in self.accounts:
return account
@property
def hash(self) -> bytes:
h = sha256()
+1 -3
View File
@@ -151,9 +151,7 @@ class WalletNode:
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
self.data_path = tempfile.mkdtemp()
wallets_dir = os.path.join(self.data_path, 'wallets')
os.mkdir(wallets_dir)
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
wallet_file_name = os.path.join(self.data_path, 'my_wallet.json')
with open(wallet_file_name, 'w') as wallet_file:
wallet_file.write('{"version": 1, "accounts": []}\n')
self.manager = self.manager_class.from_config({
-2
View File
@@ -80,8 +80,6 @@ class Env:
self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
self.description = self.default('DESCRIPTION', '')
self.daily_fee = self.integer('DAILY_FEE', 0)
# Identities
clearnet_identity = self.clearnet_identity()
-3
View File
@@ -767,9 +767,6 @@ class ElectrumX(SessionBase):
'protocol_min': min_str,
'protocol_max': max_str,
'genesis_hash': env.coin.GENESIS_HASH,
'description': env.description,
'payment_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
}
+2 -2
View File
@@ -13,6 +13,6 @@ changedir = {toxinidir}/tests
setenv =
integration: TORBA_LEDGER={envname}
commands =
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.unit
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.unit
integration: orchstr8 download
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.integration
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.integration