Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6910b7ce1e | ||
|
|
3f89ed4579 | ||
|
|
02b01b89f2 | ||
|
|
4a6af0b938 | ||
|
|
9c5b4e3cc9 | ||
|
|
b3f087dbc7 | ||
|
|
aa90257ec7 | ||
|
|
348bbc81ca | ||
|
|
4231e7f3ff | ||
|
|
4cbc82619d | ||
|
|
1b0d802941 | ||
|
|
dc72a1e58d | ||
|
|
1d5333c737 | ||
|
|
d289041ff3 | ||
|
|
f287483397 | ||
|
|
1ac3b6c173 | ||
|
|
1d090765ba | ||
|
|
6c6050528a | ||
|
|
835d376ca6 | ||
|
|
68e9fd8541 | ||
|
|
e49f8f8c61 | ||
|
|
b396322f1e | ||
|
|
ea434e3469 | ||
|
|
84587ac232 | ||
|
|
9fbc83fe9d | ||
|
|
965972a719 | ||
|
|
34d19c4a72 | ||
|
|
9b0910767d | ||
|
|
32cc077e1f | ||
|
|
1e7bec48b6 | ||
|
|
c035837a14 | ||
|
|
cb5eb8983b | ||
|
|
9c86cb7954 | ||
|
|
ef94c1eccc | ||
|
|
64eaa5e3aa | ||
|
|
826f584aa8 | ||
|
|
c5530aee5f | ||
|
|
817f01593e | ||
|
|
79727f0e97 | ||
|
|
abac82c1c6 | ||
|
|
02c51c6059 | ||
|
|
46b45d0387 | ||
|
|
e6c549c457 | ||
|
|
1aa2983216 | ||
|
|
a095f89c81 | ||
|
|
287902f01b | ||
|
|
7dc3e4e7a1 | ||
|
|
0b31aad901 | ||
|
|
1791d8e4ee | ||
|
|
b7c2054a66 | ||
|
|
81f16d7061 |
No files matched your search
+447
-107
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.41.0"
|
||||
__version__ = "0.42.1"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -83,8 +83,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
self.writer.write(data)
|
||||
|
||||
+13
-3
@@ -148,7 +148,12 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
@staticmethod
|
||||
def _parse_list(l):
|
||||
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
|
||||
if l == ['null']:
|
||||
return None
|
||||
assert len(l) == 2, (
|
||||
'Max key fee is made up of either two values: '
|
||||
'"AMOUNT CURRENCY", or "null" (to set no limit)'
|
||||
)
|
||||
try:
|
||||
amount = float(l[0])
|
||||
except ValueError:
|
||||
@@ -176,7 +181,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
nargs=2,
|
||||
nargs='+',
|
||||
metavar=('AMOUNT', 'CURRENCY'),
|
||||
default=NOT_SET
|
||||
)
|
||||
@@ -480,6 +485,10 @@ class Config(CLIConfig):
|
||||
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
wallets = Strings(
|
||||
"Wallet files in 'wallet_dir' to load at startup.",
|
||||
['default_wallet']
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
@@ -527,7 +536,8 @@ class Config(CLIConfig):
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
|
||||
+22
-3
@@ -7,6 +7,7 @@ import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
import logging.handlers
|
||||
import typing
|
||||
from docopt import docopt
|
||||
|
||||
import aiohttp
|
||||
@@ -21,6 +22,23 @@ log = logging.getLogger('lbry')
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
async def _task_decorator(coro: typing.Coroutine):
|
||||
try:
|
||||
return await coro
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except BaseException as e:
|
||||
log.exception('unhandled error in task')
|
||||
raise e
|
||||
|
||||
|
||||
def task_factory(loop: asyncio.AbstractEventLoop, coro: typing.Coroutine):
|
||||
task = asyncio.tasks.Task(_task_decorator(coro), loop=loop)
|
||||
if task._source_traceback:
|
||||
del task._source_traceback[-1]
|
||||
return task
|
||||
|
||||
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
@@ -45,7 +63,7 @@ async def execute_command(conf, method, params, callback=display):
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
@@ -65,10 +83,10 @@ def remove_brackets(key):
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = None
|
||||
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
@@ -241,6 +259,7 @@ def run_daemon(args: list, conf: Config):
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.set_task_factory(task_factory)
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
+410
-242
File diff suppressed because it is too large.
Load diff
@@ -6,6 +6,8 @@ from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.bip32 import PubKey
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.ledger import MainNetLedger, Account
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
@@ -67,6 +69,13 @@ def encode_account_doc():
|
||||
}
|
||||
|
||||
|
||||
def encode_wallet_doc():
|
||||
return {
|
||||
'id': 'wallet_id',
|
||||
'name': 'optional wallet name',
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
@@ -110,6 +119,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
@@ -118,6 +129,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -219,6 +232,12 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_wallet(self, wallet):
|
||||
return {
|
||||
'id': wallet.id,
|
||||
'name': wallet.name
|
||||
}
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -51,7 +52,6 @@ def migrate_db(conf, start, end):
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
|
||||
@@ -125,10 +125,9 @@ def do_migration(conf):
|
||||
name_metadata_cursor.execute("select txid, n, sd_hash from name_metadata").fetchall()
|
||||
}
|
||||
|
||||
sd_hash_to_stream_hash = {
|
||||
sd_hash: stream_hash for (sd_hash, stream_hash) in
|
||||
sd_hash_to_stream_hash = dict(
|
||||
lbryfile_cursor.execute("select sd_blob_hash, stream_hash from lbry_file_descriptors").fetchall()
|
||||
}
|
||||
)
|
||||
|
||||
stream_hash_to_stream_blobs = {}
|
||||
|
||||
|
||||
@@ -101,6 +101,9 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
stream_hashes = tuple(
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
|
||||
)
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
@@ -108,9 +111,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
"order by c.rowid desc", stream_hashes):
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -137,7 +138,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
list(signed_claims.keys())):
|
||||
tuple(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
@@ -147,35 +148,35 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs)
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
@@ -191,7 +192,7 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
@@ -293,17 +294,17 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
)
|
||||
).fetchall()
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
@@ -317,9 +318,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
[(int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes]
|
||||
)
|
||||
((int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run(_update_last_announced_blobs)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
@@ -330,11 +331,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1, next_announce_time=? "
|
||||
"where blob_hash=? and status='finished'", (int(now), blob_hash)
|
||||
)
|
||||
).fetchall()
|
||||
else:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
|
||||
)
|
||||
).fetchall()
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
@@ -347,22 +348,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
).fetchall()
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
return [b[0] for b in r.fetchall()]
|
||||
).fetchall()
|
||||
return [b[0] for b in r]
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
@@ -370,22 +371,18 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
finished_blob_hashes = tuple(
|
||||
blob_hash for (blob_hash, ) in transaction.execute(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
).fetchall()
|
||||
)
|
||||
finished_blobs_set = set(finished_blob_hashes)
|
||||
to_update_set = finished_blobs_set.difference(blob_files)
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
((blob_hash, ) for blob_hash in to_update_set)
|
||||
).fetchall()
|
||||
return blob_files.intersection(finished_blobs_set)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
@@ -478,13 +475,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
else:
|
||||
removed.append((stream_hash,))
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
@@ -492,7 +488,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
@@ -501,22 +497,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
@@ -537,13 +533,13 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
((stream_hash, ) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
@@ -555,14 +551,16 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
transaction.execute(
|
||||
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
|
||||
).fetchall()
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
).fetchall()
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -581,7 +579,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for support_info in _batched_select(
|
||||
transaction,
|
||||
"select * from support where claim_id in {}",
|
||||
tuple(claim_ids)
|
||||
claim_ids
|
||||
)
|
||||
]
|
||||
|
||||
@@ -612,7 +610,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
)
|
||||
).fetchall()
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
@@ -699,7 +697,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
# update the claim associated to the file
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
|
||||
).fetchall()
|
||||
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
@@ -722,11 +722,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
return self.db.execute_fetchall(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
)
|
||||
return self.db.execute(
|
||||
return self.db.execute_fetchall(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ def get_platform() -> dict:
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
import distro
|
||||
import distro # pylint: disable=import-outside-toplevel
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ types_map = {
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.flac': ('audio/flac', 'audio'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
|
||||
@@ -4,7 +4,10 @@ from typing import NamedTuple, Tuple
|
||||
|
||||
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.com/
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
@@ -17,7 +20,7 @@ def _create_url_regex():
|
||||
|
||||
def _claim(name, prefix=""):
|
||||
return _group(
|
||||
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
|
||||
@@ -401,8 +401,8 @@ class StreamManager:
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
), 5) if self.config.max_key_fee else None
|
||||
if max_fee_amount and fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
|
||||
+10
-10
@@ -139,34 +139,34 @@ class Account(BaseAccount):
|
||||
return details
|
||||
|
||||
def get_transaction_history(self, **constraints):
|
||||
return self.ledger.get_transaction_history(account=self, **constraints)
|
||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
return self.ledger.get_transaction_history_count(account=self, **constraints)
|
||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.get_claims(account=self, **constraints)
|
||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
return self.ledger.get_claim_count(account=self, **constraints)
|
||||
return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
return self.ledger.get_streams(account=self, **constraints)
|
||||
return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
return self.ledger.get_stream_count(account=self, **constraints)
|
||||
return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
return self.ledger.get_channels(account=self, **constraints)
|
||||
return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.get_channel_count(account=self, **constraints)
|
||||
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.ledger.get_supports(account=self, **constraints)
|
||||
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
return self.ledger.get_support_count(account=self, **constraints)
|
||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_summary(self):
|
||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
||||
|
||||
@@ -22,18 +22,18 @@ class WalletDatabase(BaseDatabase):
|
||||
claim_id text,
|
||||
claim_name text
|
||||
);
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
create index if not exists txo_claim_id_idx on txo (claim_id);
|
||||
create index if not exists txo_txo_type_idx on txo (txo_type);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXO_INDEX +
|
||||
BaseDatabase.CREATE_TXI_TABLE +
|
||||
BaseDatabase.CREATE_TXI_INDEX
|
||||
BaseDatabase.PRAGMAS +
|
||||
BaseDatabase.CREATE_ACCOUNT_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXI_TABLE
|
||||
)
|
||||
|
||||
def txo_to_row(self, tx, address, txo):
|
||||
@@ -50,18 +50,16 @@ class WalletDatabase(BaseDatabase):
|
||||
row['claim_name'] = txo.claim_name
|
||||
return row
|
||||
|
||||
async def get_txos(self, **constraints) -> List[Output]:
|
||||
my_accounts = constraints.get('my_accounts', constraints.get('accounts', []))
|
||||
|
||||
txos = await super().get_txos(**constraints)
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints) -> List[Output]:
|
||||
txos = await super().get_txos(wallet=wallet, no_tx=no_tx, **constraints)
|
||||
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and my_accounts:
|
||||
for account in my_accounts:
|
||||
if txo.claim.is_channel and wallet:
|
||||
for account in wallet.accounts:
|
||||
private_key = account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
@@ -73,7 +71,7 @@ class WalletDatabase(BaseDatabase):
|
||||
channels = {
|
||||
txo.claim_id: txo for txo in
|
||||
(await self.get_claims(
|
||||
my_accounts=my_accounts,
|
||||
wallet=wallet,
|
||||
claim_id__in=channel_ids
|
||||
))
|
||||
}
|
||||
@@ -134,20 +132,20 @@ class WalletDatabase(BaseDatabase):
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
" SELECT address from account_address WHERE account = ?"
|
||||
" )", (account.public_key.address, )
|
||||
)
|
||||
|
||||
def get_supports_summary(self, account_id):
|
||||
return self.db.execute_fetchall(f"""
|
||||
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
|
||||
(txo.txid in
|
||||
(select txi.txid from txi join pubkey_address a on txi.address = a.address
|
||||
(select txi.txid from txi join account_address a on txi.address = a.address
|
||||
where a.account = ?)) as from_me,
|
||||
(txo.address in (select address from pubkey_address where account=?)) as to_me,
|
||||
(txo.address in (select address from account_address where account=?)) as to_me,
|
||||
tx.height
|
||||
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
|
||||
""", (account_id, account_id))
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from functools import partial
|
||||
from typing import Tuple, List
|
||||
from datetime import datetime
|
||||
|
||||
@@ -60,7 +61,8 @@ class MainNetLedger(BaseLedger):
|
||||
return outputs.inflate(txs), outputs.offset, outputs.total
|
||||
|
||||
async def resolve(self, urls):
|
||||
txos = (await self._inflate_outputs(self.network.resolve(urls)))[0]
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
txos = (await self._inflate_outputs(resolve(urls)))[0]
|
||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||
result = {}
|
||||
for url, txo in zip(urls, txos):
|
||||
@@ -121,39 +123,30 @@ class MainNetLedger(BaseLedger):
|
||||
return super().get_utxo_count(**constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claims(**constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claim_count(**constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_streams(**constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_stream_count(**constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channels(**constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channel_count(**constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_supports(**constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_support_count(**constraints)
|
||||
|
||||
async def get_transaction_history(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
txs = await self.db.get_transactions(**constraints)
|
||||
headers = self.headers
|
||||
history = []
|
||||
@@ -249,7 +242,6 @@ class MainNetLedger(BaseLedger):
|
||||
return history
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
|
||||
|
||||
@@ -138,20 +138,23 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if not os.path.exists(wallets_directory):
|
||||
os.mkdir(wallets_directory)
|
||||
|
||||
wallet_file_path = os.path.join(wallets_directory, 'default_wallet')
|
||||
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(wallet_file_path)
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(
|
||||
os.path.join(wallets_directory, 'default_wallet')
|
||||
)
|
||||
|
||||
manager = cls.from_config({
|
||||
'ledgers': {ledger_id: ledger_config},
|
||||
'wallets': [wallet_file_path]
|
||||
'wallets': [
|
||||
os.path.join(wallets_directory, wallet_file) for wallet_file in settings.wallets
|
||||
]
|
||||
})
|
||||
ledger = manager.get_or_create_ledger(ledger_id)
|
||||
ledger.coin_selection_strategy = settings.coin_selection_strategy
|
||||
if manager.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', wallet_file_path)
|
||||
manager.default_wallet.generate_account(ledger)
|
||||
manager.default_wallet.save()
|
||||
default_wallet = manager.default_wallet
|
||||
if default_wallet.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', default_wallet.id)
|
||||
default_wallet.generate_account(ledger)
|
||||
default_wallet.save()
|
||||
if receiving_addresses or change_addresses:
|
||||
if not os.path.exists(ledger.path):
|
||||
os.mkdir(ledger.path)
|
||||
|
||||
@@ -2,6 +2,7 @@ import time
|
||||
import struct
|
||||
import sqlite3
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from typing import Tuple, List, Dict, Union, Type, Optional
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
@@ -336,7 +337,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from math import sqrt
|
||||
|
||||
TRENDING_WINDOW = 650 # number of blocks, ~24hr period
|
||||
TRENDING_WINDOW = 134 # number of blocks in ~6hr period (21600 seconds / 161 seconds per block)
|
||||
TRENDING_DATA_POINTS = 7 # WINDOW * DATA_POINTS = ~1 week worth of trending data
|
||||
|
||||
CREATE_TREND_TABLE = """
|
||||
|
||||
@@ -203,7 +203,7 @@ class Transaction(BaseTransaction):
|
||||
|
||||
@classmethod
|
||||
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address))
|
||||
return cls.create([], [output], funding_accounts, change_account)
|
||||
|
||||
@@ -211,7 +211,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_create(
|
||||
cls, name: str, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
claim_output = Output.pay_claim_name_pubkey_hash(
|
||||
amount, name, claim, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -223,7 +223,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_update(
|
||||
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
claim, ledger.address_to_hash160(holding_address)
|
||||
@@ -239,7 +239,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -248,7 +248,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
claim_output = Output.purchase_claim_pubkey_hash(
|
||||
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
|
||||
)
|
||||
|
||||
@@ -13,13 +13,15 @@ from lbry.extras.daemon.Daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from lbry.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
|
||||
encode_wallet_doc
|
||||
)
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
RETURN_DOCS = {
|
||||
'Account': encode_account_doc(),
|
||||
'Wallet': encode_wallet_doc(),
|
||||
'File': encode_file_doc(),
|
||||
'Transaction': encode_tx_doc(),
|
||||
'Output': encode_txo_doc(),
|
||||
@@ -103,6 +105,13 @@ class Examples(CommandTestCase):
|
||||
'preference', 'get'
|
||||
)
|
||||
|
||||
# wallets
|
||||
|
||||
await r(
|
||||
'List your wallets',
|
||||
'wallet', 'list'
|
||||
)
|
||||
|
||||
# accounts
|
||||
|
||||
await r(
|
||||
|
||||
@@ -11,14 +11,6 @@ def extract(d, keys):
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
async def test_sqlite_binding_error(self):
|
||||
tasks = [
|
||||
self.loop.create_task(self.daemon.jsonrpc_account_create('second account' + str(x))) for x in range(100)
|
||||
]
|
||||
await asyncio.wait(tasks)
|
||||
for result in tasks:
|
||||
self.assertFalse(isinstance(result.result(), Exception))
|
||||
|
||||
async def test_account_list_set_create_remove_add(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
@@ -93,7 +85,7 @@ class AccountManagement(CommandTestCase):
|
||||
stream_list = self.daemon.jsonrpc_stream_list
|
||||
support_list = self.daemon.jsonrpc_support_list
|
||||
utxo_list = self.daemon.jsonrpc_utxo_list
|
||||
default_account = self.daemon.default_account
|
||||
default_account = self.wallet.default_account
|
||||
second_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
|
||||
tx = await self.daemon.jsonrpc_account_send(
|
||||
@@ -179,3 +171,9 @@ class AccountManagement(CommandTestCase):
|
||||
})
|
||||
self.assertEqual(history[6]['value'], '0.0')
|
||||
self.assertEqual(history[7]['value'], '10.0')
|
||||
|
||||
async def test_address_validation(self):
|
||||
address = await self.daemon.jsonrpc_address_unused()
|
||||
bad_address = address[0:20] + '9999999' + address[27:]
|
||||
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
|
||||
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
|
||||
@@ -466,45 +466,47 @@ class ChannelCommands(CommandTestCase):
|
||||
'title': 'foo', 'email': 'new@email.com'}
|
||||
)
|
||||
|
||||
# send channel to someone else
|
||||
# move channel to another account
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
# before moving
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
|
||||
|
||||
other_address = await account2.receiving.get_or_create_usable_address()
|
||||
tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
|
||||
|
||||
# after sending
|
||||
# after moving
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
|
||||
|
||||
# shoud not have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNone(txo.private_key)
|
||||
|
||||
# send the private key too
|
||||
private_key = self.account.get_channel_private_key(unhexlify(channel['public_key']))
|
||||
account2.add_channel_private_key(private_key)
|
||||
|
||||
# now should have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNotNone(txo.private_key)
|
||||
|
||||
async def test_channel_export_import_into_new_account(self):
|
||||
async def test_channel_export_import_before_sending_channel(self):
|
||||
# export
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
channel_private_key = (await self.account.get_channels())[0].private_key
|
||||
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
|
||||
|
||||
# import
|
||||
daemon2 = await self.add_daemon()
|
||||
self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
|
||||
await daemon2.jsonrpc_channel_import(exported_data)
|
||||
channels = await daemon2.jsonrpc_channel_list()
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
# second wallet can't update until channel is sent to it
|
||||
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
# now send the channel as well
|
||||
await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused())
|
||||
|
||||
# second wallet should be able to update now
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
|
||||
class StreamCommands(ClaimTestCase):
|
||||
|
||||
@@ -565,7 +567,7 @@ class StreamCommands(ClaimTestCase):
|
||||
async def test_publishing_checks_all_accounts_for_channel(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
|
||||
@@ -782,7 +784,7 @@ class StreamCommands(ClaimTestCase):
|
||||
|
||||
# send claim to someone else
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
|
||||
@@ -1079,13 +1081,12 @@ class StreamCommands(ClaimTestCase):
|
||||
class SupportCommands(CommandTestCase):
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
|
||||
account2 = self.daemon.get_account_or_error(account2_id)
|
||||
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
|
||||
account2 = wallet2.accounts[0]
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2')
|
||||
))
|
||||
await self.on_transaction_dict(result)
|
||||
|
||||
@@ -1103,7 +1104,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '1.0', True, account2_id, funding_account_ids=[account2_id])
|
||||
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(tip['txid'])
|
||||
|
||||
@@ -1122,7 +1123,7 @@ class SupportCommands(CommandTestCase):
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
@@ -1134,7 +1135,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '2.0', False, account2_id, funding_account_ids=[account2_id])
|
||||
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(support['txid'])
|
||||
|
||||
@@ -1143,7 +1144,7 @@ class SupportCommands(CommandTestCase):
|
||||
await self.assertBalance(account2, '1.999717')
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
|
||||
|
||||
@@ -277,6 +277,33 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0')
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address)
|
||||
|
||||
async def test_null_max_key_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
self.daemon.conf.max_key_fee = None
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
await self.assertBalance(self.account, '9.483893')
|
||||
response = await self.daemon.jsonrpc_get('lbry://somename')
|
||||
await self.ledger.wait(response.content_fee)
|
||||
await self.assertBalance(self.account, '8.483769')
|
||||
|
||||
# Assert the file downloads
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
# Assert the transaction is recorded to the blockchain
|
||||
starting_balance = await self.blockchain.get_balance()
|
||||
await self.generate(1)
|
||||
block_reward_and_claim_fee = 2.0
|
||||
self.assertEqual(
|
||||
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
|
||||
)
|
||||
|
||||
async def test_null_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
tx = await self.stream_create(
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AddressManagement(CommandTestCase):
|
||||
|
||||
async def test_address_list(self):
|
||||
addresses = await self.out(self.daemon.jsonrpc_address_list())
|
||||
self.assertEqual(27, len(addresses))
|
||||
|
||||
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
|
||||
self.assertEqual(1, len(single))
|
||||
self.assertEqual(single[0], addresses[11])
|
||||
@@ -86,6 +86,12 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
self.assertEqual(claim['name'], 'gibberish')
|
||||
self.assertNotIn('value', claim)
|
||||
|
||||
# resolve retries
|
||||
await self.conductor.spv_node.stop()
|
||||
resolving_future = asyncio.ensure_future(self.resolve('foo'))
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
|
||||
|
||||
async def test_winning_by_effective_amount(self):
|
||||
# first one remains winner unless something else changes
|
||||
claim_id1 = self.get_claim_id(
|
||||
|
||||
@@ -55,10 +55,41 @@ class TestURLParsing(unittest.TestCase):
|
||||
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
|
||||
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
|
||||
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
|
||||
# unicode regex edges
|
||||
_url = lambda name: url(name, stream_name=name)
|
||||
_url('\u0009')
|
||||
_url('\u000a')
|
||||
_url('\u000d')
|
||||
_url('\u0020')
|
||||
_url('\uD799')
|
||||
_url('\uE000')
|
||||
_url('\uFFFD')
|
||||
|
||||
def test_parser_invalid_urls(self):
|
||||
fail = self._fail_url
|
||||
fail("lbry://")
|
||||
fail("lbry://\u0000")
|
||||
fail("lbry://\u0008")
|
||||
fail("lbry://\u000b")
|
||||
fail("lbry://\u000c")
|
||||
fail("lbry://\u000e")
|
||||
fail("lbry://\u001f")
|
||||
fail("lbry://\uD800")
|
||||
fail("lbry://\uDFFF")
|
||||
fail("lbry://\uDFFE")
|
||||
fail("lbry://\uFFFF")
|
||||
fail("lbry://;")
|
||||
fail("lbry://\"")
|
||||
fail("lbry://\\")
|
||||
fail("lbry:///")
|
||||
fail("lbry://<") and fail("lbry://>")
|
||||
fail("lbry://{") and fail("lbry://}")
|
||||
fail("lbry://[") and fail("lbry://]")
|
||||
fail("lbry://%")
|
||||
fail("lbry://|")
|
||||
fail("lbry://^")
|
||||
fail("lbry://~")
|
||||
fail("lbry://`")
|
||||
fail("lbry://test:3$1")
|
||||
fail("lbry://test$1:1")
|
||||
fail("lbry://test#x")
|
||||
|
||||
@@ -51,6 +51,7 @@ class CLITest(AsyncioTestCase):
|
||||
self.assertEqual('3', normalize_value('3', key="name"))
|
||||
self.assertEqual('3', normalize_value('3', key="download_directory"))
|
||||
self.assertEqual('3', normalize_value('3', key="channel_name"))
|
||||
self.assertEqual('3', normalize_value('3', key="claim_name"))
|
||||
|
||||
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
|
||||
|
||||
|
||||
@@ -222,6 +222,10 @@ class ConfigurationTests(unittest.TestCase):
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
args = parser.parse_args(['--max-key-fee', 'null'])
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
# set
|
||||
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
|
||||
c = Config.create_from_arguments(args)
|
||||
|
||||
@@ -61,13 +61,17 @@ class TestAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx'
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR'
|
||||
'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(private_key)
|
||||
|
||||
def test_load_and_save_account(self):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -16,6 +17,38 @@ class NetworkTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_header.first
|
||||
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
|
||||
|
||||
async def test_server_features(self):
|
||||
self.assertEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
'daily_fee': 0,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
address = (await self.account.get_addresses(limit=1))[0]
|
||||
os.environ.update({
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'DONATION_ADDRESS': address,
|
||||
'DAILY_FEE': '42'})
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
'payment_address': address,
|
||||
'daily_fee': 42,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
@@ -82,6 +115,13 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_online_but_still_unavailable(self):
|
||||
# Edge case. See issue #2445 for context
|
||||
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.response_time = None
|
||||
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
await self.assertBalance(account1, '0.0')
|
||||
await self.assertBalance(account2, '0.0')
|
||||
|
||||
addresses = await self.account.receiving.get_addresses()
|
||||
addresses = await account1.receiving.get_addresses()
|
||||
txids = await asyncio.gather(*(
|
||||
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
|
||||
))
|
||||
|
||||
@@ -60,7 +60,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
await account.receiving._generate_keys(8, 11)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[r['pubkey'].n for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
||||
)
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
self.assertEqual(len(new_keys), 8)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[r['pubkey'].n for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
|
||||
)
|
||||
|
||||
@@ -125,14 +125,18 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address('1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J'
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9xV7rhbg6M4yWrdTeLorz3Q1GrQb4aQzzGWboP3du7W7UUztzNTUrEYTnDfz7o'
|
||||
'ptBygDxXYRppyiuenJpoBTgYP2C26E1Ah5FEALM24CsWi'
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
@@ -275,12 +279,18 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
self.assertEqual(len(new_keys), 1)
|
||||
self.assertEqual(new_keys[0], account.public_key.address)
|
||||
records = await account.receiving.get_address_records()
|
||||
pubkey = records[0].pop('pubkey')
|
||||
self.assertEqual(records, [{
|
||||
'position': 0, 'chain': 0,
|
||||
'chain': 0,
|
||||
'account': account.public_key.address,
|
||||
'address': account.public_key.address,
|
||||
'history': None,
|
||||
'used_times': 0
|
||||
}])
|
||||
self.assertEqual(
|
||||
pubkey.extended_key_string(),
|
||||
account.public_key.extended_key_string()
|
||||
)
|
||||
|
||||
# case #1: no new addresses needed
|
||||
empty = await account.receiving.ensure_address_gap()
|
||||
@@ -333,14 +343,18 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], account.public_key.address)
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address(address[0])
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, address[0]
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp'
|
||||
'5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna',
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import os
|
||||
import asyncio
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
@@ -196,13 +199,14 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': ledger_class.database_class(':memory:'),
|
||||
'headers': ledger_class.headers_class(':memory:'),
|
||||
})
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.ledger.db.close()
|
||||
|
||||
async def create_account(self):
|
||||
account = self.ledger.account_class.generate(self.ledger, Wallet())
|
||||
async def create_account(self, wallet=None):
|
||||
account = self.ledger.account_class.generate(self.ledger, wallet or self.wallet)
|
||||
await account.ensure_address_gap()
|
||||
return account
|
||||
|
||||
@@ -261,7 +265,8 @@ class TestQueries(AsyncioTestCase):
|
||||
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
|
||||
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
|
||||
for limit in range(variable_limit-2, variable_limit+2):
|
||||
txs = await self.ledger.get_transactions(limit=limit, order_by='height asc')
|
||||
txs = await self.ledger.get_transactions(
|
||||
accounts=self.wallet.accounts, limit=limit, order_by='height asc')
|
||||
self.assertEqual(len(txs), limit)
|
||||
inputs, outputs, last_tx = set(), set(), txs[0]
|
||||
for tx in txs[1:]:
|
||||
@@ -271,19 +276,23 @@ class TestQueries(AsyncioTestCase):
|
||||
last_tx = tx
|
||||
|
||||
async def test_queries(self):
|
||||
self.assertEqual(0, await self.ledger.db.get_address_count())
|
||||
account1 = await self.create_account()
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count())
|
||||
account2 = await self.create_account()
|
||||
self.assertEqual(52, await self.ledger.db.get_address_count())
|
||||
wallet1 = Wallet()
|
||||
account1 = await self.create_account(wallet1)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1]))
|
||||
wallet2 = Wallet()
|
||||
account2 = await self.create_account(wallet2)
|
||||
account3 = await self.create_account(wallet2)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2]))
|
||||
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count())
|
||||
self.assertEqual([], await self.ledger.db.get_utxos())
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx1 = await self.create_tx_from_nothing(account1, 1)
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -291,20 +300,28 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2)
|
||||
tx2b = await self.create_tx_from_nothing(account3, 2)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -313,22 +330,24 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual([3, 2, 1], [tx.height for tx in txs])
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1])
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
|
||||
self.assertEqual([tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].inputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].outputs[0].is_my_account, True)
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account2])
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
|
||||
self.assertEqual([tx3.id, tx2.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
@@ -340,18 +359,18 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(tx.id, tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account1])
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, True)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account2])
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, True)
|
||||
|
||||
# height 0 sorted to the top with the rest in descending order
|
||||
tx4 = await self.create_tx_from_nothing(account1, 0)
|
||||
txos = await self.ledger.db.get_txos()
|
||||
self.assertEqual([0, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertEqual([tx4.id, tx2.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
self.assertEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
@@ -379,13 +398,13 @@ class TestUpgrade(AsyncioTestCase):
|
||||
def add_address(self, address):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
conn.execute("""
|
||||
INSERT INTO pubkey_address (address, account, chain, position, pubkey)
|
||||
VALUES (?, 'account1', 0, 0, 'pubkey blob')
|
||||
INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth)
|
||||
VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0)
|
||||
""", (address,))
|
||||
|
||||
def get_addresses(self):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
sql = "SELECT address FROM pubkey_address ORDER BY address;"
|
||||
sql = "SELECT address FROM account_address ORDER BY address;"
|
||||
return [col[0] for col in conn.execute(sql).fetchall()]
|
||||
|
||||
async def test_reset_on_version_change(self):
|
||||
@@ -398,7 +417,7 @@ class TestUpgrade(AsyncioTestCase):
|
||||
self.ledger.db.SCHEMA_VERSION = None
|
||||
self.assertEqual(self.get_tables(), [])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo'])
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo'])
|
||||
self.assertEqual(self.get_addresses(), [])
|
||||
self.add_address('address1')
|
||||
await self.ledger.db.close()
|
||||
@@ -407,17 +426,17 @@ class TestUpgrade(AsyncioTestCase):
|
||||
self.ledger.db.SCHEMA_VERSION = '1.0'
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
|
||||
self.add_address('address2')
|
||||
await self.ledger.db.close()
|
||||
|
||||
# nothing changes
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), ['address2'])
|
||||
await self.ledger.db.close()
|
||||
|
||||
@@ -428,6 +447,101 @@ class TestUpgrade(AsyncioTestCase):
|
||||
"""
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.1')
|
||||
self.assertEqual(self.get_tables(), ['foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # all tables got reset
|
||||
await self.ledger.db.close()
|
||||
|
||||
|
||||
class TestSQLiteRace(AsyncioTestCase):
|
||||
max_misuse_attempts = 40000
|
||||
|
||||
def setup_db(self):
|
||||
self.db = sqlite3.connect(":memory:", isolation_level=None)
|
||||
self.db.executescript(
|
||||
"create table test1 (id text primary key not null, val text);\n" +
|
||||
"create table test2 (id text primary key not null, val text);\n" +
|
||||
"\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000))
|
||||
)
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
await self.loop.run_in_executor(self.executor, self.setup_db)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.loop.run_in_executor(self.executor, self.db.close)
|
||||
self.executor.shutdown()
|
||||
|
||||
async def test_binding_param_0_error(self):
|
||||
# test real param 0 binding errors
|
||||
|
||||
for supported_type in [str, int, bytes]:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 values (?, NULL)",
|
||||
[(supported_type(1), ), (supported_type(2), )]
|
||||
)
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.execute, "delete from test2 where id in (1, 2)"
|
||||
)
|
||||
for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]:
|
||||
try:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)",
|
||||
[(unsupported_type(1), ), (unsupported_type(2), )]
|
||||
)
|
||||
self.assertTrue(False)
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
|
||||
async def test_unhandled_sqlite_misuse(self):
|
||||
# test SQLITE_MISUSE being incorrectly raised as a param 0 binding error
|
||||
attempts = 0
|
||||
python_version = sys.version.split('\n')[0].rstrip(' ')
|
||||
|
||||
try:
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} "
|
||||
f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition")
|
||||
self.assertTrue(False, 'this test failing means either the sqlite race conditions '
|
||||
'have been fixed in cpython or the test max_attempts needs to be increased')
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE "
|
||||
f"after {attempts} attempts of the race condition")
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_fetchall_prevents_sqlite_misuse(self):
|
||||
# test that calling fetchall sufficiently avoids the race
|
||||
attempts = 0
|
||||
|
||||
def executemany_fetchall(query, params):
|
||||
self.db.executemany(query, params).fetchall()
|
||||
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
@@ -85,7 +85,7 @@ class AddressManager:
|
||||
class HierarchicalDeterministic(AddressManager):
|
||||
""" Implements simple version of Bitcoin Hierarchical Deterministic key management. """
|
||||
|
||||
name = "deterministic-chain"
|
||||
name: str = "deterministic-chain"
|
||||
|
||||
__slots__ = 'gap', 'maximum_uses_per_address'
|
||||
|
||||
@@ -115,7 +115,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
return self.account.public_key.child(self.chain_number).child(index)
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
addresses = await self._query_addresses(order_by="position ASC")
|
||||
addresses = await self._query_addresses(order_by="n asc")
|
||||
max_gap = 0
|
||||
current_gap = 0
|
||||
for address in addresses:
|
||||
@@ -128,7 +128,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
|
||||
async def ensure_address_gap(self) -> List[str]:
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="position DESC")
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="n desc")
|
||||
|
||||
existing_gap = 0
|
||||
for address in addresses:
|
||||
@@ -140,7 +140,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
if existing_gap == self.gap:
|
||||
return []
|
||||
|
||||
start = addresses[0]['position']+1 if addresses else 0
|
||||
start = addresses[0]['pubkey'].n+1 if addresses else 0
|
||||
end = start + (self.gap - existing_gap)
|
||||
new_keys = await self._generate_keys(start, end-1)
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
@@ -149,22 +149,22 @@ class HierarchicalDeterministic(AddressManager):
|
||||
async def _generate_keys(self, start: int, end: int) -> List[str]:
|
||||
if not self.address_generator_lock.locked():
|
||||
raise RuntimeError('Should not be called outside of address_generator_lock.')
|
||||
keys = [(index, self.public_key.child(index)) for index in range(start, end+1)]
|
||||
keys = [self.public_key.child(index) for index in range(start, end+1)]
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, keys)
|
||||
return [key[1].address for key in keys]
|
||||
return [key.address for key in keys]
|
||||
|
||||
def get_address_records(self, only_usable: bool = False, **constraints):
|
||||
if only_usable:
|
||||
constraints['used_times__lt'] = self.maximum_uses_per_address
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = "used_times ASC, position ASC"
|
||||
constraints['order_by'] = "used_times asc, n asc"
|
||||
return self._query_addresses(**constraints)
|
||||
|
||||
|
||||
class SingleKey(AddressManager):
|
||||
""" Single Key address manager always returns the same address for all operations. """
|
||||
|
||||
name = "single-address"
|
||||
name: str = "single-address"
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@@ -190,9 +190,7 @@ class SingleKey(AddressManager):
|
||||
async with self.address_generator_lock:
|
||||
exists = await self.get_address_records()
|
||||
if not exists:
|
||||
await self.account.ledger.db.add_keys(
|
||||
self.account, self.chain_number, [(0, self.public_key)]
|
||||
)
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key])
|
||||
new_keys = [self.public_key.address]
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
return new_keys
|
||||
@@ -417,16 +415,16 @@ class BaseAccount:
|
||||
}
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(account=self, **constraints)
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
return self.ledger.get_utxo_count(account=self, **constraints)
|
||||
return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
return self.ledger.get_transactions(account=self, **constraints)
|
||||
return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
return self.ledger.get_transaction_count(account=self, **constraints)
|
||||
return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
async def fund(self, to_account, amount=None, everything=False,
|
||||
outputs=1, broadcast=False, **constraints):
|
||||
|
||||
@@ -3,12 +3,12 @@ import asyncio
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Optional
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
from torba.client.basetransaction import BaseTransaction, TXRefImmutable
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
from torba.client.bip32 import PubKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,6 +19,7 @@ class AIOSQLite:
|
||||
# has to be single threaded as there is no mapping of thread:connection
|
||||
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||
self.connection: sqlite3.Connection = None
|
||||
self._closing = False
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
@@ -29,14 +30,12 @@ class AIOSQLite:
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
def __close(conn):
|
||||
self.executor.submit(conn.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
conn = self.connection
|
||||
if not conn:
|
||||
if self._closing:
|
||||
return
|
||||
self._closing = True
|
||||
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
self.connection = None
|
||||
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
|
||||
|
||||
def executemany(self, sql: str, params: Iterable):
|
||||
params = params if params is not None else []
|
||||
@@ -87,10 +86,10 @@ class AIOSQLite:
|
||||
if not foreign_keys_enabled:
|
||||
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
|
||||
try:
|
||||
self.connection.execute('pragma foreign_keys=off')
|
||||
self.connection.execute('pragma foreign_keys=off').fetchone()
|
||||
return self.__run_transaction(fun, *args, **kwargs)
|
||||
finally:
|
||||
self.connection.execute('pragma foreign_keys=on')
|
||||
self.connection.execute('pragma foreign_keys=on').fetchone()
|
||||
|
||||
|
||||
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
@@ -103,20 +102,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
if not key:
|
||||
sql.append(constraint)
|
||||
continue
|
||||
elif key.startswith('$'):
|
||||
if key.startswith('$'):
|
||||
values[key] = constraint
|
||||
continue
|
||||
elif key.endswith('__not'):
|
||||
if key.endswith('__not'):
|
||||
col, op = col[:-len('__not')], '!='
|
||||
elif key.endswith('__is_null'):
|
||||
col = col[:-len('__is_null')]
|
||||
sql.append(f'{col} IS NULL')
|
||||
continue
|
||||
elif key.endswith('__is_not_null'):
|
||||
if key.endswith('__is_not_null'):
|
||||
col = col[:-len('__is_not_null')]
|
||||
sql.append(f'{col} IS NOT NULL')
|
||||
continue
|
||||
elif key.endswith('__lt'):
|
||||
if key.endswith('__lt'):
|
||||
col, op = col[:-len('__lt')], '<'
|
||||
elif key.endswith('__lte'):
|
||||
col, op = col[:-len('__lte')], '<='
|
||||
@@ -150,7 +149,7 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
sql.append(f'({where})')
|
||||
values.update(subvalues)
|
||||
continue
|
||||
elif key.endswith('__and'):
|
||||
if key.endswith('__and'):
|
||||
where, subvalues = constraints_to_sql(constraint, ' AND ', key+tag+'_')
|
||||
sql.append(f'({where})')
|
||||
values.update(subvalues)
|
||||
@@ -160,15 +159,14 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
return joiner.join(sql) if sql else '', values
|
||||
|
||||
|
||||
def query(select, **constraints):
|
||||
def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
|
||||
sql = [select]
|
||||
limit = constraints.pop('limit', None)
|
||||
offset = constraints.pop('offset', None)
|
||||
order_by = constraints.pop('order_by', None)
|
||||
|
||||
constraints.pop('my_accounts', None)
|
||||
accounts = constraints.pop('accounts', None)
|
||||
if accounts is not None:
|
||||
accounts = constraints.pop('accounts', [])
|
||||
if accounts:
|
||||
constraints['account__in'] = [a.public_key.address for a in accounts]
|
||||
|
||||
where, values = constraints_to_sql(constraints)
|
||||
@@ -286,26 +284,33 @@ class SQLiteMixin:
|
||||
|
||||
class BaseDatabase(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
SCHEMA_VERSION = "1.1"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
"""
|
||||
|
||||
CREATE_ACCOUNT_TABLE = """
|
||||
create table if not exists account_address (
|
||||
account text not null,
|
||||
address text not null,
|
||||
chain integer not null,
|
||||
pubkey blob not null,
|
||||
chain_code blob not null,
|
||||
n integer not null,
|
||||
depth integer not null,
|
||||
primary key (account, address)
|
||||
);
|
||||
create index if not exists address_account_idx on account_address (address, account);
|
||||
"""
|
||||
|
||||
CREATE_PUBKEY_ADDRESS_TABLE = """
|
||||
create table if not exists pubkey_address (
|
||||
address text primary key,
|
||||
account text not null,
|
||||
chain integer not null,
|
||||
position integer not null,
|
||||
pubkey blob not null,
|
||||
history text,
|
||||
used_times integer not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_PUBKEY_ADDRESS_INDEX = """
|
||||
create index if not exists pubkey_address_account_idx on pubkey_address (account);
|
||||
"""
|
||||
|
||||
CREATE_TX_TABLE = """
|
||||
create table if not exists tx (
|
||||
@@ -327,8 +332,6 @@ class BaseDatabase(SQLiteMixin):
|
||||
script blob not null,
|
||||
is_reserved boolean not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_TXO_INDEX = """
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
"""
|
||||
|
||||
@@ -338,21 +341,17 @@ class BaseDatabase(SQLiteMixin):
|
||||
txoid text references txo,
|
||||
address text references pubkey_address
|
||||
);
|
||||
"""
|
||||
CREATE_TXI_INDEX = """
|
||||
create index if not exists txi_address_idx on txi (address);
|
||||
create index if not exists txi_txoid_idx on txi (txoid);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_TX_TABLE +
|
||||
CREATE_ACCOUNT_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TX_TABLE +
|
||||
CREATE_TXO_TABLE +
|
||||
CREATE_TXO_INDEX +
|
||||
CREATE_TXI_TABLE +
|
||||
CREATE_TXI_INDEX
|
||||
CREATE_TXI_TABLE
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -377,10 +376,10 @@ class BaseDatabase(SQLiteMixin):
|
||||
}
|
||||
|
||||
async def insert_transaction(self, tx):
|
||||
await self.db.execute(*self._insert_sql('tx', self.tx_to_row(tx)))
|
||||
await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx)))
|
||||
|
||||
async def update_transaction(self, tx):
|
||||
await self.db.execute(*self._update_sql("tx", {
|
||||
await self.db.execute_fetchall(*self._update_sql("tx", {
|
||||
'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
|
||||
}, 'txid = ?', (tx.id,)))
|
||||
|
||||
@@ -391,7 +390,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
|
||||
))
|
||||
)).fetchall()
|
||||
elif txo.script.is_pay_script_hash:
|
||||
# TODO: implement script hash payments
|
||||
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
|
||||
@@ -404,7 +403,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
'txid': tx.id,
|
||||
'txoid': txo.id,
|
||||
'address': address,
|
||||
}, ignore_duplicate=True))
|
||||
}, ignore_duplicate=True)).fetchall()
|
||||
|
||||
conn.execute(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
@@ -435,25 +434,22 @@ class BaseDatabase(SQLiteMixin):
|
||||
|
||||
async def select_transactions(self, cols, accounts=None, **constraints):
|
||||
if not set(constraints) & {'txid', 'txid__in'}:
|
||||
assert accounts is not None, "'accounts' argument required when no 'txid' constraint"
|
||||
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
|
||||
constraints.update({
|
||||
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
|
||||
})
|
||||
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
|
||||
where = f" WHERE account_address.account IN ({account_values})"
|
||||
constraints['txid__in'] = f"""
|
||||
SELECT txo.txid FROM txo
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
|
||||
UNION
|
||||
SELECT txi.txid FROM txi
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
|
||||
"""
|
||||
return await self.db.execute_fetchall(
|
||||
*query("SELECT {} FROM tx".format(cols), **constraints)
|
||||
)
|
||||
|
||||
async def get_transactions(self, **constraints):
|
||||
accounts = constraints.get('accounts', None)
|
||||
|
||||
async def get_transactions(self, wallet=None, **constraints):
|
||||
tx_rows = await self.select_transactions(
|
||||
'txid, raw, height, position, is_verified',
|
||||
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
|
||||
@@ -478,7 +474,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
annotated_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
my_accounts=accounts,
|
||||
wallet=wallet,
|
||||
txid__in=txids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -488,7 +484,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
referenced_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
my_accounts=accounts,
|
||||
wallet=wallet,
|
||||
txoid__in=txi_txoids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -508,6 +504,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs
|
||||
|
||||
async def get_transaction_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -520,23 +517,24 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs[0]
|
||||
|
||||
async def select_txos(self, cols, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM txo"
|
||||
" JOIN pubkey_address USING (address)"
|
||||
" JOIN tx USING (txid)".format(cols), **constraints
|
||||
))
|
||||
sql = "SELECT {} FROM txo JOIN tx USING (txid)"
|
||||
if 'accounts' in constraints:
|
||||
sql += " JOIN account_address USING (address)"
|
||||
return await self.db.execute_fetchall(*query(sql.format(cols), **constraints))
|
||||
|
||||
async def get_txos(self, my_accounts=None, no_tx=False, **constraints):
|
||||
my_accounts = [
|
||||
(a.public_key.address if isinstance(a, BaseAccount) else a)
|
||||
for a in (my_accounts or constraints.get('accounts', []))
|
||||
]
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints):
|
||||
my_accounts = set(a.public_key.address for a in wallet.accounts) if wallet else set()
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"]
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
rows = await self.select_txos(
|
||||
"tx.txid, raw, tx.height, tx.position, tx.is_verified, "
|
||||
"txo.position, chain, account, amount, script",
|
||||
"""
|
||||
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
|
||||
select group_concat(account||"|"||chain) from account_address
|
||||
where account_address.address=txo.address
|
||||
)
|
||||
""",
|
||||
**constraints
|
||||
)
|
||||
txos = []
|
||||
@@ -545,8 +543,8 @@ class BaseDatabase(SQLiteMixin):
|
||||
for row in rows:
|
||||
if no_tx:
|
||||
txo = output_class(
|
||||
amount=row[8],
|
||||
script=output_class.script_class(row[9]),
|
||||
amount=row[6],
|
||||
script=output_class.script_class(row[7]),
|
||||
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
|
||||
position=row[5]
|
||||
)
|
||||
@@ -556,12 +554,18 @@ class BaseDatabase(SQLiteMixin):
|
||||
row[1], height=row[2], position=row[3], is_verified=row[4]
|
||||
)
|
||||
txo = txs[row[0]].outputs[row[5]]
|
||||
txo.is_change = row[6] == 1
|
||||
txo.is_my_account = row[7] in my_accounts
|
||||
row_accounts = dict(a.split('|') for a in row[8].split(','))
|
||||
account_match = set(row_accounts) & my_accounts
|
||||
if account_match:
|
||||
txo.is_my_account = True
|
||||
txo.is_change = row_accounts[account_match.pop()] == '1'
|
||||
else:
|
||||
txo.is_change = txo.is_my_account = False
|
||||
txos.append(txo)
|
||||
return txos
|
||||
|
||||
async def get_txo_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -581,45 +585,60 @@ class BaseDatabase(SQLiteMixin):
|
||||
self.constrain_utxo(constraints)
|
||||
return self.get_txo_count(**constraints)
|
||||
|
||||
async def get_balance(self, **constraints):
|
||||
async def get_balance(self, wallet=None, accounts=None, **constraints):
|
||||
assert wallet or accounts, \
|
||||
"'wallet' or 'accounts' constraints required to calculate balance"
|
||||
constraints['accounts'] = accounts or wallet.accounts
|
||||
self.constrain_utxo(constraints)
|
||||
balance = await self.select_txos('SUM(amount)', **constraints)
|
||||
return balance[0][0] or 0
|
||||
|
||||
async def select_addresses(self, cols, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM pubkey_address".format(cols), **constraints
|
||||
"SELECT {} FROM pubkey_address JOIN account_address USING (address)".format(cols),
|
||||
**constraints
|
||||
))
|
||||
|
||||
async def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'),
|
||||
**constraints):
|
||||
addresses = await self.select_addresses(', '.join(cols), **constraints)
|
||||
return rows_to_dict(addresses, cols)
|
||||
async def get_addresses(self, cols=None, **constraints):
|
||||
cols = cols or (
|
||||
'address', 'account', 'chain', 'history', 'used_times',
|
||||
'pubkey', 'chain_code', 'n', 'depth'
|
||||
)
|
||||
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
|
||||
for address in addresses:
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
return addresses
|
||||
|
||||
async def get_address_count(self, **constraints):
|
||||
count = await self.select_addresses('count(*)', **constraints)
|
||||
return count[0][0]
|
||||
|
||||
async def get_address(self, **constraints):
|
||||
addresses = await self.get_addresses(
|
||||
cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
|
||||
limit=1, **constraints
|
||||
)
|
||||
addresses = await self.get_addresses(limit=1, **constraints)
|
||||
if addresses:
|
||||
return addresses[0]
|
||||
|
||||
async def add_keys(self, account, chain, keys):
|
||||
async def add_keys(self, account, chain, pubkeys):
|
||||
await self.db.executemany(
|
||||
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
|
||||
(
|
||||
(pubkey.address, account.public_key.address, chain, position,
|
||||
sqlite3.Binary(pubkey.pubkey_bytes))
|
||||
for position, pubkey in keys
|
||||
)
|
||||
"insert or ignore into account_address "
|
||||
"(account, address, chain, pubkey, chain_code, n, depth) values "
|
||||
"(?, ?, ?, ?, ?, ?, ?)", ((
|
||||
account.id, k.address, chain,
|
||||
sqlite3.Binary(k.pubkey_bytes),
|
||||
sqlite3.Binary(k.chain_code),
|
||||
k.n, k.depth
|
||||
) for k in pubkeys)
|
||||
)
|
||||
await self.db.executemany(
|
||||
"insert or ignore into pubkey_address (address) values (?)",
|
||||
((pubkey.address,) for pubkey in pubkeys)
|
||||
)
|
||||
|
||||
async def _set_address_history(self, address, history):
|
||||
await self.db.execute(
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
(history, history.count(':')//2, address)
|
||||
)
|
||||
|
||||
@@ -19,6 +19,7 @@ from torba.client.coinselection import CoinSelector
|
||||
from torba.client.constants import COIN, NULL_HASH32
|
||||
from torba.stream import StreamController
|
||||
from torba.client.hash import hash160, double_sha256, sha256, Base58
|
||||
from torba.client.bip32 import PubKey, PrivateKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -160,7 +161,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
decoded = Base58.decode(address)
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
@@ -178,27 +179,29 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def add_account(self, account: baseaccount.BaseAccount):
|
||||
self.accounts.append(account)
|
||||
|
||||
async def _get_account_and_address_info_for_address(self, address):
|
||||
match = await self.db.get_address(address=address)
|
||||
async def _get_account_and_address_info_for_address(self, wallet, address):
|
||||
match = await self.db.get_address(accounts=wallet.accounts, address=address)
|
||||
if match:
|
||||
for account in self.accounts:
|
||||
for account in wallet.accounts:
|
||||
if match['account'] == account.public_key.address:
|
||||
return account, match
|
||||
|
||||
async def get_private_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
account, address_info = match
|
||||
return account.get_private_key(address_info['chain'], address_info['position'])
|
||||
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
|
||||
return None
|
||||
|
||||
async def get_public_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
account, address_info = match
|
||||
return account.get_public_key(address_info['chain'], address_info['position'])
|
||||
_, address_info = match
|
||||
return address_info['pubkey']
|
||||
return None
|
||||
|
||||
async def get_account_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_account_for_address(self, wallet, address):
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
return match[0]
|
||||
|
||||
@@ -210,6 +213,12 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
estimators.append(utxo.get_estimator(self))
|
||||
return estimators
|
||||
|
||||
async def get_addresses(self, **constraints):
|
||||
return await self.db.get_addresses(**constraints)
|
||||
|
||||
def get_address_count(self, **constraints):
|
||||
return self.db.get_address_count(**constraints)
|
||||
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts):
|
||||
async with self._utxo_reservation_lock:
|
||||
txos = await self.get_effective_amount_estimators(funding_accounts)
|
||||
@@ -229,29 +238,16 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def release_tx(self, tx):
|
||||
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
|
||||
|
||||
def constraint_account_or_all(self, constraints):
|
||||
if 'accounts' in constraints:
|
||||
return
|
||||
account = constraints.pop('account', None)
|
||||
if account:
|
||||
constraints['accounts'] = [account]
|
||||
else:
|
||||
constraints['accounts'] = self.accounts
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxos(**constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxo_count(**constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transactions(**constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
async def get_local_status_and_history(self, address, history=None):
|
||||
@@ -562,7 +558,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
addresses.add(
|
||||
self.hash160_to_address(txo.script.values['pubkey_hash'])
|
||||
)
|
||||
records = await self.db.get_addresses(cols=('address',), address__in=addresses)
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
_, pending = await asyncio.wait([
|
||||
self.on_transaction.where(partial(
|
||||
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Type, MutableSequence, MutableMapping
|
||||
from typing import Type, MutableSequence, MutableMapping, Optional
|
||||
|
||||
from torba.client.baseledger import BaseLedger, LedgerRegistry
|
||||
from torba.client.wallet import Wallet, WalletStorage
|
||||
@@ -41,16 +41,6 @@ class BaseWalletManager:
|
||||
self.wallets.append(wallet)
|
||||
return wallet
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@property
|
||||
def default_wallet(self):
|
||||
for wallet in self.wallets:
|
||||
@@ -78,3 +68,14 @@ class BaseWalletManager:
|
||||
l.stop() for l in self.ledgers.values()
|
||||
))
|
||||
self.running = False
|
||||
|
||||
def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet:
|
||||
if wallet_id is None:
|
||||
return self.default_wallet
|
||||
return self.get_wallet_or_error(wallet_id)
|
||||
|
||||
def get_wallet_or_error(self, wallet_id: str) -> Wallet:
|
||||
for wallet in self.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
return wallet
|
||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
||||
@@ -185,6 +185,9 @@ class BaseNetwork:
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
|
||||
@@ -258,6 +261,9 @@ class BaseNetwork:
|
||||
self.client.abort()
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
def get_server_features(self):
|
||||
return self.rpc('server.features', (), restricted=True)
|
||||
|
||||
|
||||
class SessionPool:
|
||||
|
||||
@@ -281,7 +287,7 @@ class SessionPool:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions],
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
import typing
|
||||
from typing import List, Iterable, Optional
|
||||
from typing import List, Iterable, Optional, Tuple
|
||||
from binascii import hexlify
|
||||
|
||||
from torba.client.basescript import BaseInputScript, BaseOutputScript
|
||||
@@ -12,7 +12,7 @@ from torba.client.util import ReadOnlyList
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from torba.client import baseledger
|
||||
from torba.client import baseledger, wallet as basewallet
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
@@ -431,21 +431,32 @@ class BaseTransaction:
|
||||
self.locktime = stream.read_uint32()
|
||||
|
||||
@classmethod
|
||||
def ensure_all_have_same_ledger(cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> 'baseledger.BaseLedger':
|
||||
ledger = None
|
||||
def ensure_all_have_same_ledger_and_wallet(
|
||||
cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> Tuple['baseledger.BaseLedger', 'basewallet.Wallet']:
|
||||
ledger = wallet = None
|
||||
for account in funding_accounts:
|
||||
if ledger is None:
|
||||
ledger = account.ledger
|
||||
wallet = account.wallet
|
||||
if ledger != account.ledger:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be on the same ledger.'
|
||||
)
|
||||
if change_account is not None and change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if wallet != account.wallet:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be from the same wallet.'
|
||||
)
|
||||
if change_account is not None:
|
||||
if change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if change_account.wallet != wallet:
|
||||
raise ValueError('Change account must use same wallet as funding accounts.')
|
||||
if ledger is None:
|
||||
raise ValueError('No ledger found.')
|
||||
return ledger
|
||||
if wallet is None:
|
||||
raise ValueError('No wallet found.')
|
||||
return ledger, wallet
|
||||
|
||||
@classmethod
|
||||
async def create(cls, inputs: Iterable[BaseInput], outputs: Iterable[BaseOutput],
|
||||
@@ -458,7 +469,7 @@ class BaseTransaction:
|
||||
.add_inputs(inputs) \
|
||||
.add_outputs(outputs)
|
||||
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
|
||||
# value of the outputs plus associated fees
|
||||
cost = (
|
||||
@@ -496,18 +507,17 @@ class BaseTransaction:
|
||||
|
||||
if tx._outputs:
|
||||
break
|
||||
else:
|
||||
# this condition and the outer range(5) loop cover an edge case
|
||||
# whereby a single input is just enough to cover the fee and
|
||||
# has some change left over, but the change left over is less
|
||||
# than the cost_of_change: thus the input is completely
|
||||
# consumed and no output is added, which is an invalid tx.
|
||||
# to be able to spend this input we must increase the cost
|
||||
# of the TX and run through the balance algorithm a second time
|
||||
# adding an extra input and change output, making tx valid.
|
||||
# we do this 5 times in case the other UTXOs added are also
|
||||
# less than the fee, after 5 attempts we give up and go home
|
||||
cost += cost_of_change + 1
|
||||
# this condition and the outer range(5) loop cover an edge case
|
||||
# whereby a single input is just enough to cover the fee and
|
||||
# has some change left over, but the change left over is less
|
||||
# than the cost_of_change: thus the input is completely
|
||||
# consumed and no output is added, which is an invalid tx.
|
||||
# to be able to spend this input we must increase the cost
|
||||
# of the TX and run through the balance algorithm a second time
|
||||
# adding an extra input and change output, making tx valid.
|
||||
# we do this 5 times in case the other UTXOs added are also
|
||||
# less than the fee, after 5 attempts we give up and go home
|
||||
cost += cost_of_change + 1
|
||||
|
||||
if sign:
|
||||
await tx.sign(funding_accounts)
|
||||
@@ -524,14 +534,15 @@ class BaseTransaction:
|
||||
return hash_type
|
||||
|
||||
async def sign(self, funding_accounts: Iterable[BaseAccount]):
|
||||
ledger = self.ensure_all_have_same_ledger(funding_accounts)
|
||||
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
|
||||
for i, txi in enumerate(self._inputs):
|
||||
assert txi.script is not None
|
||||
assert txi.txo_ref.txo is not None
|
||||
txo_script = txi.txo_ref.txo.script
|
||||
if txo_script.is_pay_pubkey_hash:
|
||||
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
|
||||
private_key = await ledger.get_private_key_for_address(address)
|
||||
private_key = await ledger.get_private_key_for_address(wallet, address)
|
||||
assert private_key is not None, 'Cannot find private key for signing output.'
|
||||
tx = self._serialize_for_signature(i)
|
||||
txi.script.values['signature'] = \
|
||||
private_key.sign(tx) + bytes((self.signature_hash_type(1),))
|
||||
|
||||
@@ -3,7 +3,7 @@ import stat
|
||||
import json
|
||||
import zlib
|
||||
import typing
|
||||
from typing import Sequence, MutableSequence
|
||||
from typing import List, Sequence, MutableSequence, Optional
|
||||
from hashlib import sha256
|
||||
from operator import attrgetter
|
||||
from torba.client.hash import better_aes_encrypt, better_aes_decrypt
|
||||
@@ -25,12 +25,51 @@ class Wallet:
|
||||
self.accounts = accounts or []
|
||||
self.storage = storage or WalletStorage()
|
||||
|
||||
def add_account(self, account):
|
||||
@property
|
||||
def id(self):
|
||||
if self.storage.path:
|
||||
return os.path.basename(self.storage.path)
|
||||
return self.name
|
||||
|
||||
def add_account(self, account: 'baseaccount.BaseAccount'):
|
||||
self.accounts.append(account)
|
||||
|
||||
def generate_account(self, ledger: 'baseledger.BaseLedger') -> 'baseaccount.BaseAccount':
|
||||
return ledger.account_class.generate(ledger, self)
|
||||
|
||||
@property
|
||||
def default_account(self) -> Optional['baseaccount.BaseAccount']:
|
||||
for account in self.accounts:
|
||||
return account
|
||||
return None
|
||||
|
||||
def get_account_or_default(self, account_id: str) -> Optional['baseaccount.BaseAccount']:
|
||||
if account_id is None:
|
||||
return self.default_account
|
||||
return self.get_account_or_error(account_id)
|
||||
|
||||
def get_account_or_error(self, account_id: str) -> 'baseaccount.BaseAccount':
|
||||
for account in self.accounts:
|
||||
if account.id == account_id:
|
||||
return account
|
||||
raise ValueError(f"Couldn't find account: {account_id}.")
|
||||
|
||||
def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['baseaccount.BaseAccount']:
|
||||
return [
|
||||
self.get_account_or_error(account_id)
|
||||
for account_id in account_ids
|
||||
] if account_ids else self.accounts
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@classmethod
|
||||
def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':
|
||||
json_dict = storage.read()
|
||||
@@ -54,11 +93,6 @@ class Wallet:
|
||||
def save(self):
|
||||
self.storage.write(self.to_dict())
|
||||
|
||||
@property
|
||||
def default_account(self):
|
||||
for account in self.accounts:
|
||||
return account
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256()
|
||||
|
||||
@@ -14,11 +14,11 @@ from torba.client.baseheader import BaseHeaders, ArithUint256
|
||||
|
||||
|
||||
class MainHeaders(BaseHeaders):
|
||||
header_size = 80
|
||||
chunk_size = 2016
|
||||
max_target = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
header_size: int = 80
|
||||
chunk_size: int = 2016
|
||||
max_target: int = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash: Optional[bytes] = b'000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
|
||||
target_timespan = 14 * 24 * 60 * 60
|
||||
target_timespan: int = 14 * 24 * 60 * 60
|
||||
|
||||
@staticmethod
|
||||
def serialize(header: dict) -> bytes:
|
||||
|
||||
@@ -151,7 +151,9 @@ class WalletNode:
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
wallet_file_name = os.path.join(self.data_path, 'my_wallet.json')
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
with open(wallet_file_name, 'w') as wallet_file:
|
||||
wallet_file.write('{"version": 1, "accounts": []}\n')
|
||||
self.manager = self.manager_class.from_config({
|
||||
|
||||
@@ -80,6 +80,8 @@ class Env:
|
||||
self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
|
||||
@@ -767,6 +767,9 @@ class ElectrumX(SessionBase):
|
||||
'protocol_min': min_str,
|
||||
'protocol_max': max_str,
|
||||
'genesis_hash': env.coin.GENESIS_HASH,
|
||||
'description': env.description,
|
||||
'payment_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -13,6 +13,6 @@ changedir = {toxinidir}/tests
|
||||
setenv =
|
||||
integration: TORBA_LEDGER={envname}
|
||||
commands =
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.unit
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.unit
|
||||
integration: orchstr8 download
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.integration
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.integration
|
||||
Reference in new issue
Block a user