Compare commits
92
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4cbc82619d | ||
|
|
1b0d802941 | ||
|
|
dc72a1e58d | ||
|
|
1d5333c737 | ||
|
|
d289041ff3 | ||
|
|
f287483397 | ||
|
|
1ac3b6c173 | ||
|
|
1d090765ba | ||
|
|
6c6050528a | ||
|
|
835d376ca6 | ||
|
|
68e9fd8541 | ||
|
|
e49f8f8c61 | ||
|
|
b396322f1e | ||
|
|
ea434e3469 | ||
|
|
84587ac232 | ||
|
|
9fbc83fe9d | ||
|
|
965972a719 | ||
|
|
34d19c4a72 | ||
|
|
9b0910767d | ||
|
|
32cc077e1f | ||
|
|
1e7bec48b6 | ||
|
|
c035837a14 | ||
|
|
cb5eb8983b | ||
|
|
9c86cb7954 | ||
|
|
ef94c1eccc | ||
|
|
64eaa5e3aa | ||
|
|
826f584aa8 | ||
|
|
c5530aee5f | ||
|
|
817f01593e | ||
|
|
79727f0e97 | ||
|
|
abac82c1c6 | ||
|
|
02c51c6059 | ||
|
|
46b45d0387 | ||
|
|
e6c549c457 | ||
|
|
1aa2983216 | ||
|
|
a095f89c81 | ||
|
|
287902f01b | ||
|
|
7dc3e4e7a1 | ||
|
|
0b31aad901 | ||
|
|
1791d8e4ee | ||
|
|
b7c2054a66 | ||
|
|
81f16d7061 | ||
|
|
5c151a3a09 | ||
|
|
3c46c6cbb9 | ||
|
|
d31f09c174 | ||
|
|
063eeb34ec | ||
|
|
f06192601c | ||
|
|
091221ba2c | ||
|
|
7c22111ae0 | ||
|
|
daac3eed09 | ||
|
|
32ee7feea8 | ||
|
|
2026848a77 | ||
|
|
a7b205603c | ||
|
|
291d14f2d4 | ||
|
|
4a68137cfa | ||
|
|
b2f0374426 | ||
|
|
5fcca6ed0c | ||
|
|
a05c44ad1e | ||
|
|
f9880c8931 | ||
|
|
b876896628 | ||
|
|
375c2a3363 | ||
|
|
5dbc749000 | ||
|
|
36fbd91280 | ||
|
|
d2ad3e7ff1 | ||
|
|
072f1f112e | ||
|
|
6c79f98d3c | ||
|
|
8edebbb684 | ||
|
|
4839ca631a | ||
|
|
5018fe90de | ||
|
|
379144bcfe | ||
|
|
d2cd0ece5f | ||
|
|
10b7ccaa92 | ||
|
|
925eb618de | ||
|
|
96800de052 | ||
|
|
9df659b647 | ||
|
|
aec0bd5d11 | ||
|
|
c1c9c2c578 | ||
|
|
c739fa3ca1 | ||
|
|
355dfcb6c1 | ||
|
|
8f7c291e33 | ||
|
|
71fbb3df24 | ||
|
|
9cfa9b2c57 | ||
|
|
b9b8178e30 | ||
|
|
089cefb77b | ||
|
|
ee9ea1faab | ||
|
|
22b2cf8b0c | ||
|
|
10bd3c9dda | ||
|
|
6c3147fcf4 | ||
|
|
5e8d169c50 | ||
|
|
8657a35fd7 | ||
|
|
97c0b60a22 | ||
|
|
8a284d6c49 |
No files matched your search
+552
-111
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.39.2"
|
||||
__version__ = "0.42.0"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+15
-6
@@ -148,7 +148,12 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
@staticmethod
|
||||
def _parse_list(l):
|
||||
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
|
||||
if l == ['null']:
|
||||
return None
|
||||
assert len(l) == 2, (
|
||||
'Max key fee is made up of either two values: '
|
||||
'"AMOUNT CURRENCY", or "null" (to set no limit)'
|
||||
)
|
||||
try:
|
||||
amount = float(l[0])
|
||||
except ValueError:
|
||||
@@ -176,7 +181,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
nargs=2,
|
||||
nargs='+',
|
||||
metavar=('AMOUNT', 'CURRENCY'),
|
||||
default=NOT_SET
|
||||
)
|
||||
@@ -377,12 +382,11 @@ class BaseConfig:
|
||||
|
||||
@contextmanager
|
||||
def update_config(self):
|
||||
if not isinstance(self.persisted, ConfigFileAccess):
|
||||
raise TypeError("Config file cannot be updated.")
|
||||
self._updating_config = True
|
||||
yield self
|
||||
self._updating_config = False
|
||||
self.persisted.save()
|
||||
if isinstance(self.persisted, ConfigFileAccess):
|
||||
self.persisted.save()
|
||||
|
||||
@property
|
||||
def modify_order(self):
|
||||
@@ -481,6 +485,10 @@ class Config(CLIConfig):
|
||||
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
wallets = Strings(
|
||||
"Wallet files in 'wallet_dir' to load at startup.",
|
||||
['default_wallet']
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
@@ -528,7 +536,8 @@ class Config(CLIConfig):
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
|
||||
@@ -45,7 +45,7 @@ async def execute_command(conf, method, params, callback=display):
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
|
||||
+562
-271
File diff suppressed because it is too large.
Load diff
@@ -57,8 +57,7 @@ async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) ->
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body, headers=headers) as response:
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
|
||||
@@ -6,6 +6,8 @@ from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.bip32 import PubKey
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.ledger import MainNetLedger, Account
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
@@ -67,6 +69,13 @@ def encode_account_doc():
|
||||
}
|
||||
|
||||
|
||||
def encode_wallet_doc():
|
||||
return {
|
||||
'id': 'wallet_id',
|
||||
'name': 'optional wallet name',
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
@@ -110,6 +119,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
@@ -118,6 +129,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -172,7 +185,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'normalized_name': txo.normalized_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
'meta': self.encode_claim_meta(txo.meta)
|
||||
'meta': self.encode_claim_meta(txo.meta.copy())
|
||||
})
|
||||
if 'short_url' in output['meta']:
|
||||
output['short_url'] = output['meta'].pop('short_url')
|
||||
@@ -184,6 +197,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
@@ -217,16 +232,51 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_wallet(self, wallet):
|
||||
return {
|
||||
'id': wallet.id,
|
||||
'name': wallet.name
|
||||
}
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
file = managed_stream.as_dict()
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
file.update({
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
'nout': managed_stream.nout,
|
||||
'outpoint': managed_stream.outpoint,
|
||||
'metadata': managed_stream.metadata,
|
||||
'protobuf': managed_stream.metadata_protobuf,
|
||||
'channel_claim_id': managed_stream.channel_claim_id,
|
||||
'channel_name': managed_stream.channel_name,
|
||||
'claim_name': managed_stream.claim_name,
|
||||
'content_fee': managed_stream.content_fee,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
})
|
||||
return file
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
|
||||
@@ -90,20 +90,6 @@ def get_claims_from_stream_hashes(transaction: sqlite3.Connection,
|
||||
}
|
||||
|
||||
|
||||
def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
|
||||
outpoint: str) -> typing.Optional[StoredStreamClaim]:
|
||||
query = (
|
||||
"select content_claim.stream_hash, c.*, case when c.channel_claim_id is not null then "
|
||||
" (select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
" else null end as channel_name "
|
||||
" from content_claim "
|
||||
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.claim_outpoint=?"
|
||||
)
|
||||
claim_fields = transaction.execute(query, (outpoint, )).fetchone()
|
||||
if claim_fields:
|
||||
return StoredStreamClaim(*claim_fields)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
for start_index in range(0, len(parameters), batch_size):
|
||||
current_batch = parameters[start_index:start_index+batch_size]
|
||||
@@ -115,6 +101,9 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
stream_hashes = tuple(
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
|
||||
)
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
@@ -122,9 +111,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
"order by c.rowid desc", stream_hashes):
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -151,7 +138,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
list(signed_claims.keys())):
|
||||
tuple(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
@@ -161,35 +148,35 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs)
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
@@ -205,7 +192,7 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
@@ -307,17 +294,17 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
)
|
||||
).fetchall()
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
@@ -325,40 +312,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def should_announce(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select should_announce from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select count(*) from blob where should_announce=1 and status='finished'"
|
||||
)
|
||||
|
||||
def get_all_should_announce_blobs(self):
|
||||
return self.run_and_return_list(
|
||||
"select blob_hash from blob where should_announce=1 and status='finished'"
|
||||
)
|
||||
|
||||
def get_all_finished_blobs(self):
|
||||
return self.run_and_return_list(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
)
|
||||
|
||||
def count_finished_blobs(self):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select count(*) from blob where status='finished'"
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
[(int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes]
|
||||
)
|
||||
((int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run(_update_last_announced_blobs)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
@@ -369,11 +331,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1, next_announce_time=? "
|
||||
"where blob_hash=? and status='finished'", (int(now), blob_hash)
|
||||
)
|
||||
).fetchall()
|
||||
else:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
|
||||
)
|
||||
).fetchall()
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
@@ -386,22 +348,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
).fetchall()
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
return [b[0] for b in r.fetchall()]
|
||||
).fetchall()
|
||||
return [b[0] for b in r]
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
@@ -409,43 +371,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def sync_files_to_blobs(self):
|
||||
def _sync_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
transaction.execute(
|
||||
"select distinct sb.stream_hash from stream_blob sb "
|
||||
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
|
||||
finished_blob_hashes = tuple(
|
||||
blob_hash for (blob_hash, ) in transaction.execute(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
|
||||
def _set_streaming(transaction: sqlite3.Connection):
|
||||
finished_blobs_set = set(finished_blob_hashes)
|
||||
to_update_set = finished_blobs_set.difference(blob_files)
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
|
||||
return self.db.run(_set_streaming)
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
((blob_hash, ) for blob_hash in to_update_set)
|
||||
).fetchall()
|
||||
return blob_files.intersection(finished_blobs_set)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
@@ -459,11 +397,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
|
||||
return self.run_and_return_one_or_none(
|
||||
"select rowid from file where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
return self.db.run(store_stream, sd_blob, descriptor)
|
||||
|
||||
@@ -509,12 +442,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
|
||||
)
|
||||
|
||||
def get_stream_info_for_sd_hash(self, sd_blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
|
||||
sd_blob_hash
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
@@ -554,7 +481,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
@@ -562,7 +489,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
@@ -571,22 +498,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
@@ -607,13 +534,13 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
((stream_hash, ) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
@@ -625,14 +552,16 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
transaction.execute(
|
||||
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
|
||||
).fetchall()
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
).fetchall()
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -651,7 +580,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for support_info in _batched_select(
|
||||
transaction,
|
||||
"select * from support where claim_id in {}",
|
||||
tuple(claim_ids)
|
||||
claim_ids
|
||||
)
|
||||
]
|
||||
|
||||
@@ -682,7 +611,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
)
|
||||
).fetchall()
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
@@ -769,7 +698,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
# update the claim associated to the file
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
|
||||
).fetchall()
|
||||
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
@@ -788,64 +719,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_claims_from_stream_hashes(self, stream_hashes: typing.List[str],
|
||||
include_supports: typing.Optional[bool] = True):
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, stream_hashes)
|
||||
return {stream_hash: claim_info.as_dict() for stream_hash, claim_info in claims.items()}
|
||||
|
||||
async def get_claim(self, claim_outpoint, include_supports=True):
|
||||
claim_info = await self.db.run(get_content_claim_from_outpoint, claim_outpoint)
|
||||
if not claim_info:
|
||||
return
|
||||
result = claim_info.as_dict()
|
||||
if include_supports:
|
||||
supports = await self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
return result
|
||||
|
||||
def get_unknown_certificate_ids(self):
|
||||
def _get_unknown_certificate_claim_ids(transaction):
|
||||
return [
|
||||
claim_id for (claim_id,) in transaction.execute(
|
||||
"select distinct c1.channel_claim_id from claim as c1 "
|
||||
"where c1.channel_claim_id!='' "
|
||||
"and c1.channel_claim_id not in "
|
||||
"(select c2.claim_id from claim as c2)"
|
||||
).fetchall()
|
||||
]
|
||||
return self.db.run(_get_unknown_certificate_claim_ids)
|
||||
|
||||
async def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = await self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
results = {} # {txid: [nout, ...]}
|
||||
for outpoint_str in claim_outpoints:
|
||||
txid, nout = outpoint_str.split(":")
|
||||
outputs = results.get(txid, [])
|
||||
outputs.append(int(nout))
|
||||
results[txid] = outputs
|
||||
if results:
|
||||
log.debug("missing transaction heights for %i claims", len(results))
|
||||
return results
|
||||
|
||||
def save_claim_tx_heights(self, claim_tx_heights):
|
||||
def _save_claim_heights(transaction):
|
||||
for outpoint, height in claim_tx_heights.items():
|
||||
transaction.execute(
|
||||
"update claim set height=? where claim_outpoint=? and height=-1",
|
||||
(height, outpoint)
|
||||
)
|
||||
return self.db.run(_save_claim_heights)
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
return self.db.execute_fetchall(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
)
|
||||
return self.db.execute(
|
||||
return self.db.execute_fetchall(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
|
||||
@@ -139,6 +139,7 @@ types_map = {
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.flac': ('audio/flac', 'audio'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
|
||||
@@ -4,7 +4,10 @@ from typing import NamedTuple, Tuple
|
||||
|
||||
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.com/
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
@@ -17,7 +20,7 @@ def _create_url_regex():
|
||||
|
||||
def _claim(name, prefix=""):
|
||||
return _group(
|
||||
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
|
||||
@@ -123,6 +123,14 @@ class ManagedStream:
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
return self.written_bytes >= self.descriptor.lower_bound_decrypted_length()
|
||||
|
||||
@property
|
||||
def stream_url(self):
|
||||
return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}"
|
||||
|
||||
async def update_status(self, status: str):
|
||||
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
|
||||
self._status = status
|
||||
@@ -203,47 +211,6 @@ class ManagedStream:
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path
|
||||
file_name = self.file_name
|
||||
download_directory = self.download_directory
|
||||
if not self.output_file_exists:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
return {
|
||||
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
|
||||
'completed': self.written_bytes >= self.descriptor.lower_bound_decrypted_length(),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not self.running,
|
||||
'stream_hash': self.stream_hash,
|
||||
'stream_name': self.descriptor.stream_name,
|
||||
'suggested_file_name': self.descriptor.suggested_file_name,
|
||||
'sd_hash': self.descriptor.sd_hash,
|
||||
'download_path': full_path,
|
||||
'mime_type': self.mime_type,
|
||||
'key': self.descriptor.key,
|
||||
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': self.written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
'nout': self.nout,
|
||||
'outpoint': self.outpoint,
|
||||
'metadata': self.metadata,
|
||||
'protobuf': self.metadata_protobuf,
|
||||
'channel_claim_id': self.channel_claim_id,
|
||||
'channel_name': self.channel_name,
|
||||
'claim_name': self.claim_name,
|
||||
'content_fee': self.content_fee
|
||||
}
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
|
||||
@@ -401,8 +401,8 @@ class StreamManager:
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
), 5) if self.config.max_key_fee else None
|
||||
if max_fee_amount and fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
|
||||
+22
-15
@@ -6,6 +6,7 @@ from string import hexdigits
|
||||
|
||||
import ecdsa
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.constants import TXO_TYPES
|
||||
|
||||
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
|
||||
|
||||
@@ -27,6 +28,7 @@ class Account(BaseAccount):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.channel_keys = {}
|
||||
self.preferences = {}
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
@@ -38,6 +40,7 @@ class Account(BaseAccount):
|
||||
def apply(self, d: dict):
|
||||
super().apply(d)
|
||||
self.channel_keys.update(d.get('certificates', {}))
|
||||
self.preferences.update(d.get('preferences', {}))
|
||||
|
||||
def add_channel_private_key(self, private_key):
|
||||
public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
@@ -74,7 +77,7 @@ class Account(BaseAccount):
|
||||
|
||||
def get_balance(self, confirmations=0, include_claims=False, **constraints):
|
||||
if not include_claims:
|
||||
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
|
||||
constraints.update({'txo_type': 0})
|
||||
return super().get_balance(confirmations, **constraints)
|
||||
|
||||
async def get_granular_balances(self, confirmations=0, reserved_subtotals=False):
|
||||
@@ -82,7 +85,7 @@ class Account(BaseAccount):
|
||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
||||
total = await get_total_balance()
|
||||
if reserved_subtotals:
|
||||
claims_balance = await get_total_balance(claim_type__or={'is_claim': True, 'is_update': True})
|
||||
claims_balance = await get_total_balance(txo_type__in=[TXO_TYPES['stream'], TXO_TYPES['channel']])
|
||||
for amount, spent, from_me, to_me, height in await self.get_support_summary():
|
||||
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
|
||||
continue
|
||||
@@ -94,8 +97,7 @@ class Account(BaseAccount):
|
||||
reserved = claims_balance + supports_balance + tips_balance
|
||||
else:
|
||||
reserved = await self.get_balance(
|
||||
confirmations=confirmations, include_claims=True,
|
||||
claim_type__or={'is_claim': True, 'is_support': True, 'is_update': True}
|
||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
||||
)
|
||||
return {
|
||||
'total': dewies_to_lbc(total),
|
||||
@@ -118,48 +120,53 @@ class Account(BaseAccount):
|
||||
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
|
||||
account = super().from_dict(ledger, wallet, d)
|
||||
account.channel_keys = d.get('certificates', {})
|
||||
account.preferences = d.get('preferences', {})
|
||||
return account
|
||||
|
||||
def to_dict(self, include_channel_keys=True):
|
||||
def to_dict(self, include_channel_keys=True, include_preferences=True):
|
||||
d = super().to_dict()
|
||||
if include_channel_keys:
|
||||
d['certificates'] = self.channel_keys
|
||||
if include_preferences and self.preferences:
|
||||
d['preferences'] = self.preferences
|
||||
return d
|
||||
|
||||
async def get_details(self, **kwargs):
|
||||
details = await super().get_details(**kwargs)
|
||||
details['certificates'] = len(self.channel_keys)
|
||||
if self.preferences:
|
||||
details['preferences'] = self.preferences
|
||||
return details
|
||||
|
||||
def get_transaction_history(self, **constraints):
|
||||
return self.ledger.get_transaction_history(account=self, **constraints)
|
||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
return self.ledger.get_transaction_history_count(account=self, **constraints)
|
||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.get_claims(account=self, **constraints)
|
||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
return self.ledger.get_claim_count(account=self, **constraints)
|
||||
return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
return self.ledger.get_streams(account=self, **constraints)
|
||||
return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
return self.ledger.get_stream_count(account=self, **constraints)
|
||||
return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
return self.ledger.get_channels(account=self, **constraints)
|
||||
return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.get_channel_count(account=self, **constraints)
|
||||
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.ledger.get_supports(account=self, **constraints)
|
||||
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
return self.ledger.get_support_count(account=self, **constraints)
|
||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_summary(self):
|
||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
TXO_TYPES = {
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3
|
||||
}
|
||||
@@ -3,6 +3,7 @@ from typing import List
|
||||
from torba.client.basedatabase import BaseDatabase
|
||||
|
||||
from lbry.wallet.transaction import Output
|
||||
from lbry.wallet.constants import TXO_TYPES
|
||||
|
||||
|
||||
class WalletDatabase(BaseDatabase):
|
||||
@@ -17,53 +18,48 @@ class WalletDatabase(BaseDatabase):
|
||||
script blob not null,
|
||||
is_reserved boolean not null default 0,
|
||||
|
||||
txo_type integer not null default 0,
|
||||
claim_id text,
|
||||
claim_name text,
|
||||
is_claim boolean not null default 0,
|
||||
is_update boolean not null default 0,
|
||||
is_support boolean not null default 0,
|
||||
is_buy boolean not null default 0,
|
||||
is_sell boolean not null default 0
|
||||
claim_name text
|
||||
);
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
create index if not exists txo_claim_id_idx on txo (claim_id);
|
||||
create index if not exists txo_txo_type_idx on txo (txo_type);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXO_INDEX +
|
||||
BaseDatabase.CREATE_TXI_TABLE +
|
||||
BaseDatabase.CREATE_TXI_INDEX
|
||||
BaseDatabase.PRAGMAS +
|
||||
BaseDatabase.CREATE_ACCOUNT_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXI_TABLE
|
||||
)
|
||||
|
||||
def txo_to_row(self, tx, address, txo):
|
||||
row = super().txo_to_row(tx, address, txo)
|
||||
row.update({
|
||||
'is_claim': txo.script.is_claim_name,
|
||||
'is_update': txo.script.is_update_claim,
|
||||
'is_support': txo.script.is_support_claim,
|
||||
'is_buy': txo.script.is_buy_claim,
|
||||
'is_sell': txo.script.is_sell_claim,
|
||||
})
|
||||
if txo.is_claim:
|
||||
if txo.can_decode_claim:
|
||||
row['txo_type'] = TXO_TYPES.get(txo.claim.claim_type, TXO_TYPES['stream'])
|
||||
else:
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
row['txo_type'] = TXO_TYPES['support']
|
||||
if txo.script.is_claim_involved:
|
||||
row['claim_id'] = txo.claim_id
|
||||
row['claim_name'] = txo.claim_name
|
||||
return row
|
||||
|
||||
async def get_txos(self, **constraints) -> List[Output]:
|
||||
my_accounts = constraints.get('my_accounts', constraints.get('accounts', []))
|
||||
|
||||
txos = await super().get_txos(**constraints)
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints) -> List[Output]:
|
||||
txos = await super().get_txos(wallet=wallet, no_tx=no_tx, **constraints)
|
||||
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and my_accounts:
|
||||
for account in my_accounts:
|
||||
if txo.claim.is_channel and wallet:
|
||||
for account in wallet.accounts:
|
||||
private_key = account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
@@ -75,7 +71,7 @@ class WalletDatabase(BaseDatabase):
|
||||
channels = {
|
||||
txo.claim_id: txo for txo in
|
||||
(await self.get_claims(
|
||||
my_accounts=my_accounts,
|
||||
wallet=wallet,
|
||||
claim_id__in=channel_ids
|
||||
))
|
||||
}
|
||||
@@ -87,7 +83,9 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
@staticmethod
|
||||
def constrain_claims(constraints):
|
||||
constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1}
|
||||
constraints['txo_type__in'] = [
|
||||
TXO_TYPES['stream'], TXO_TYPES['channel']
|
||||
]
|
||||
|
||||
async def get_claims(self, **constraints) -> List[Output]:
|
||||
self.constrain_claims(constraints)
|
||||
@@ -99,8 +97,7 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
@staticmethod
|
||||
def constrain_streams(constraints):
|
||||
if 'claim_name' not in constraints or 'claim_id' not in constraints:
|
||||
constraints['claim_name__not_like'] = '@%'
|
||||
constraints['txo_type'] = TXO_TYPES['stream']
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
self.constrain_streams(constraints)
|
||||
@@ -112,8 +109,7 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
@staticmethod
|
||||
def constrain_channels(constraints):
|
||||
if 'claim_name' not in constraints or 'claim_id' not in constraints:
|
||||
constraints['claim_name__like'] = '@%'
|
||||
constraints['txo_type'] = TXO_TYPES['channel']
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
self.constrain_channels(constraints)
|
||||
@@ -125,7 +121,7 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
@staticmethod
|
||||
def constrain_supports(constraints):
|
||||
constraints['is_support'] = 1
|
||||
constraints['txo_type'] = TXO_TYPES['support']
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
self.constrain_supports(constraints)
|
||||
@@ -136,20 +132,20 @@ class WalletDatabase(BaseDatabase):
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
" SELECT address from account_address WHERE account = ?"
|
||||
" )", (account.public_key.address, )
|
||||
)
|
||||
|
||||
def get_supports_summary(self, account_id):
|
||||
return self.db.execute_fetchall("""
|
||||
return self.db.execute_fetchall(f"""
|
||||
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
|
||||
(txo.txid in
|
||||
(select txi.txid from txi join pubkey_address a on txi.address = a.address
|
||||
(select txi.txid from txi join account_address a on txi.address = a.address
|
||||
where a.account = ?)) as from_me,
|
||||
(txo.address in (select address from pubkey_address where account=?)) as to_me,
|
||||
(txo.address in (select address from account_address where account=?)) as to_me,
|
||||
tx.height
|
||||
from txo join tx using (txid) where is_support=1
|
||||
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
|
||||
""", (account_id, account_id))
|
||||
@@ -110,7 +110,7 @@ class MainNetLedger(BaseLedger):
|
||||
|
||||
@staticmethod
|
||||
def constraint_spending_utxos(constraints):
|
||||
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
|
||||
constraints['txo_type'] = 0
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_spending_utxos(constraints)
|
||||
@@ -121,39 +121,30 @@ class MainNetLedger(BaseLedger):
|
||||
return super().get_utxo_count(**constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claims(**constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claim_count(**constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_streams(**constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_stream_count(**constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channels(**constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channel_count(**constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_supports(**constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_support_count(**constraints)
|
||||
|
||||
async def get_transaction_history(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
txs = await self.db.get_transactions(**constraints)
|
||||
headers = self.headers
|
||||
history = []
|
||||
@@ -249,7 +240,6 @@ class MainNetLedger(BaseLedger):
|
||||
return history
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
|
||||
|
||||
@@ -138,20 +138,23 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if not os.path.exists(wallets_directory):
|
||||
os.mkdir(wallets_directory)
|
||||
|
||||
wallet_file_path = os.path.join(wallets_directory, 'default_wallet')
|
||||
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(wallet_file_path)
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(
|
||||
os.path.join(wallets_directory, 'default_wallet')
|
||||
)
|
||||
|
||||
manager = cls.from_config({
|
||||
'ledgers': {ledger_id: ledger_config},
|
||||
'wallets': [wallet_file_path]
|
||||
'wallets': [
|
||||
os.path.join(wallets_directory, wallet_file) for wallet_file in settings.wallets
|
||||
]
|
||||
})
|
||||
ledger = manager.get_or_create_ledger(ledger_id)
|
||||
ledger.coin_selection_strategy = settings.coin_selection_strategy
|
||||
if manager.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', wallet_file_path)
|
||||
manager.default_wallet.generate_account(ledger)
|
||||
manager.default_wallet.save()
|
||||
default_wallet = manager.default_wallet
|
||||
if default_wallet.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', default_wallet.id)
|
||||
default_wallet.generate_account(ledger)
|
||||
default_wallet.save()
|
||||
if receiving_addresses or change_addresses:
|
||||
if not os.path.exists(ledger.path):
|
||||
os.mkdir(ledger.path)
|
||||
@@ -199,6 +202,8 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if not tx:
|
||||
try:
|
||||
raw = await self.ledger.network.get_transaction(txid)
|
||||
if not raw:
|
||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||
height = await self.ledger.network.get_transaction_height(txid)
|
||||
except CodeMessageError as e:
|
||||
return {'success': False, 'code': e.code, 'message': e.message}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Union, Tuple, Set, List
|
||||
from itertools import chain
|
||||
@@ -705,7 +706,8 @@ class LBRYDB(DB):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.sql = SQLDB(self, 'claims.db')
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
self.sql = SQLDB(self, path)
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
|
||||
@@ -52,7 +52,7 @@ class APICallMetrics:
|
||||
self.errored_queries = set()
|
||||
self.interrupted_queries = set()
|
||||
|
||||
def to_json_and_reset(self):
|
||||
def to_json(self):
|
||||
return {
|
||||
# total requests received
|
||||
"receive_count": self.receive_count,
|
||||
@@ -128,7 +128,7 @@ class ServerLoadData:
|
||||
def to_json_and_reset(self, status):
|
||||
try:
|
||||
return {
|
||||
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
|
||||
'api': {name: api.to_json() for name, api in self._apis.items()},
|
||||
'status': status
|
||||
}
|
||||
finally:
|
||||
|
||||
@@ -64,9 +64,10 @@ class LBRYSessionManager(SessionManager):
|
||||
|
||||
async def start_other(self):
|
||||
self.running = True
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
args = dict(
|
||||
initializer=reader.initializer,
|
||||
initargs=(self.logger, 'claims.db', self.env.coin.NET, self.env.database_query_timeout,
|
||||
initargs=(self.logger, path, self.env.coin.NET, self.env.database_query_timeout,
|
||||
self.env.track_metrics)
|
||||
)
|
||||
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
|
||||
@@ -112,33 +113,44 @@ class LBRYElectrumX(ElectrumX):
|
||||
}
|
||||
self.request_handlers.update(handlers)
|
||||
|
||||
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
|
||||
def get_metrics_or_placeholder_for_api(self, query_name):
|
||||
""" Do not hold on to a reference to the metrics
|
||||
returned by this method past an `await` or
|
||||
you may be working with a stale metrics object.
|
||||
"""
|
||||
if self.env.track_metrics:
|
||||
return self.session_mgr.metrics.for_api(query_name)
|
||||
else:
|
||||
return APICallMetrics(query_name)
|
||||
|
||||
async def run_in_executor(self, query_name, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, error.metrics)
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, {})
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
|
||||
if self.env.track_metrics:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
(result, metrics_data) = result
|
||||
metrics.query_response(start, metrics_data)
|
||||
|
||||
return base64.b64encode(result).decode()
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
if self.env.track_metrics:
|
||||
metrics = self.session_mgr.metrics.for_api(query_name)
|
||||
else:
|
||||
metrics = APICallMetrics(query_name)
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.start()
|
||||
cache = self.session_mgr.search_cache[query_name]
|
||||
cache_key = str(kwargs)
|
||||
@@ -151,9 +163,10 @@ class LBRYElectrumX(ElectrumX):
|
||||
async with cache_item.lock:
|
||||
if cache_item.result is None:
|
||||
cache_item.result = await self.run_in_executor(
|
||||
metrics, function, kwargs
|
||||
query_name, function, kwargs
|
||||
)
|
||||
else:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
|
||||
|
||||
@@ -203,7 +203,7 @@ class Transaction(BaseTransaction):
|
||||
|
||||
@classmethod
|
||||
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address))
|
||||
return cls.create([], [output], funding_accounts, change_account)
|
||||
|
||||
@@ -211,7 +211,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_create(
|
||||
cls, name: str, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
claim_output = Output.pay_claim_name_pubkey_hash(
|
||||
amount, name, claim, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -223,7 +223,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_update(
|
||||
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
claim, ledger.address_to_hash160(holding_address)
|
||||
@@ -239,7 +239,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -248,7 +248,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
claim_output = Output.purchase_claim_pubkey_hash(
|
||||
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ RUN mkdir -p /home/$user/
|
||||
RUN chown -R $user:$user /home/$user/
|
||||
|
||||
# install python, pip, git and clean up
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create and chown database dir
|
||||
ARG db_dir=/database
|
||||
@@ -53,4 +53,5 @@ ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
|
||||
COPY ./lbry/scripts/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -13,13 +13,15 @@ from lbry.extras.daemon.Daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from lbry.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
|
||||
encode_wallet_doc
|
||||
)
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
RETURN_DOCS = {
|
||||
'Account': encode_account_doc(),
|
||||
'Wallet': encode_wallet_doc(),
|
||||
'File': encode_file_doc(),
|
||||
'Transaction': encode_tx_doc(),
|
||||
'Output': encode_txo_doc(),
|
||||
@@ -78,11 +80,38 @@ class Examples(CommandTestCase):
|
||||
'Get version',
|
||||
'version'
|
||||
)
|
||||
|
||||
# settings
|
||||
|
||||
await r(
|
||||
'Get settings',
|
||||
'settings', 'get'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Set settings',
|
||||
'settings', 'set', '"tcp_port"', '99'
|
||||
)
|
||||
|
||||
# preferences
|
||||
|
||||
await r(
|
||||
'Set preference',
|
||||
'preference', 'set', '"theme"', '"dark"'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Get preferences',
|
||||
'preference', 'get'
|
||||
)
|
||||
|
||||
# wallets
|
||||
|
||||
await r(
|
||||
'List your wallets',
|
||||
'wallet', 'list'
|
||||
)
|
||||
|
||||
# accounts
|
||||
|
||||
await r(
|
||||
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
# entrypoint for wallet server Docker image
|
||||
|
||||
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
|
||||
|
||||
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
|
||||
wget -O wallet_snapshot.tar.bz2 "$SNAPSHOT_URL"
|
||||
echo "Extracting snapshot..."
|
||||
tar xvjf wallet_snapshot.tar.bz2 --directory /database
|
||||
rm wallet_snapshot.tar.bz2
|
||||
fi
|
||||
|
||||
/home/lbry/.local/bin/torba-server "$@"
|
||||
@@ -1,5 +1,9 @@
|
||||
import asyncio
|
||||
import sqlite3
|
||||
from lbry.testcase import CommandTestCase
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.account import Account
|
||||
|
||||
|
||||
def extract(d, keys):
|
||||
@@ -7,7 +11,6 @@ def extract(d, keys):
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
async def test_account_list_set_create_remove_add(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
@@ -82,7 +85,7 @@ class AccountManagement(CommandTestCase):
|
||||
stream_list = self.daemon.jsonrpc_stream_list
|
||||
support_list = self.daemon.jsonrpc_support_list
|
||||
utxo_list = self.daemon.jsonrpc_utxo_list
|
||||
default_account = self.daemon.default_account
|
||||
default_account = self.wallet.default_account
|
||||
second_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
|
||||
tx = await self.daemon.jsonrpc_account_send(
|
||||
@@ -168,3 +171,9 @@ class AccountManagement(CommandTestCase):
|
||||
})
|
||||
self.assertEqual(history[6]['value'], '0.0')
|
||||
self.assertEqual(history[7]['value'], '10.0')
|
||||
|
||||
async def test_address_validation(self):
|
||||
address = await self.daemon.jsonrpc_address_unused()
|
||||
bad_address = address[0:20] + '9999999' + address[27:]
|
||||
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
|
||||
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
|
||||
@@ -466,79 +466,46 @@ class ChannelCommands(CommandTestCase):
|
||||
'title': 'foo', 'email': 'new@email.com'}
|
||||
)
|
||||
|
||||
# send channel to someone else
|
||||
# move channel to another account
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
# before moving
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
|
||||
|
||||
other_address = await account2.receiving.get_or_create_usable_address()
|
||||
tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
|
||||
|
||||
# after sending
|
||||
# after moving
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
|
||||
|
||||
# shoud not have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNone(txo.private_key)
|
||||
|
||||
# send the private key too
|
||||
private_key = self.account.get_channel_private_key(unhexlify(channel['public_key']))
|
||||
account2.add_channel_private_key(private_key)
|
||||
|
||||
# now should have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNotNone(txo.private_key)
|
||||
|
||||
async def test_channel_export_import_into_new_account(self):
|
||||
async def test_channel_export_import_before_sending_channel(self):
|
||||
# export
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
channel_private_key = (await self.account.get_channels())[0].private_key
|
||||
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
|
||||
|
||||
# import
|
||||
daemon2 = await self.add_daemon()
|
||||
|
||||
# before importing channel
|
||||
self.assertEqual(1, len(daemon2.default_wallet.accounts))
|
||||
|
||||
# importing channel which will create a new single key account
|
||||
self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
|
||||
await daemon2.jsonrpc_channel_import(exported_data)
|
||||
|
||||
# after import
|
||||
self.assertEqual(2, len(daemon2.default_wallet.accounts))
|
||||
new_account = daemon2.default_wallet.accounts[1]
|
||||
await daemon2.ledger._update_tasks.done.wait()
|
||||
channels = await new_account.get_channels()
|
||||
channels = await daemon2.jsonrpc_channel_list()
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
async def test_channel_export_import_into_existing_account(self):
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
channel_private_key = (await self.account.get_channels())[0].private_key
|
||||
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
|
||||
# second wallet can't update until channel is sent to it
|
||||
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
daemon2 = await self.add_daemon(seed=self.account.seed)
|
||||
await daemon2.ledger._update_tasks.done.wait() # will sync channel previously created
|
||||
# now send the channel as well
|
||||
await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused())
|
||||
|
||||
# before importing channel key, has channel without key
|
||||
self.assertEqual(1, len(daemon2.default_wallet.accounts))
|
||||
channels = await daemon2.default_account.get_channels()
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertIsNone(channels[0].private_key)
|
||||
|
||||
# importing channel will add it to existing account
|
||||
await daemon2.jsonrpc_channel_import(exported_data)
|
||||
|
||||
# after import, still just one account but with private key now
|
||||
self.assertEqual(1, len(daemon2.default_wallet.accounts))
|
||||
channels = await daemon2.default_account.get_channels()
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
# second wallet should be able to update now
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
|
||||
class StreamCommands(ClaimTestCase):
|
||||
@@ -600,7 +567,7 @@ class StreamCommands(ClaimTestCase):
|
||||
async def test_publishing_checks_all_accounts_for_channel(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
|
||||
@@ -653,6 +620,15 @@ class StreamCommands(ClaimTestCase):
|
||||
'hovercraft5', '0.1', channel_name='@baz', channel_account_id=[account1_id]
|
||||
)
|
||||
|
||||
# signing with channel works even if channel and certificate are in different accounts
|
||||
await self.channel_update(
|
||||
baz_id, account_id=account2_id,
|
||||
claim_address=await self.daemon.jsonrpc_address_unused(account1_id)
|
||||
)
|
||||
await self.stream_create(
|
||||
'hovercraft5', '0.1', channel_id=baz_id
|
||||
)
|
||||
|
||||
async def test_preview_works_with_signed_streams(self):
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
signed = await self.out(self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False))
|
||||
@@ -808,7 +784,7 @@ class StreamCommands(ClaimTestCase):
|
||||
|
||||
# send claim to someone else
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
|
||||
@@ -1105,13 +1081,12 @@ class StreamCommands(ClaimTestCase):
|
||||
class SupportCommands(CommandTestCase):
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
|
||||
account2 = self.daemon.get_account_or_error(account2_id)
|
||||
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
|
||||
account2 = wallet2.accounts[0]
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2')
|
||||
))
|
||||
await self.on_transaction_dict(result)
|
||||
|
||||
@@ -1129,7 +1104,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '1.0', True, account2_id, funding_account_ids=[account2_id])
|
||||
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(tip['txid'])
|
||||
|
||||
@@ -1148,7 +1123,7 @@ class SupportCommands(CommandTestCase):
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
@@ -1160,7 +1135,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '2.0', False, account2_id, funding_account_ids=[account2_id])
|
||||
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(support['txid'])
|
||||
|
||||
@@ -1169,7 +1144,7 @@ class SupportCommands(CommandTestCase):
|
||||
await self.assertBalance(account2, '1.999717')
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
|
||||
|
||||
@@ -26,6 +26,7 @@ class MockedCommentServer:
|
||||
'signing_ts': None,
|
||||
'timestamp': None,
|
||||
'channel_url': None,
|
||||
'is_hidden': False,
|
||||
}
|
||||
|
||||
def __init__(self, port=2903):
|
||||
@@ -45,10 +46,9 @@ class MockedCommentServer:
|
||||
|
||||
@staticmethod
|
||||
def clean(d: dict):
|
||||
return {k: v for k, v in d.items() if v}
|
||||
return {k: v for k, v in d.items() if v or isinstance(v, bool)}
|
||||
|
||||
def create_comment(self, channel_name=None, channel_id=None, **kwargs):
|
||||
self.comment_id += 1
|
||||
comment_id = self.comment_id
|
||||
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
|
||||
comment = self._create_comment(
|
||||
@@ -60,57 +60,97 @@ class MockedCommentServer:
|
||||
**kwargs
|
||||
)
|
||||
self.comments.append(comment)
|
||||
self.comment_id += 1
|
||||
return self.clean(comment)
|
||||
|
||||
def delete_comment(self, comment_id: int, channel_id: str, **kwargs):
|
||||
def abandon_comment(self, comment_id: int, channel_id: str, **kwargs):
|
||||
deleted = False
|
||||
try:
|
||||
if 0 <= comment_id <= len(self.comments) and self.comments[comment_id - 1]['channel_id'] == channel_id:
|
||||
self.comments.pop(comment_id - 1)
|
||||
if 0 <= comment_id < len(self.comments) and self.comments[comment_id]['channel_id'] == channel_id:
|
||||
self.comments.pop(comment_id)
|
||||
deleted = True
|
||||
finally:
|
||||
return {
|
||||
str(comment_id): {
|
||||
'deleted': deleted
|
||||
'abandoned': deleted
|
||||
}
|
||||
}
|
||||
|
||||
def get_claim_comments(self, page=1, page_size=50, **kwargs):
|
||||
def hide_comment(self, comment_id, signing_ts, signature):
|
||||
comment_id = int(comment_id) if not isinstance(comment_id, int) else comment_id
|
||||
if 0 <= comment_id < len(self.comments) and len(signature) == 128 and signing_ts.isalnum():
|
||||
self.comments[comment_id]['is_hidden'] = True
|
||||
return True
|
||||
return False
|
||||
|
||||
def hide_comments(self, pieces: list):
|
||||
comments_hidden = []
|
||||
for p in pieces:
|
||||
if self.hide_comment(**p):
|
||||
comments_hidden.append(p['comment_id'])
|
||||
return {'hidden': comments_hidden}
|
||||
|
||||
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
return {
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'total_pages': ceil(len(self.comments)/page_size),
|
||||
'total_items': len(self.comments),
|
||||
'items': [self.clean(c) for c in (self.comments[::-1])[(page - 1) * page_size: page * page_size]]
|
||||
'total_pages': ceil(len(comments)/page_size),
|
||||
'total_items': len(comments),
|
||||
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
|
||||
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
|
||||
}
|
||||
|
||||
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
|
||||
return {
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'total_pages': ceil(len(select_comments) / page_size),
|
||||
'total_items': len(select_comments),
|
||||
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
|
||||
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
|
||||
}
|
||||
|
||||
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
|
||||
comment = self.comments[comment_id - 1]
|
||||
comment = self.comments[comment_id]
|
||||
return {
|
||||
'channel_id': comment.get('channel_id'),
|
||||
'channel_name': comment.get('channel_name')
|
||||
}
|
||||
|
||||
def get_comments_by_id(self, comment_ids: list):
|
||||
comment_ids = [int(c) if not isinstance(c, int) else c for c in comment_ids]
|
||||
comments = [self.comments[cmnt_id] for cmnt_id in comment_ids if 0 <= cmnt_id < len(self.comments)]
|
||||
return comments
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
'get_comments_by_id': get_comments_by_id,
|
||||
'create_comment': create_comment,
|
||||
'delete_comment': delete_comment,
|
||||
'abandon_comment': abandon_comment,
|
||||
'get_channel_from_comment_id': get_comment_channel_by_id,
|
||||
'get_claim_hidden_comments': get_claim_hidden_comments,
|
||||
'hide_comments': hide_comments,
|
||||
}
|
||||
|
||||
def process_json(self, body) -> dict:
|
||||
response = {'jsonrpc': '2.0', 'id': body['id']}
|
||||
try:
|
||||
if body['method'] in self.methods:
|
||||
params = body.get('params', {})
|
||||
if 'comment_id' in params and type(params['comment_id']) is str:
|
||||
params['comment_id'] = int(params['comment_id'])
|
||||
params: dict = body.get('params', {})
|
||||
comment_id = params.get('comment_id')
|
||||
if comment_id and not isinstance(comment_id, int):
|
||||
params['comment_id'] = int(comment_id)
|
||||
|
||||
result = self.methods[body['method']](self, **params)
|
||||
response['result'] = result
|
||||
else:
|
||||
response['error'] = self.ERRORS['INVALID_METHOD']
|
||||
except Exception:
|
||||
except Exception as err:
|
||||
response['error'] = self.ERRORS['UNKNOWN']
|
||||
response['error'].update({'exception': f'{type(err).__name__}: {err}'})
|
||||
return response
|
||||
|
||||
async def start(self):
|
||||
@@ -181,20 +221,19 @@ class CommentCommands(CommandTestCase):
|
||||
async def test02_unsigned_comment_list(self):
|
||||
stream = (await self.stream_create())['outputs'][0]
|
||||
comments = []
|
||||
for i in range(28):
|
||||
num_items = 28
|
||||
for i in range(num_items):
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment=f'{i}',
|
||||
claim_id=stream['claim_id'],
|
||||
)
|
||||
self.assertIn('comment_id', comment)
|
||||
comments.append(comment)
|
||||
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(
|
||||
claim_id=stream['claim_id']
|
||||
)
|
||||
self.assertIs(comment_list['page_size'], 50)
|
||||
self.assertIs(comment_list['page'], 1)
|
||||
self.assertIs(comment_list['total_items'], 28)
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
|
||||
for field in list_fields:
|
||||
self.assertIn(field, comment_list)
|
||||
self.assertEqual(comment_list['total_items'], num_items)
|
||||
for comment in comment_list['items']:
|
||||
self.assertEqual(comment['comment'], comments.pop()['comment'])
|
||||
|
||||
@@ -216,10 +255,12 @@ class CommentCommands(CommandTestCase):
|
||||
)
|
||||
self.assertIn('comment_id', comment)
|
||||
comments.append(comment)
|
||||
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(
|
||||
claim_id=stream['claim_id']
|
||||
)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, comment_list)
|
||||
self.assertIs(comment_list['page_size'], 50)
|
||||
self.assertIs(comment_list['page'], 1)
|
||||
self.assertIs(comment_list['total_items'], 28)
|
||||
@@ -241,10 +282,120 @@ class CommentCommands(CommandTestCase):
|
||||
channel_id=rswanson['claim_id']
|
||||
)
|
||||
self.assertIn('signature', comment)
|
||||
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], deleted)
|
||||
self.assertTrue(deleted[comment['comment_id']]['deleted'])
|
||||
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], abandoned)
|
||||
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
|
||||
|
||||
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], deleted)
|
||||
self.assertFalse(deleted[comment['comment_id']]['deleted'])
|
||||
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertFalse(abandoned[comment['comment_id']]['abandoned'])
|
||||
|
||||
async def test05_comment_hide(self):
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
moth_id = moth['claim_id']
|
||||
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
|
||||
claim_id = stream['outputs'][0]['claim_id']
|
||||
|
||||
comment1 = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Who on earth would want to sit around on a lamp all day',
|
||||
claim_id=claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
self.assertFalse(comment1['is_hidden'])
|
||||
|
||||
comment2 = await self.daemon.jsonrpc_comment_create(
|
||||
comment='silence mortal',
|
||||
claim_id=claim_id,
|
||||
channel_id=moth_id,
|
||||
)
|
||||
self.assertFalse(comment2['is_hidden'])
|
||||
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertIn('has_hidden_comments', comments)
|
||||
self.assertFalse(comments['has_hidden_comments'])
|
||||
|
||||
hidden = await self.daemon.jsonrpc_comment_hide([comment1['comment_id']])
|
||||
self.assertIn('hidden', hidden)
|
||||
hidden = hidden['hidden']
|
||||
self.assertIn(comment1['comment_id'], hidden)
|
||||
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertIn('has_hidden_comments', comments)
|
||||
self.assertTrue(comments['has_hidden_comments'])
|
||||
hidden_cmts1 = list(filter(lambda c: c['is_hidden'], comments['items']))
|
||||
self.assertTrue(len(hidden_cmts1) == 1)
|
||||
hidden_comment = hidden_cmts1[0]
|
||||
self.assertEqual(hidden_comment['comment_id'], hidden[0])
|
||||
|
||||
hidden_comments = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
|
||||
self.assertIn('has_hidden_comments', hidden_comments)
|
||||
self.assertTrue(hidden_comments['has_hidden_comments'])
|
||||
self.assertLess(hidden_comments['total_items'], comments['total_items'])
|
||||
self.assertListEqual(hidden_comments['items'], hidden_cmts1)
|
||||
|
||||
visible_comments = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
|
||||
self.assertIn('has_hidden_comments', visible_comments)
|
||||
self.assertTrue(visible_comments['has_hidden_comments'])
|
||||
self.assertLess(visible_comments['total_items'], comments['total_items'])
|
||||
total_hidden = hidden_comments['total_items']
|
||||
total_visible = visible_comments['total_items']
|
||||
self.assertEqual(total_hidden + total_visible, comments['total_items'])
|
||||
|
||||
items_hidden = hidden_comments['items']
|
||||
items_visible = visible_comments['items']
|
||||
for item in items_visible + items_hidden:
|
||||
self.assertIn(item, comments['items'])
|
||||
|
||||
async def test06_comment_list_test(self):
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
moth_id = moth['claim_id']
|
||||
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
|
||||
claim_id = stream['outputs'][0]['claim_id']
|
||||
hidden_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Who on earth would want to sit around on a lamp all day',
|
||||
claim_id=claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
await self.daemon.jsonrpc_comment_hide([hidden_comment['comment_id']])
|
||||
owner_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Go away you yellow freak',
|
||||
claim_id=claim_id,
|
||||
channel_id=moth_id,
|
||||
)
|
||||
other_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='I got my swim trunks and my flippy-floppies',
|
||||
claim_id=claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
anon_comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim_id,
|
||||
comment='Anonymous comment'
|
||||
)
|
||||
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, normal_list)
|
||||
self.assertEqual(normal_list['total_items'], 4)
|
||||
self.assertTrue(normal_list['has_hidden_comments'])
|
||||
for i, cmnt in enumerate(all_comments):
|
||||
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
|
||||
|
||||
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
|
||||
self.assertTrue(hidden['has_hidden_comments'])
|
||||
for field in list_fields:
|
||||
self.assertIn(field, hidden)
|
||||
self.assertEqual(hidden['total_items'], 1)
|
||||
|
||||
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, visible)
|
||||
self.assertTrue(visible['has_hidden_comments'])
|
||||
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
|
||||
|
||||
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, valid_list)
|
||||
self.assertTrue(visible['has_hidden_comments'])
|
||||
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
|
||||
@@ -277,6 +277,33 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0')
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address)
|
||||
|
||||
async def test_null_max_key_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
self.daemon.conf.max_key_fee = None
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
await self.assertBalance(self.account, '9.483893')
|
||||
response = await self.daemon.jsonrpc_get('lbry://somename')
|
||||
await self.ledger.wait(response.content_fee)
|
||||
await self.assertBalance(self.account, '8.483769')
|
||||
|
||||
# Assert the file downloads
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
# Assert the transaction is recorded to the blockchain
|
||||
starting_balance = await self.blockchain.get_balance()
|
||||
await self.generate(1)
|
||||
block_reward_and_claim_fee = 2.0
|
||||
self.assertEqual(
|
||||
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
|
||||
)
|
||||
|
||||
async def test_null_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
tx = await self.stream_create(
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AddressManagement(CommandTestCase):
|
||||
|
||||
async def test_address_list(self):
|
||||
addresses = await self.out(self.daemon.jsonrpc_address_list())
|
||||
self.assertEqual(27, len(addresses))
|
||||
|
||||
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
|
||||
self.assertEqual(1, len(single))
|
||||
self.assertEqual(single[0], addresses[11])
|
||||
@@ -84,3 +84,31 @@ class AccountSynchronization(AsyncioTestCase):
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
|
||||
self.assertEqual(self.account.channel_keys, {'abcdefg1234:0': '---PRIVATE KEY---'})
|
||||
|
||||
@mock.patch('time.time', mock.Mock(return_value=12345))
|
||||
def test_account_preferences_syncing(self):
|
||||
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
|
||||
self.account.modified_on = 123.456
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
|
||||
self.assertFalse(self.daemon.jsonrpc_preference_get())
|
||||
|
||||
hash_w_pref = '2fe43f0b2f8bbf1fbb55537f862d8bcb0823791019a7151c848bd5f5bd32d336'
|
||||
add_pref = (
|
||||
'czo4MTkyOjE2OjE6Jgn3nAGrfYP2usMA4KQ/73+YHAwMyiGdSWuxCmgZKlpwSpnfQv8R7R0tum/n2oTSBQxjdL'
|
||||
'OlTW+tv/G5L2GfQ5op3xaT89gN+F/JJnvf3cdWvYH7Nc+uTUMb7cKhJP7hQvFW5bb1Y3jX3EBBY00Jkqyj9RCR'
|
||||
'XPtbLVu71KbVRvCAR/oAnMEsgD+ITsC3WkXMwE3BS2LjJDQmeqbH4YXNdcjJN/JzQ6fxOmr3Uk1GqnpuhFsta8'
|
||||
'H14ViRilq1pLKOSZIN80rrm5cKq45nFO5kFeoqBCEaal4u2/OkX9nOnpQlO3E95wD8hkCmZ3i20aSte6nqwqXx'
|
||||
'ZKVRZqR2a0TjwVWB8kPXPA2ewKvPILaj190bXPl8EVu+TAnTCQwMgytinYjtcKNZmMz3ENJyI2mCANwpWlX7xl'
|
||||
'y/J+qLi5b9N+agghTxggs5rVJ/hkaue7GS542dXDrwMrw9nwGqNw3dS/lcU+1wRUQ0fnHwb/85XbbwyO2aDj2i'
|
||||
'DFNkdyLyUIiIUvB1JfWAnWqX3vQcL1REK1ePgUei7dCHJ3WyWdsRx3cVXzlK8yOPkf0N6d3AKrZQWVebwDC7Nd'
|
||||
'eL4sDW8AkaXuBIrbuZw6XUHd6WI0NvU/q10j2qMm0YoXSu+dExou1/1THwx5g86MxcX5nwodKUEVCOTzKMyrLz'
|
||||
'CRsitH/+dAXhZNRp/FbnDCGBMyD3MOYCjZvAFbCZUasoRwqponxILw=='
|
||||
)
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_pref)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
|
||||
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["apple", "orange"]})
|
||||
|
||||
self.daemon.jsonrpc_preference_set("fruit", ["peach", "apricot"])
|
||||
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["peach", "apricot"]})
|
||||
self.assertNotEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
|
||||
@@ -20,7 +20,9 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
await self.ledger.stop()
|
||||
self.conductor.spv_node.session_timeout = 1
|
||||
await self.conductor.start_spv()
|
||||
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
|
||||
session = ClientSession(
|
||||
network=None, server=(self.conductor.spv_node.hostname, self.conductor.spv_node.port), timeout=0.2
|
||||
)
|
||||
await session.create_connection()
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 1)
|
||||
|
||||
@@ -55,10 +55,41 @@ class TestURLParsing(unittest.TestCase):
|
||||
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
|
||||
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
|
||||
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
|
||||
# unicode regex edges
|
||||
_url = lambda name: url(name, stream_name=name)
|
||||
_url('\u0009')
|
||||
_url('\u000a')
|
||||
_url('\u000d')
|
||||
_url('\u0020')
|
||||
_url('\uD799')
|
||||
_url('\uE000')
|
||||
_url('\uFFFD')
|
||||
|
||||
def test_parser_invalid_urls(self):
|
||||
fail = self._fail_url
|
||||
fail("lbry://")
|
||||
fail("lbry://\u0000")
|
||||
fail("lbry://\u0008")
|
||||
fail("lbry://\u000b")
|
||||
fail("lbry://\u000c")
|
||||
fail("lbry://\u000e")
|
||||
fail("lbry://\u001f")
|
||||
fail("lbry://\uD800")
|
||||
fail("lbry://\uDFFF")
|
||||
fail("lbry://\uDFFE")
|
||||
fail("lbry://\uFFFF")
|
||||
fail("lbry://;")
|
||||
fail("lbry://\"")
|
||||
fail("lbry://\\")
|
||||
fail("lbry:///")
|
||||
fail("lbry://<") and fail("lbry://>")
|
||||
fail("lbry://{") and fail("lbry://}")
|
||||
fail("lbry://[") and fail("lbry://]")
|
||||
fail("lbry://%")
|
||||
fail("lbry://|")
|
||||
fail("lbry://^")
|
||||
fail("lbry://~")
|
||||
fail("lbry://`")
|
||||
fail("lbry://test:3$1")
|
||||
fail("lbry://test$1:1")
|
||||
fail("lbry://test#x")
|
||||
|
||||
@@ -43,11 +43,11 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
async def test_status_file_completed(self):
|
||||
await self._test_transfer_stream(10)
|
||||
self.assertTrue(self.stream.output_file_exists)
|
||||
self.assertTrue(self.stream.as_dict()['completed'])
|
||||
self.assertTrue(self.stream.completed)
|
||||
with open(self.stream.full_path, 'w+b') as outfile:
|
||||
outfile.truncate(1)
|
||||
self.assertTrue(self.stream.output_file_exists)
|
||||
self.assertFalse(self.stream.as_dict()['completed'])
|
||||
self.assertFalse(self.stream.completed)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
|
||||
await self.setup_stream(blob_count)
|
||||
|
||||
@@ -51,6 +51,7 @@ class CLITest(AsyncioTestCase):
|
||||
self.assertEqual('3', normalize_value('3', key="name"))
|
||||
self.assertEqual('3', normalize_value('3', key="download_directory"))
|
||||
self.assertEqual('3', normalize_value('3', key="channel_name"))
|
||||
self.assertEqual('3', normalize_value('3', key="claim_name"))
|
||||
|
||||
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
|
||||
|
||||
|
||||
@@ -222,6 +222,10 @@ class ConfigurationTests(unittest.TestCase):
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
args = parser.parse_args(['--max-key-fee', 'null'])
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
# set
|
||||
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
|
||||
c = Config.create_from_arguments(args)
|
||||
|
||||
@@ -61,13 +61,17 @@ class TestAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx'
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR'
|
||||
'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(private_key)
|
||||
|
||||
def test_load_and_save_account(self):
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from unittest.mock import Mock
|
||||
|
||||
from torba.client.basenetwork import BaseNetwork
|
||||
from torba.orchstr8.node import SPVNode
|
||||
from torba.rpc import RPCSession
|
||||
from torba.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
|
||||
@@ -15,11 +17,63 @@ class NetworkTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_header.first
|
||||
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
|
||||
|
||||
async def test_server_features(self):
|
||||
self.assertEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
'daily_fee': 0,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
address = (await self.account.get_addresses(limit=1))[0]
|
||||
os.environ.update({
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'DONATION_ADDRESS': address,
|
||||
'DAILY_FEE': '42'})
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
'payment_address': address,
|
||||
'daily_fee': 42,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_multiple_servers(self):
|
||||
# we have a secondary node that connects later, so
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
await asyncio.wait_for(self.ledger.stop(), timeout=1)
|
||||
await asyncio.wait_for(self.ledger.start(), timeout=1)
|
||||
self.ledger.network.session_pool.new_connection_event.clear()
|
||||
await node2.start(self.blockchain)
|
||||
# this is only to speed up the test as retrying would take 4+ seconds
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
|
||||
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
switch_event = self.ledger.network.on_connected.first
|
||||
await node2.stop(True)
|
||||
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(switch_event, timeout=1)
|
||||
|
||||
async def test_connection_drop_still_receives_events_after_reconnected(self):
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
# disconnect and send a new tx, should reconnect and get it
|
||||
@@ -35,10 +89,11 @@ class ReconnectTests(IntegrationTestCase):
|
||||
# is it real? are we rich!? let me see this tx...
|
||||
d = self.ledger.network.get_transaction(sendtxid)
|
||||
# what's that smoke on my ethernet cable? oh no!
|
||||
master_client = self.ledger.network.client
|
||||
self.ledger.network.client.connection_lost(Exception())
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await d
|
||||
self.assertIsNone(self.ledger.network.client.response_time) # response time unknown as it failed
|
||||
await d
|
||||
self.assertIsNone(master_client.response_time) # response time unknown as it failed
|
||||
# rich but offline? no way, no water, let's retry
|
||||
with self.assertRaisesRegex(ConnectionError, 'connection is not available'):
|
||||
await self.ledger.network.get_transaction(sendtxid)
|
||||
@@ -104,4 +159,4 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
|
||||
# ensure we are connected to all of them after a while
|
||||
await asyncio.sleep(1)
|
||||
self.assertEqual(len(network.session_pool.available_sessions), 3)
|
||||
self.assertEqual(len(list(network.session_pool.available_sessions)), 3)
|
||||
@@ -1,6 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import random
|
||||
from itertools import chain
|
||||
from random import shuffle
|
||||
|
||||
from torba.testcase import IntegrationTestCase
|
||||
from torba.client.util import satoshis_to_coins, coins_to_satoshis
|
||||
|
||||
@@ -85,7 +88,7 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
await self.assertBalance(account1, '0.0')
|
||||
await self.assertBalance(account2, '0.0')
|
||||
|
||||
addresses = await self.account.receiving.get_addresses()
|
||||
addresses = await account1.receiving.get_addresses()
|
||||
txids = await asyncio.gather(*(
|
||||
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
|
||||
))
|
||||
@@ -129,3 +132,48 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address2)
|
||||
self.assertEqual(tx.outputs[0].is_change, False)
|
||||
self.assertEqual(tx.outputs[1].is_change, True)
|
||||
|
||||
async def test_history_edge_cases(self):
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
# evil trick: mempool is unsorted on real life, but same order between python instances. reproduce it
|
||||
original_summary = self.conductor.spv_node.server.mempool.transaction_summaries
|
||||
|
||||
async def random_summary(*args, **kwargs):
|
||||
summary = await original_summary(*args, **kwargs)
|
||||
if summary and len(summary) > 2:
|
||||
ordered = summary.copy()
|
||||
while summary == ordered:
|
||||
random.shuffle(summary)
|
||||
return summary
|
||||
self.conductor.spv_node.server.mempool.transaction_summaries = random_summary
|
||||
# 10 unconfirmed txs, all from blockchain wallet
|
||||
sends = list(self.blockchain.send_to_address(address, 10) for _ in range(10))
|
||||
# use batching to reduce issues with send_to_address on cli
|
||||
for batch in range(0, len(sends), 10):
|
||||
txids = await asyncio.gather(*sends[batch:batch + 10])
|
||||
await asyncio.wait([self.on_transaction_id(txid) for txid in txids])
|
||||
remote_status = await self.ledger.network.subscribe_address(address)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
# 20 unconfirmed txs, 10 from blockchain, 10 from local to local
|
||||
utxos = await self.account.get_utxos()
|
||||
txs = []
|
||||
for utxo in utxos:
|
||||
tx = await self.ledger.transaction_class.create(
|
||||
[self.ledger.transaction_class.input_class.spend(utxo)],
|
||||
[],
|
||||
[self.account], self.account
|
||||
)
|
||||
await self.broadcast(tx)
|
||||
txs.append(tx)
|
||||
await asyncio.wait([self.on_transaction_address(tx, address) for tx in txs], timeout=1)
|
||||
remote_status = await self.ledger.network.subscribe_address(address)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
# server history grows unordered
|
||||
txid = await self.blockchain.send_to_address(address, 1)
|
||||
await self.on_transaction_id(txid)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
|
||||
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
|
||||
# should be another test, but it would be too much to setup just for that and it affects sync
|
||||
self.assertIsNone(await self.ledger.network.retriable_call(self.ledger.network.get_transaction, '1'*64))
|
||||
@@ -60,7 +60,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
await account.receiving._generate_keys(8, 11)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[r['pubkey'].n for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
||||
)
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
self.assertEqual(len(new_keys), 8)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[r['pubkey'].n for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
|
||||
)
|
||||
|
||||
@@ -125,14 +125,18 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address('1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J'
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9xV7rhbg6M4yWrdTeLorz3Q1GrQb4aQzzGWboP3du7W7UUztzNTUrEYTnDfz7o'
|
||||
'ptBygDxXYRppyiuenJpoBTgYP2C26E1Ah5FEALM24CsWi'
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
@@ -275,12 +279,18 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
self.assertEqual(len(new_keys), 1)
|
||||
self.assertEqual(new_keys[0], account.public_key.address)
|
||||
records = await account.receiving.get_address_records()
|
||||
pubkey = records[0].pop('pubkey')
|
||||
self.assertEqual(records, [{
|
||||
'position': 0, 'chain': 0,
|
||||
'chain': 0,
|
||||
'account': account.public_key.address,
|
||||
'address': account.public_key.address,
|
||||
'history': None,
|
||||
'used_times': 0
|
||||
}])
|
||||
self.assertEqual(
|
||||
pubkey.extended_key_string(),
|
||||
account.public_key.extended_key_string()
|
||||
)
|
||||
|
||||
# case #1: no new addresses needed
|
||||
empty = await account.receiving.ensure_address_gap()
|
||||
@@ -333,14 +343,18 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], account.public_key.address)
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address(address[0])
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, address[0]
|
||||
)
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp'
|
||||
'5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna',
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import asyncio
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
@@ -194,13 +199,14 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': ledger_class.database_class(':memory:'),
|
||||
'headers': ledger_class.headers_class(':memory:'),
|
||||
})
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.ledger.db.close()
|
||||
|
||||
async def create_account(self):
|
||||
account = self.ledger.account_class.generate(self.ledger, Wallet())
|
||||
async def create_account(self, wallet=None):
|
||||
account = self.ledger.account_class.generate(self.ledger, wallet or self.wallet)
|
||||
await account.ensure_address_gap()
|
||||
return account
|
||||
|
||||
@@ -259,7 +265,8 @@ class TestQueries(AsyncioTestCase):
|
||||
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
|
||||
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
|
||||
for limit in range(variable_limit-2, variable_limit+2):
|
||||
txs = await self.ledger.get_transactions(limit=limit, order_by='height asc')
|
||||
txs = await self.ledger.get_transactions(
|
||||
accounts=self.wallet.accounts, limit=limit, order_by='height asc')
|
||||
self.assertEqual(len(txs), limit)
|
||||
inputs, outputs, last_tx = set(), set(), txs[0]
|
||||
for tx in txs[1:]:
|
||||
@@ -268,21 +275,24 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(len(tx.outputs), 1)
|
||||
last_tx = tx
|
||||
|
||||
|
||||
async def test_queries(self):
|
||||
self.assertEqual(0, await self.ledger.db.get_address_count())
|
||||
account1 = await self.create_account()
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count())
|
||||
account2 = await self.create_account()
|
||||
self.assertEqual(52, await self.ledger.db.get_address_count())
|
||||
wallet1 = Wallet()
|
||||
account1 = await self.create_account(wallet1)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1]))
|
||||
wallet2 = Wallet()
|
||||
account2 = await self.create_account(wallet2)
|
||||
account3 = await self.create_account(wallet2)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2]))
|
||||
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count())
|
||||
self.assertEqual([], await self.ledger.db.get_utxos())
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx1 = await self.create_tx_from_nothing(account1, 1)
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -290,20 +300,28 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2)
|
||||
tx2b = await self.create_tx_from_nothing(account3, 2)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -312,22 +330,24 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual([3, 2, 1], [tx.height for tx in txs])
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1])
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
|
||||
self.assertEqual([tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].inputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].outputs[0].is_my_account, True)
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account2])
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
|
||||
self.assertEqual([tx3.id, tx2.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
@@ -339,18 +359,189 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(tx.id, tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account1])
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, True)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account2])
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, True)
|
||||
|
||||
# height 0 sorted to the top with the rest in descending order
|
||||
tx4 = await self.create_tx_from_nothing(account1, 0)
|
||||
txos = await self.ledger.db.get_txos()
|
||||
self.assertEqual([0, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertEqual([tx4.id, tx2.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
self.assertEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
|
||||
|
||||
class TestUpgrade(AsyncioTestCase):
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.path = tempfile.mktemp()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
os.remove(self.path)
|
||||
|
||||
def get_version(self):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
versions = conn.execute('select version from version').fetchall()
|
||||
assert len(versions) == 1
|
||||
return versions[0][0]
|
||||
|
||||
def get_tables(self):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
|
||||
return [col[0] for col in conn.execute(sql).fetchall()]
|
||||
|
||||
def add_address(self, address):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
conn.execute("""
|
||||
INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth)
|
||||
VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0)
|
||||
""", (address,))
|
||||
|
||||
def get_addresses(self):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
sql = "SELECT address FROM account_address ORDER BY address;"
|
||||
return [col[0] for col in conn.execute(sql).fetchall()]
|
||||
|
||||
async def test_reset_on_version_change(self):
|
||||
self.ledger = ledger_class({
|
||||
'db': ledger_class.database_class(self.path),
|
||||
'headers': ledger_class.headers_class(':memory:'),
|
||||
})
|
||||
|
||||
# initial open, pre-version enabled db
|
||||
self.ledger.db.SCHEMA_VERSION = None
|
||||
self.assertEqual(self.get_tables(), [])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo'])
|
||||
self.assertEqual(self.get_addresses(), [])
|
||||
self.add_address('address1')
|
||||
await self.ledger.db.close()
|
||||
|
||||
# initial open after version enabled
|
||||
self.ledger.db.SCHEMA_VERSION = '1.0'
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
|
||||
self.add_address('address2')
|
||||
await self.ledger.db.close()
|
||||
|
||||
# nothing changes
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), ['address2'])
|
||||
await self.ledger.db.close()
|
||||
|
||||
# upgrade version, database reset
|
||||
self.ledger.db.SCHEMA_VERSION = '1.1'
|
||||
self.ledger.db.CREATE_TABLES_QUERY += """
|
||||
create table if not exists foo (bar text);
|
||||
"""
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.1')
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # all tables got reset
|
||||
await self.ledger.db.close()
|
||||
|
||||
|
||||
class TestSQLiteRace(AsyncioTestCase):
|
||||
max_misuse_attempts = 40000
|
||||
|
||||
def setup_db(self):
|
||||
self.db = sqlite3.connect(":memory:", isolation_level=None)
|
||||
self.db.executescript(
|
||||
"create table test1 (id text primary key not null, val text);\n" +
|
||||
"create table test2 (id text primary key not null, val text);\n" +
|
||||
"\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000))
|
||||
)
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
await self.loop.run_in_executor(self.executor, self.setup_db)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.loop.run_in_executor(self.executor, self.db.close)
|
||||
self.executor.shutdown()
|
||||
|
||||
async def test_binding_param_0_error(self):
|
||||
# test real param 0 binding errors
|
||||
|
||||
for supported_type in [str, int, bytes]:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 values (?, NULL)",
|
||||
[(supported_type(1), ), (supported_type(2), )]
|
||||
)
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.execute, "delete from test2 where id in (1, 2)"
|
||||
)
|
||||
for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]:
|
||||
try:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)",
|
||||
[(unsupported_type(1), ), (unsupported_type(2), )]
|
||||
)
|
||||
self.assertTrue(False)
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
|
||||
async def test_unhandled_sqlite_misuse(self):
|
||||
# test SQLITE_MISUSE being incorrectly raised as a param 0 binding error
|
||||
attempts = 0
|
||||
python_version = sys.version.split('\n')[0].rstrip(' ')
|
||||
|
||||
try:
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} "
|
||||
f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition")
|
||||
self.assertTrue(False, 'this test failing means either the sqlite race conditions '
|
||||
'have been fixed in cpython or the test max_attempts needs to be increased')
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE "
|
||||
f"after {attempts} attempts of the race condition")
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_fetchall_prevents_sqlite_misuse(self):
|
||||
# test that calling fetchall sufficiently avoids the race
|
||||
attempts = 0
|
||||
|
||||
def executemany_fetchall(query, params):
|
||||
self.db.executemany(query, params).fetchall()
|
||||
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
@@ -29,7 +29,7 @@ class MockNetwork:
|
||||
async def get_merkle(self, txid, height):
|
||||
return {'merkle': ['abcd01'], 'pos': 1}
|
||||
|
||||
async def get_transaction(self, tx_hash):
|
||||
async def get_transaction(self, tx_hash, _=None):
|
||||
self.get_transaction_called.append(tx_hash)
|
||||
return self.transaction[tx_hash]
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
return self.account.public_key.child(self.chain_number).child(index)
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
addresses = await self._query_addresses(order_by="position ASC")
|
||||
addresses = await self._query_addresses(order_by="n asc")
|
||||
max_gap = 0
|
||||
current_gap = 0
|
||||
for address in addresses:
|
||||
@@ -128,7 +128,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
|
||||
async def ensure_address_gap(self) -> List[str]:
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="position DESC")
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="n desc")
|
||||
|
||||
existing_gap = 0
|
||||
for address in addresses:
|
||||
@@ -140,7 +140,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
if existing_gap == self.gap:
|
||||
return []
|
||||
|
||||
start = addresses[0]['position']+1 if addresses else 0
|
||||
start = addresses[0]['pubkey'].n+1 if addresses else 0
|
||||
end = start + (self.gap - existing_gap)
|
||||
new_keys = await self._generate_keys(start, end-1)
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
@@ -149,15 +149,15 @@ class HierarchicalDeterministic(AddressManager):
|
||||
async def _generate_keys(self, start: int, end: int) -> List[str]:
|
||||
if not self.address_generator_lock.locked():
|
||||
raise RuntimeError('Should not be called outside of address_generator_lock.')
|
||||
keys = [(index, self.public_key.child(index)) for index in range(start, end+1)]
|
||||
keys = [self.public_key.child(index) for index in range(start, end+1)]
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, keys)
|
||||
return [key[1].address for key in keys]
|
||||
return [key.address for key in keys]
|
||||
|
||||
def get_address_records(self, only_usable: bool = False, **constraints):
|
||||
if only_usable:
|
||||
constraints['used_times__lt'] = self.maximum_uses_per_address
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = "used_times ASC, position ASC"
|
||||
constraints['order_by'] = "used_times asc, n asc"
|
||||
return self._query_addresses(**constraints)
|
||||
|
||||
|
||||
@@ -190,9 +190,7 @@ class SingleKey(AddressManager):
|
||||
async with self.address_generator_lock:
|
||||
exists = await self.get_address_records()
|
||||
if not exists:
|
||||
await self.account.ledger.db.add_keys(
|
||||
self.account, self.chain_number, [(0, self.public_key)]
|
||||
)
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key])
|
||||
new_keys = [self.public_key.address]
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
return new_keys
|
||||
@@ -417,16 +415,16 @@ class BaseAccount:
|
||||
}
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(account=self, **constraints)
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
return self.ledger.get_utxo_count(account=self, **constraints)
|
||||
return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
return self.ledger.get_transactions(account=self, **constraints)
|
||||
return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
return self.ledger.get_transaction_count(account=self, **constraints)
|
||||
return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
async def fund(self, to_account, amount=None, everything=False,
|
||||
outputs=1, broadcast=False, **constraints):
|
||||
|
||||
+188
-125
@@ -1,15 +1,14 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from asyncio import wrap_future
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
from torba.client.basetransaction import BaseTransaction, TXRefImmutable
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
from torba.client.bip32 import PubKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,43 +19,48 @@ class AIOSQLite:
|
||||
# has to be single threaded as there is no mapping of thread:connection
|
||||
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||
self.connection: sqlite3.Connection = None
|
||||
self._closing = False
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
def _connect():
|
||||
return sqlite3.connect(path, *args, **kwargs)
|
||||
db = cls()
|
||||
db.connection = await wrap_future(db.executor.submit(sqlite3.connect, path, *args, **kwargs))
|
||||
db.connection = await asyncio.get_event_loop().run_in_executor(db.executor, _connect)
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
def __close(conn):
|
||||
self.executor.submit(conn.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
conn = self.connection
|
||||
if not conn:
|
||||
if self._closing:
|
||||
return
|
||||
self._closing = True
|
||||
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
self.connection = None
|
||||
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
|
||||
|
||||
def executemany(self, sql: str, params: Iterable):
|
||||
def __executemany_in_a_transaction(conn: sqlite3.Connection, *args, **kwargs):
|
||||
return conn.executemany(*args, **kwargs)
|
||||
return self.run(__executemany_in_a_transaction, sql, params)
|
||||
params = params if params is not None else []
|
||||
# this fetchall is needed to prevent SQLITE_MISUSE
|
||||
return self.run(lambda conn: conn.executemany(sql, params).fetchall())
|
||||
|
||||
def executescript(self, script: str) -> Awaitable:
|
||||
return wrap_future(self.executor.submit(self.connection.executescript, script))
|
||||
return self.run(lambda conn: conn.executescript(script))
|
||||
|
||||
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
|
||||
parameters = parameters if parameters is not None else []
|
||||
def __fetchall(conn: sqlite3.Connection, *args, **kwargs):
|
||||
return conn.execute(*args, **kwargs).fetchall()
|
||||
return wrap_future(self.executor.submit(__fetchall, self.connection, sql, parameters))
|
||||
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
|
||||
|
||||
def execute_fetchone(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
|
||||
parameters = parameters if parameters is not None else []
|
||||
return self.run(lambda conn: conn.execute(sql, parameters).fetchone())
|
||||
|
||||
def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]:
|
||||
parameters = parameters if parameters is not None else []
|
||||
return self.run(lambda conn, sql, parameters: conn.execute(sql, parameters), sql, parameters)
|
||||
return self.run(lambda conn: conn.execute(sql, parameters))
|
||||
|
||||
def run(self, fun, *args, **kwargs) -> Awaitable:
|
||||
return wrap_future(self.executor.submit(self.__run_transaction, fun, *args, **kwargs))
|
||||
return asyncio.get_event_loop().run_in_executor(
|
||||
self.executor, lambda: self.__run_transaction(fun, *args, **kwargs)
|
||||
)
|
||||
|
||||
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
|
||||
self.connection.execute('begin')
|
||||
@@ -64,27 +68,28 @@ class AIOSQLite:
|
||||
result = fun(self.connection, *args, **kwargs) # type: ignore
|
||||
self.connection.commit()
|
||||
return result
|
||||
except (Exception, OSError): # as e:
|
||||
#log.exception('Error running transaction:', exc_info=e)
|
||||
except (Exception, OSError) as e:
|
||||
log.exception('Error running transaction:', exc_info=e)
|
||||
self.connection.rollback()
|
||||
log.warning("rolled back")
|
||||
raise
|
||||
|
||||
def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable:
|
||||
return wrap_future(
|
||||
self.executor.submit(self.__run_transaction_with_foreign_keys_disabled, fun, *args, **kwargs)
|
||||
return asyncio.get_event_loop().run_in_executor(
|
||||
self.executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
|
||||
)
|
||||
|
||||
def __run_transaction_with_foreign_keys_disabled(self,
|
||||
fun: Callable[[sqlite3.Connection, Any, Any], Any],
|
||||
*args, **kwargs):
|
||||
args, kwargs):
|
||||
foreign_keys_enabled, = self.connection.execute("pragma foreign_keys").fetchone()
|
||||
if not foreign_keys_enabled:
|
||||
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
|
||||
try:
|
||||
self.connection.execute('pragma foreign_keys=off')
|
||||
self.connection.execute('pragma foreign_keys=off').fetchone()
|
||||
return self.__run_transaction(fun, *args, **kwargs)
|
||||
finally:
|
||||
self.connection.execute('pragma foreign_keys=on')
|
||||
self.connection.execute('pragma foreign_keys=on').fetchone()
|
||||
|
||||
|
||||
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
@@ -154,15 +159,14 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
return joiner.join(sql) if sql else '', values
|
||||
|
||||
|
||||
def query(select, **constraints):
|
||||
def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
|
||||
sql = [select]
|
||||
limit = constraints.pop('limit', None)
|
||||
offset = constraints.pop('offset', None)
|
||||
order_by = constraints.pop('order_by', None)
|
||||
|
||||
constraints.pop('my_accounts', None)
|
||||
accounts = constraints.pop('accounts', None)
|
||||
if accounts is not None:
|
||||
accounts = constraints.pop('accounts', [])
|
||||
if accounts:
|
||||
constraints['account__in'] = [a.public_key.address for a in accounts]
|
||||
|
||||
where, values = constraints_to_sql(constraints)
|
||||
@@ -210,9 +214,16 @@ def rows_to_dict(rows, fields):
|
||||
|
||||
class SQLiteMixin:
|
||||
|
||||
SCHEMA_VERSION: Optional[str] = None
|
||||
CREATE_TABLES_QUERY: str
|
||||
MAX_QUERY_VARIABLES = 900
|
||||
|
||||
CREATE_VERSION_TABLE = """
|
||||
create table if not exists version (
|
||||
version text
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self._db_path = path
|
||||
self.db: AIOSQLite = None
|
||||
@@ -220,23 +231,40 @@ class SQLiteMixin:
|
||||
|
||||
async def open(self):
|
||||
log.info("connecting to database: %s", self._db_path)
|
||||
self.db = await AIOSQLite.connect(self._db_path)
|
||||
self.db = await AIOSQLite.connect(self._db_path, isolation_level=None)
|
||||
if self.SCHEMA_VERSION:
|
||||
tables = [t[0] for t in await self.db.execute_fetchall(
|
||||
"SELECT name FROM sqlite_master WHERE type='table';"
|
||||
)]
|
||||
if tables:
|
||||
if 'version' in tables:
|
||||
version = await self.db.execute_fetchone("SELECT version FROM version LIMIT 1;")
|
||||
if version == (self.SCHEMA_VERSION,):
|
||||
return
|
||||
await self.db.executescript('\n'.join(
|
||||
f"DROP TABLE {table};" for table in tables
|
||||
))
|
||||
await self.db.execute(self.CREATE_VERSION_TABLE)
|
||||
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
|
||||
await self.db.executescript(self.CREATE_TABLES_QUERY)
|
||||
|
||||
async def close(self):
|
||||
await self.db.close()
|
||||
|
||||
@staticmethod
|
||||
def _insert_sql(table: str, data: dict, ignore_duplicate: bool = False) -> Tuple[str, List]:
|
||||
def _insert_sql(table: str, data: dict, ignore_duplicate: bool = False,
|
||||
replace: bool = False) -> Tuple[str, List]:
|
||||
columns, values = [], []
|
||||
for column, value in data.items():
|
||||
columns.append(column)
|
||||
values.append(value)
|
||||
or_ignore = ""
|
||||
policy = ""
|
||||
if ignore_duplicate:
|
||||
or_ignore = " OR IGNORE"
|
||||
policy = " OR IGNORE"
|
||||
if replace:
|
||||
policy = " OR REPLACE"
|
||||
sql = "INSERT{} INTO {} ({}) VALUES ({})".format(
|
||||
or_ignore, table, ', '.join(columns), ', '.join(['?'] * len(values))
|
||||
policy, table, ', '.join(columns), ', '.join(['?'] * len(values))
|
||||
)
|
||||
return sql, values
|
||||
|
||||
@@ -256,24 +284,33 @@ class SQLiteMixin:
|
||||
|
||||
class BaseDatabase(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.1"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
"""
|
||||
|
||||
CREATE_ACCOUNT_TABLE = """
|
||||
create table if not exists account_address (
|
||||
account text not null,
|
||||
address text not null,
|
||||
chain integer not null,
|
||||
pubkey blob not null,
|
||||
chain_code blob not null,
|
||||
n integer not null,
|
||||
depth integer not null,
|
||||
primary key (account, address)
|
||||
);
|
||||
create index if not exists address_account_idx on account_address (address, account);
|
||||
"""
|
||||
|
||||
CREATE_PUBKEY_ADDRESS_TABLE = """
|
||||
create table if not exists pubkey_address (
|
||||
address text primary key,
|
||||
account text not null,
|
||||
chain integer not null,
|
||||
position integer not null,
|
||||
pubkey blob not null,
|
||||
history text,
|
||||
used_times integer not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_PUBKEY_ADDRESS_INDEX = """
|
||||
create index if not exists pubkey_address_account_idx on pubkey_address (account);
|
||||
"""
|
||||
|
||||
CREATE_TX_TABLE = """
|
||||
create table if not exists tx (
|
||||
@@ -295,8 +332,6 @@ class BaseDatabase(SQLiteMixin):
|
||||
script blob not null,
|
||||
is_reserved boolean not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_TXO_INDEX = """
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
"""
|
||||
|
||||
@@ -306,21 +341,17 @@ class BaseDatabase(SQLiteMixin):
|
||||
txoid text references txo,
|
||||
address text references pubkey_address
|
||||
);
|
||||
"""
|
||||
CREATE_TXI_INDEX = """
|
||||
create index if not exists txi_address_idx on txi (address);
|
||||
create index if not exists txi_txoid_idx on txi (txoid);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_TX_TABLE +
|
||||
CREATE_ACCOUNT_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TX_TABLE +
|
||||
CREATE_TXO_TABLE +
|
||||
CREATE_TXO_INDEX +
|
||||
CREATE_TXI_TABLE +
|
||||
CREATE_TXI_INDEX
|
||||
CREATE_TXI_TABLE
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -334,49 +365,59 @@ class BaseDatabase(SQLiteMixin):
|
||||
'script': sqlite3.Binary(txo.script.source)
|
||||
}
|
||||
|
||||
async def insert_transaction(self, tx):
|
||||
await self.db.execute(*self._insert_sql('tx', {
|
||||
@staticmethod
|
||||
def tx_to_row(tx):
|
||||
return {
|
||||
'txid': tx.id,
|
||||
'raw': sqlite3.Binary(tx.raw),
|
||||
'height': tx.height,
|
||||
'position': tx.position,
|
||||
'is_verified': tx.is_verified
|
||||
}))
|
||||
}
|
||||
|
||||
async def insert_transaction(self, tx):
|
||||
await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx)))
|
||||
|
||||
async def update_transaction(self, tx):
|
||||
await self.db.execute(*self._update_sql("tx", {
|
||||
await self.db.execute_fetchall(*self._update_sql("tx", {
|
||||
'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
|
||||
}, 'txid = ?', (tx.id,)))
|
||||
|
||||
def _transaction_io(self, conn: sqlite3.Connection, tx: BaseTransaction, address, txhash, history):
|
||||
conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True))
|
||||
|
||||
for txo in tx.outputs:
|
||||
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
|
||||
)).fetchall()
|
||||
elif txo.script.is_pay_script_hash:
|
||||
# TODO: implement script hash payments
|
||||
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
txo = txi.txo_ref.txo
|
||||
if txo.get_address(self.ledger) == address:
|
||||
conn.execute(*self._insert_sql("txi", {
|
||||
'txid': tx.id,
|
||||
'txoid': txo.id,
|
||||
'address': address,
|
||||
}, ignore_duplicate=True)).fetchall()
|
||||
|
||||
conn.execute(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
(history, history.count(':') // 2, address)
|
||||
)
|
||||
|
||||
def save_transaction_io(self, tx: BaseTransaction, address, txhash, history):
|
||||
return self.db.run(self._transaction_io, tx, address, txhash, history)
|
||||
|
||||
def _transaction(conn: sqlite3.Connection, tx: BaseTransaction, address, txhash, history):
|
||||
|
||||
for txo in tx.outputs:
|
||||
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
|
||||
))
|
||||
elif txo.script.is_pay_script_hash:
|
||||
# TODO: implement script hash payments
|
||||
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
txo = txi.txo_ref.txo
|
||||
if txo.get_address(self.ledger) == address:
|
||||
conn.execute(*self._insert_sql("txi", {
|
||||
'txid': tx.id,
|
||||
'txoid': txo.id,
|
||||
'address': address,
|
||||
}, ignore_duplicate=True))
|
||||
|
||||
conn.execute(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
(history, history.count(':')//2, address)
|
||||
)
|
||||
|
||||
return self.db.run(_transaction, tx, address, txhash, history)
|
||||
def save_transaction_io_batch(self, txs: Iterable[BaseTransaction], address, txhash, history):
|
||||
def __many(conn):
|
||||
for tx in txs:
|
||||
self._transaction_io(conn, tx, address, txhash, history)
|
||||
return self.db.run(__many)
|
||||
|
||||
async def reserve_outputs(self, txos, is_reserved=True):
|
||||
txoids = ((is_reserved, txo.id) for txo in txos)
|
||||
@@ -393,25 +434,22 @@ class BaseDatabase(SQLiteMixin):
|
||||
|
||||
async def select_transactions(self, cols, accounts=None, **constraints):
|
||||
if not set(constraints) & {'txid', 'txid__in'}:
|
||||
assert accounts is not None, "'accounts' argument required when no 'txid' constraint"
|
||||
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
|
||||
constraints.update({
|
||||
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
|
||||
})
|
||||
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
|
||||
where = f" WHERE account_address.account IN ({account_values})"
|
||||
constraints['txid__in'] = f"""
|
||||
SELECT txo.txid FROM txo
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
|
||||
UNION
|
||||
SELECT txi.txid FROM txi
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
|
||||
"""
|
||||
return await self.db.execute_fetchall(
|
||||
*query("SELECT {} FROM tx".format(cols), **constraints)
|
||||
)
|
||||
|
||||
async def get_transactions(self, **constraints):
|
||||
accounts = constraints.get('accounts', None)
|
||||
|
||||
async def get_transactions(self, wallet=None, **constraints):
|
||||
tx_rows = await self.select_transactions(
|
||||
'txid, raw, height, position, is_verified',
|
||||
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
|
||||
@@ -436,7 +474,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
annotated_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
my_accounts=accounts,
|
||||
wallet=wallet,
|
||||
txid__in=txids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -446,7 +484,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
referenced_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
my_accounts=accounts,
|
||||
wallet=wallet,
|
||||
txoid__in=txi_txoids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -466,6 +504,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs
|
||||
|
||||
async def get_transaction_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -478,23 +517,24 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs[0]
|
||||
|
||||
async def select_txos(self, cols, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM txo"
|
||||
" JOIN pubkey_address USING (address)"
|
||||
" JOIN tx USING (txid)".format(cols), **constraints
|
||||
))
|
||||
sql = "SELECT {} FROM txo JOIN tx USING (txid)"
|
||||
if 'accounts' in constraints:
|
||||
sql += " JOIN account_address USING (address)"
|
||||
return await self.db.execute_fetchall(*query(sql.format(cols), **constraints))
|
||||
|
||||
async def get_txos(self, my_accounts=None, no_tx=False, **constraints):
|
||||
my_accounts = [
|
||||
(a.public_key.address if isinstance(a, BaseAccount) else a)
|
||||
for a in (my_accounts or constraints.get('accounts', []))
|
||||
]
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints):
|
||||
my_accounts = set(a.public_key.address for a in wallet.accounts) if wallet else set()
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"]
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
rows = await self.select_txos(
|
||||
"tx.txid, raw, tx.height, tx.position, tx.is_verified, "
|
||||
"txo.position, chain, account, amount, script",
|
||||
"""
|
||||
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
|
||||
select group_concat(account||"|"||chain) from account_address
|
||||
where account_address.address=txo.address
|
||||
)
|
||||
""",
|
||||
**constraints
|
||||
)
|
||||
txos = []
|
||||
@@ -503,8 +543,8 @@ class BaseDatabase(SQLiteMixin):
|
||||
for row in rows:
|
||||
if no_tx:
|
||||
txo = output_class(
|
||||
amount=row[8],
|
||||
script=output_class.script_class(row[9]),
|
||||
amount=row[6],
|
||||
script=output_class.script_class(row[7]),
|
||||
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
|
||||
position=row[5]
|
||||
)
|
||||
@@ -514,12 +554,18 @@ class BaseDatabase(SQLiteMixin):
|
||||
row[1], height=row[2], position=row[3], is_verified=row[4]
|
||||
)
|
||||
txo = txs[row[0]].outputs[row[5]]
|
||||
txo.is_change = row[6] == 1
|
||||
txo.is_my_account = row[7] in my_accounts
|
||||
row_accounts = {k: v for k, v in (a.split('|') for a in row[8].split(','))}
|
||||
account_match = set(row_accounts) & my_accounts
|
||||
if account_match:
|
||||
txo.is_my_account = True
|
||||
txo.is_change = row_accounts[account_match.pop()] == '1'
|
||||
else:
|
||||
txo.is_change = txo.is_my_account = False
|
||||
txos.append(txo)
|
||||
return txos
|
||||
|
||||
async def get_txo_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -539,43 +585,60 @@ class BaseDatabase(SQLiteMixin):
|
||||
self.constrain_utxo(constraints)
|
||||
return self.get_txo_count(**constraints)
|
||||
|
||||
async def get_balance(self, **constraints):
|
||||
async def get_balance(self, wallet=None, accounts=None, **constraints):
|
||||
assert wallet or accounts, \
|
||||
"'wallet' or 'accounts' constraints required to calculate balance"
|
||||
constraints['accounts'] = accounts or wallet.accounts
|
||||
self.constrain_utxo(constraints)
|
||||
balance = await self.select_txos('SUM(amount)', **constraints)
|
||||
return balance[0][0] or 0
|
||||
|
||||
async def select_addresses(self, cols, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM pubkey_address".format(cols), **constraints
|
||||
"SELECT {} FROM pubkey_address JOIN account_address USING (address)".format(cols),
|
||||
**constraints
|
||||
))
|
||||
|
||||
async def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'),
|
||||
**constraints):
|
||||
addresses = await self.select_addresses(', '.join(cols), **constraints)
|
||||
return rows_to_dict(addresses, cols)
|
||||
async def get_addresses(self, cols=None, **constraints):
|
||||
cols = cols or (
|
||||
'address', 'account', 'chain', 'history', 'used_times',
|
||||
'pubkey', 'chain_code', 'n', 'depth'
|
||||
)
|
||||
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
|
||||
for address in addresses:
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
return addresses
|
||||
|
||||
async def get_address_count(self, **constraints):
|
||||
count = await self.select_addresses('count(*)', **constraints)
|
||||
return count[0][0]
|
||||
|
||||
async def get_address(self, **constraints):
|
||||
addresses = await self.get_addresses(
|
||||
cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
|
||||
limit=1, **constraints
|
||||
)
|
||||
addresses = await self.get_addresses(limit=1, **constraints)
|
||||
if addresses:
|
||||
return addresses[0]
|
||||
|
||||
async def add_keys(self, account, chain, keys):
|
||||
async def add_keys(self, account, chain, pubkeys):
|
||||
await self.db.executemany(
|
||||
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
|
||||
((pubkey.address, account.public_key.address, chain,
|
||||
position, sqlite3.Binary(pubkey.pubkey_bytes))
|
||||
for position, pubkey in keys)
|
||||
"insert or ignore into account_address "
|
||||
"(account, address, chain, pubkey, chain_code, n, depth) values "
|
||||
"(?, ?, ?, ?, ?, ?, ?)", ((
|
||||
account.id, k.address, chain,
|
||||
sqlite3.Binary(k.pubkey_bytes),
|
||||
sqlite3.Binary(k.chain_code),
|
||||
k.n, k.depth
|
||||
) for k in pubkeys)
|
||||
)
|
||||
await self.db.executemany(
|
||||
"insert or ignore into pubkey_address (address) values (?)",
|
||||
((pubkey.address,) for pubkey in pubkeys)
|
||||
)
|
||||
|
||||
async def _set_address_history(self, address, history):
|
||||
await self.db.execute(
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
(history, history.count(':')//2, address)
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from operator import itemgetter
|
||||
from collections import namedtuple
|
||||
|
||||
import pylru
|
||||
from torba.client.basetransaction import BaseTransaction
|
||||
from torba.tasks import TaskGroup
|
||||
from torba.client import baseaccount, basenetwork, basetransaction
|
||||
from torba.client.basedatabase import BaseDatabase
|
||||
@@ -18,6 +19,7 @@ from torba.client.coinselection import CoinSelector
|
||||
from torba.client.constants import COIN, NULL_HASH32
|
||||
from torba.stream import StreamController
|
||||
from torba.client.hash import hash160, double_sha256, sha256, Base58
|
||||
from torba.client.bip32 import PubKey, PrivateKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -142,6 +144,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
self._address_update_locks: Dict[str, asyncio.Lock] = {}
|
||||
|
||||
self.coin_selection_strategy = None
|
||||
self._known_addresses_out_of_sync = set()
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
@@ -158,7 +161,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
decoded = Base58.decode(address)
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
@@ -176,27 +179,29 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def add_account(self, account: baseaccount.BaseAccount):
|
||||
self.accounts.append(account)
|
||||
|
||||
async def _get_account_and_address_info_for_address(self, address):
|
||||
match = await self.db.get_address(address=address)
|
||||
async def _get_account_and_address_info_for_address(self, wallet, address):
|
||||
match = await self.db.get_address(accounts=wallet.accounts, address=address)
|
||||
if match:
|
||||
for account in self.accounts:
|
||||
for account in wallet.accounts:
|
||||
if match['account'] == account.public_key.address:
|
||||
return account, match
|
||||
|
||||
async def get_private_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
account, address_info = match
|
||||
return account.get_private_key(address_info['chain'], address_info['position'])
|
||||
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
|
||||
return None
|
||||
|
||||
async def get_public_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
account, address_info = match
|
||||
return account.get_public_key(address_info['chain'], address_info['position'])
|
||||
_, address_info = match
|
||||
return address_info['pubkey']
|
||||
return None
|
||||
|
||||
async def get_account_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
async def get_account_for_address(self, wallet, address):
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
return match[0]
|
||||
|
||||
@@ -208,6 +213,12 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
estimators.append(utxo.get_estimator(self))
|
||||
return estimators
|
||||
|
||||
async def get_addresses(self, **constraints):
|
||||
return await self.db.get_addresses(**constraints)
|
||||
|
||||
def get_address_count(self, **constraints):
|
||||
return self.db.get_address_count(**constraints)
|
||||
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts):
|
||||
async with self._utxo_reservation_lock:
|
||||
txos = await self.get_effective_amount_estimators(funding_accounts)
|
||||
@@ -227,32 +238,22 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def release_tx(self, tx):
|
||||
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
|
||||
|
||||
def constraint_account_or_all(self, constraints):
|
||||
account = constraints.pop('account', None)
|
||||
if account:
|
||||
constraints['accounts'] = [account]
|
||||
else:
|
||||
constraints['accounts'] = self.accounts
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxos(**constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxo_count(**constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transactions(**constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
async def get_local_status_and_history(self, address):
|
||||
address_details = await self.db.get_address(address=address)
|
||||
history = address_details['history'] or ''
|
||||
async def get_local_status_and_history(self, address, history=None):
|
||||
if not history:
|
||||
address_details = await self.db.get_address(address=address)
|
||||
history = address_details['history'] or ''
|
||||
parts = history.split(':')[:-1]
|
||||
return (
|
||||
hexlify(sha256(history.encode())).decode() if history else None,
|
||||
@@ -284,7 +285,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
await self.join_network()
|
||||
self.network.on_connected.listen(self.join_network)
|
||||
|
||||
async def join_network(self, *args):
|
||||
async def join_network(self, *_):
|
||||
log.info("Subscribing and updating accounts.")
|
||||
async with self._header_processing_lock:
|
||||
await self.update_headers()
|
||||
@@ -411,24 +412,31 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
address_manager: baseaccount.AddressManager = None):
|
||||
|
||||
async with self._address_update_locks.setdefault(address, asyncio.Lock()):
|
||||
self._known_addresses_out_of_sync.discard(address)
|
||||
|
||||
local_status, local_history = await self.get_local_status_and_history(address)
|
||||
|
||||
if local_status == remote_status:
|
||||
return
|
||||
return True
|
||||
|
||||
remote_history = await self.network.retriable_call(self.network.get_history, address)
|
||||
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
||||
we_need = set(remote_history) - set(local_history)
|
||||
if not we_need:
|
||||
return True
|
||||
|
||||
cache_tasks = []
|
||||
cache_tasks: List[asyncio.Future[BaseTransaction]] = []
|
||||
synced_history = StringIO()
|
||||
for i, (txid, remote_height) in enumerate(map(itemgetter('tx_hash', 'height'), remote_history)):
|
||||
if i < len(local_history) and local_history[i] == (txid, remote_height):
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
|
||||
synced_history.write(f'{txid}:{remote_height}:')
|
||||
else:
|
||||
check_local = (txid, remote_height) not in we_need
|
||||
cache_tasks.append(asyncio.ensure_future(
|
||||
self.cache_transaction(txid, remote_height)
|
||||
self.cache_transaction(txid, remote_height, check_local=check_local)
|
||||
))
|
||||
|
||||
synced_txs = []
|
||||
for task in cache_tasks:
|
||||
tx = await task
|
||||
|
||||
@@ -457,12 +465,15 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
txi.txo_ref = referenced_txo.ref
|
||||
|
||||
synced_history.write(f'{tx.id}:{tx.height}:')
|
||||
synced_txs.append(tx)
|
||||
|
||||
await self.db.save_transaction_io(
|
||||
tx, address, self.address_to_hash160(address), synced_history.getvalue()
|
||||
)
|
||||
|
||||
await self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
|
||||
)
|
||||
await asyncio.wait([
|
||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
for tx in synced_txs
|
||||
])
|
||||
|
||||
if address_manager is None:
|
||||
address_manager = await self.get_address_manager_for_address(address)
|
||||
@@ -470,7 +481,23 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
if address_manager is not None:
|
||||
await address_manager.ensure_address_gap()
|
||||
|
||||
async def cache_transaction(self, txid, remote_height):
|
||||
local_status, local_history = \
|
||||
await self.get_local_status_and_history(address, synced_history.getvalue())
|
||||
if local_status != remote_status:
|
||||
if local_history == remote_history:
|
||||
return True
|
||||
log.warning(
|
||||
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
|
||||
remote_status, len(remote_history), local_status, len(local_history)
|
||||
)
|
||||
log.warning("local: %s", local_history)
|
||||
log.warning("remote: %s", remote_history)
|
||||
self._known_addresses_out_of_sync.add(address)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
async def cache_transaction(self, txid, remote_height, check_local=True):
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = self._tx_cache[txid] = TransactionCacheItem()
|
||||
@@ -483,28 +510,21 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
tx = cache_item.tx
|
||||
|
||||
if tx is None:
|
||||
if tx is None and check_local:
|
||||
# check local db
|
||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
||||
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw = await self.network.retriable_call(self.network.get_transaction, txid)
|
||||
_raw = await self.network.retriable_call(self.network.get_transaction, txid, remote_height)
|
||||
if _raw:
|
||||
tx = self.transaction_class(unhexlify(_raw))
|
||||
await self.maybe_verify_transaction(tx, remote_height)
|
||||
await self.db.insert_transaction(tx)
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
return tx
|
||||
|
||||
if tx is None:
|
||||
raise ValueError(f'Transaction {txid} was not in database and not on network.')
|
||||
|
||||
if remote_height > 0 and not tx.is_verified:
|
||||
# tx from cache / db is not up-to-date
|
||||
await self.maybe_verify_transaction(tx, remote_height)
|
||||
await self.db.update_transaction(tx)
|
||||
|
||||
await self.maybe_verify_transaction(tx, remote_height)
|
||||
return tx
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height):
|
||||
@@ -538,7 +558,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
addresses.add(
|
||||
self.hash160_to_address(txo.script.values['pubkey_hash'])
|
||||
)
|
||||
records = await self.db.get_addresses(cols=('address',), address__in=addresses)
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
_, pending = await asyncio.wait([
|
||||
self.on_transaction.where(partial(
|
||||
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Type, MutableSequence, MutableMapping
|
||||
from typing import Type, MutableSequence, MutableMapping, Optional
|
||||
|
||||
from torba.client.baseledger import BaseLedger, LedgerRegistry
|
||||
from torba.client.wallet import Wallet, WalletStorage
|
||||
@@ -41,16 +41,6 @@ class BaseWalletManager:
|
||||
self.wallets.append(wallet)
|
||||
return wallet
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@property
|
||||
def default_wallet(self):
|
||||
for wallet in self.wallets:
|
||||
@@ -78,3 +68,14 @@ class BaseWalletManager:
|
||||
l.stop() for l in self.ledgers.values()
|
||||
))
|
||||
self.running = False
|
||||
|
||||
def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet:
|
||||
if wallet_id is None:
|
||||
return self.default_wallet
|
||||
return self.get_wallet_or_error(wallet_id)
|
||||
|
||||
def get_wallet_or_error(self, wallet_id: str) -> Wallet:
|
||||
for wallet in self.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
return wallet
|
||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
||||
@@ -4,7 +4,7 @@ from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
from time import perf_counter
|
||||
|
||||
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
|
||||
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
|
||||
from torba import __version__
|
||||
from torba.stream import StreamController
|
||||
@@ -29,10 +29,12 @@ class ClientSession(BaseClientSession):
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
# one request per second of timeout, conservative default
|
||||
self._semaphore = asyncio.Semaphore(self.timeout * 2)
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
return not self.is_closing() and self._can_send.is_set() and self.response_time is not None
|
||||
return not self.is_closing() and self.response_time is not None
|
||||
|
||||
@property
|
||||
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
|
||||
@@ -53,27 +55,33 @@ class ClientSession(BaseClientSession):
|
||||
self._response_samples += 1
|
||||
return result
|
||||
|
||||
async def send_request(self, method, args=(), timeout=None):
|
||||
log.info("send %s to %s:%i", method, *self.server)
|
||||
timeout = timeout or self.timeout
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
async with self._semaphore:
|
||||
return await self._send_request(method, args)
|
||||
|
||||
async def _send_request(self, method, args=()):
|
||||
log.debug("send %s to %s:%i", method, *self.server)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
reply = await self.send_timed_server_version_request(args, timeout)
|
||||
reply = await self.send_timed_server_version_request(args, self.timeout)
|
||||
else:
|
||||
reply = await asyncio.wait_for(
|
||||
super().send_request(method, args), timeout=timeout
|
||||
super().send_request(method, args), timeout=self.timeout
|
||||
)
|
||||
log.info("got reply for %s from %s:%i", method, *self.server)
|
||||
log.debug("got reply for %s from %s:%i", method, *self.server)
|
||||
return reply
|
||||
except RPCError as e:
|
||||
except (RPCError, ProtocolError) as e:
|
||||
if str(e).find('.*no such .*transaction.*'):
|
||||
# shouldnt the server return none instead?
|
||||
return None
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
raise e
|
||||
except ConnectionError:
|
||||
log.warning("connection to %s:%i lost", *self.server)
|
||||
self.synchronous_close()
|
||||
raise asyncio.CancelledError(f"connection to {self.server[0]}:{self.server[1]} lost")
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise
|
||||
@@ -87,7 +95,7 @@ class ClientSession(BaseClientSession):
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 0.1
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
@@ -109,7 +117,9 @@ class ClientSession(BaseClientSession):
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required='1.2', timeout=3):
|
||||
return await self.send_request('server.version', [__version__, required], timeout)
|
||||
return await asyncio.wait_for(
|
||||
self.send_request('server.version', [__version__, required]), timeout=timeout
|
||||
)
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
@@ -137,6 +147,7 @@ class BaseNetwork:
|
||||
self.config = ledger.config
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
|
||||
@@ -154,53 +165,42 @@ class BaseNetwork:
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
}
|
||||
|
||||
async def switch_to_fastest(self):
|
||||
try:
|
||||
client = await asyncio.wait_for(self.session_pool.wait_for_fastest_session(), 30)
|
||||
except asyncio.TimeoutError:
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
self.client = None
|
||||
for session in self.session_pool.sessions:
|
||||
session.synchronous_close()
|
||||
log.warning("not connected to any wallet servers")
|
||||
return
|
||||
if not self.client or client.server_address_and_port != self.client.server_address_and_port:
|
||||
current_client = self.client
|
||||
self.client = client
|
||||
async def switch_forever(self):
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
self._on_connected_controller.add(True)
|
||||
try:
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
if current_client:
|
||||
await current_client.close()
|
||||
log.info("Closed connection to %s:%i", *current_client.server)
|
||||
except asyncio.TimeoutError:
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
self.client = current_client
|
||||
return
|
||||
self._on_connected_controller.add(True)
|
||||
await asyncio.sleep(30)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
while self.running:
|
||||
await self.switch_to_fastest()
|
||||
|
||||
async def stop(self):
|
||||
self.running = False
|
||||
self.session_pool.stop()
|
||||
if self.running:
|
||||
self.running = False
|
||||
self._switch_task.cancel()
|
||||
self.session_pool.stop()
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args, session=None):
|
||||
session = session or self.session_pool.fastest_session
|
||||
if session:
|
||||
def rpc(self, list_or_method, args, restricted=True):
|
||||
session = self.client if restricted else self.session_pool.fastest_session
|
||||
if session and not session.is_closing():
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
@@ -208,42 +208,58 @@ class BaseNetwork:
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
if not self.is_connected and self.running:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
await self.on_connected.first
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
def get_history(self, address):
|
||||
return self.rpc('blockchain.address.get_history', [address])
|
||||
def get_transaction(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = not known_height or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
||||
|
||||
def get_transaction(self, tx_hash):
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash])
|
||||
|
||||
def get_transaction_height(self, tx_hash):
|
||||
return self.rpc('blockchain.transaction.get_height', [tx_hash])
|
||||
def get_transaction_height(self, tx_hash, known_height=None):
|
||||
restricted = not known_height or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
|
||||
|
||||
def get_merkle(self, tx_hash, height):
|
||||
return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height])
|
||||
restricted = 0 > height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height], restricted)
|
||||
|
||||
def get_headers(self, height, count=10000):
|
||||
return self.rpc('blockchain.block.headers', [height, count])
|
||||
|
||||
# --- Subscribes and broadcasts are always aimed towards the master client directly
|
||||
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
|
||||
def get_history(self, address):
|
||||
return self.rpc('blockchain.address.get_history', [address], True)
|
||||
|
||||
def broadcast(self, raw_transaction):
|
||||
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], session=self.client)
|
||||
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], True)
|
||||
|
||||
def subscribe_headers(self):
|
||||
return self.rpc('blockchain.headers.subscribe', [True], session=self.client)
|
||||
return self.rpc('blockchain.headers.subscribe', [True], True)
|
||||
|
||||
def subscribe_address(self, address):
|
||||
return self.rpc('blockchain.address.subscribe', [address], session=self.client)
|
||||
async def subscribe_address(self, address):
|
||||
try:
|
||||
return await self.rpc('blockchain.address.subscribe', [address], True)
|
||||
except asyncio.TimeoutError:
|
||||
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
|
||||
if self.client:
|
||||
self.client.abort()
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
def get_server_features(self):
|
||||
return self.rpc('server.features', (), restricted=True)
|
||||
|
||||
|
||||
class SessionPool:
|
||||
@@ -260,11 +276,11 @@ class SessionPool:
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return [session for session in self.sessions if session.available]
|
||||
return (session for session in self.sessions if session.available)
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.available_sessions:
|
||||
if not self.online:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
@@ -315,8 +331,9 @@ class SessionPool:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
for task in self.sessions.values():
|
||||
for session, task in self.sessions.items():
|
||||
task.cancel()
|
||||
session.synchronous_close()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
import typing
|
||||
from typing import List, Iterable, Optional
|
||||
from typing import List, Iterable, Optional, Tuple
|
||||
from binascii import hexlify
|
||||
|
||||
from torba.client.basescript import BaseInputScript, BaseOutputScript
|
||||
@@ -12,7 +12,7 @@ from torba.client.util import ReadOnlyList
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from torba.client import baseledger
|
||||
from torba.client import baseledger, wallet as basewallet
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
@@ -431,21 +431,32 @@ class BaseTransaction:
|
||||
self.locktime = stream.read_uint32()
|
||||
|
||||
@classmethod
|
||||
def ensure_all_have_same_ledger(cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> 'baseledger.BaseLedger':
|
||||
ledger = None
|
||||
def ensure_all_have_same_ledger_and_wallet(
|
||||
cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> Tuple['baseledger.BaseLedger', 'basewallet.Wallet']:
|
||||
ledger = wallet = None
|
||||
for account in funding_accounts:
|
||||
if ledger is None:
|
||||
ledger = account.ledger
|
||||
wallet = account.wallet
|
||||
if ledger != account.ledger:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be on the same ledger.'
|
||||
)
|
||||
if change_account is not None and change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if wallet != account.wallet:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be from the same wallet.'
|
||||
)
|
||||
if change_account is not None:
|
||||
if change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if change_account.wallet != wallet:
|
||||
raise ValueError('Change account must use same wallet as funding accounts.')
|
||||
if ledger is None:
|
||||
raise ValueError('No ledger found.')
|
||||
return ledger
|
||||
if wallet is None:
|
||||
raise ValueError('No wallet found.')
|
||||
return ledger, wallet
|
||||
|
||||
@classmethod
|
||||
async def create(cls, inputs: Iterable[BaseInput], outputs: Iterable[BaseOutput],
|
||||
@@ -458,7 +469,7 @@ class BaseTransaction:
|
||||
.add_inputs(inputs) \
|
||||
.add_outputs(outputs)
|
||||
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
|
||||
# value of the outputs plus associated fees
|
||||
cost = (
|
||||
@@ -524,14 +535,15 @@ class BaseTransaction:
|
||||
return hash_type
|
||||
|
||||
async def sign(self, funding_accounts: Iterable[BaseAccount]):
|
||||
ledger = self.ensure_all_have_same_ledger(funding_accounts)
|
||||
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
|
||||
for i, txi in enumerate(self._inputs):
|
||||
assert txi.script is not None
|
||||
assert txi.txo_ref.txo is not None
|
||||
txo_script = txi.txo_ref.txo.script
|
||||
if txo_script.is_pay_pubkey_hash:
|
||||
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
|
||||
private_key = await ledger.get_private_key_for_address(address)
|
||||
private_key = await ledger.get_private_key_for_address(wallet, address)
|
||||
assert private_key is not None, 'Cannot find private key for signing output.'
|
||||
tx = self._serialize_for_signature(i)
|
||||
txi.script.values['signature'] = \
|
||||
private_key.sign(tx) + bytes((self.signature_hash_type(1),))
|
||||
|
||||
@@ -3,7 +3,7 @@ import stat
|
||||
import json
|
||||
import zlib
|
||||
import typing
|
||||
from typing import Sequence, MutableSequence
|
||||
from typing import List, Sequence, MutableSequence, Optional
|
||||
from hashlib import sha256
|
||||
from operator import attrgetter
|
||||
from torba.client.hash import better_aes_encrypt, better_aes_decrypt
|
||||
@@ -25,12 +25,51 @@ class Wallet:
|
||||
self.accounts = accounts or []
|
||||
self.storage = storage or WalletStorage()
|
||||
|
||||
def add_account(self, account):
|
||||
@property
|
||||
def id(self):
|
||||
if self.storage.path:
|
||||
return os.path.basename(self.storage.path)
|
||||
return self.name
|
||||
|
||||
def add_account(self, account: 'baseaccount.BaseAccount'):
|
||||
self.accounts.append(account)
|
||||
|
||||
def generate_account(self, ledger: 'baseledger.BaseLedger') -> 'baseaccount.BaseAccount':
|
||||
return ledger.account_class.generate(ledger, self)
|
||||
|
||||
@property
|
||||
def default_account(self) -> Optional['baseaccount.BaseAccount']:
|
||||
for account in self.accounts:
|
||||
return account
|
||||
return None
|
||||
|
||||
def get_account_or_default(self, account_id: str) -> Optional['baseaccount.BaseAccount']:
|
||||
if account_id is None:
|
||||
return self.default_account
|
||||
return self.get_account_or_error(account_id)
|
||||
|
||||
def get_account_or_error(self, account_id: str) -> 'baseaccount.BaseAccount':
|
||||
for account in self.accounts:
|
||||
if account.id == account_id:
|
||||
return account
|
||||
raise ValueError(f"Couldn't find account: {account_id}.")
|
||||
|
||||
def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['baseaccount.BaseAccount']:
|
||||
return [
|
||||
self.get_account_or_error(account_id)
|
||||
for account_id in account_ids
|
||||
] if account_ids else self.accounts
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@classmethod
|
||||
def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':
|
||||
json_dict = storage.read()
|
||||
@@ -54,11 +93,6 @@ class Wallet:
|
||||
def save(self):
|
||||
self.storage.write(self.to_dict())
|
||||
|
||||
@property
|
||||
def default_account(self):
|
||||
for account in self.accounts:
|
||||
return account
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256()
|
||||
|
||||
@@ -151,7 +151,9 @@ class WalletNode:
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
wallet_file_name = os.path.join(self.data_path, 'my_wallet.json')
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
with open(wallet_file_name, 'w') as wallet_file:
|
||||
wallet_file.write('{"version": 1, "accounts": []}\n')
|
||||
self.manager = self.manager_class.from_config({
|
||||
@@ -190,14 +192,15 @@ class WalletNode:
|
||||
|
||||
class SPVNode:
|
||||
|
||||
def __init__(self, coin_class):
|
||||
def __init__(self, coin_class, node_number=1):
|
||||
self.coin_class = coin_class
|
||||
self.controller = None
|
||||
self.data_path = None
|
||||
self.server = None
|
||||
self.hostname = 'localhost'
|
||||
self.port = 50001 + 1 # avoid conflict with default daemon
|
||||
self.port = 50001 + node_number # avoid conflict with default daemon
|
||||
self.session_timeout = 600
|
||||
self.rpc_port = '0' # disabled by default
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode'):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
@@ -210,6 +213,7 @@ class SPVNode:
|
||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
'RPC_PORT': self.rpc_port
|
||||
}
|
||||
# TODO: don't use os.environ
|
||||
os.environ.update(conf)
|
||||
|
||||
@@ -10,7 +10,8 @@ def get_argument_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="torba-server"
|
||||
)
|
||||
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.")
|
||||
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
|
||||
nargs="?", default="lbry.wallet.server.coin.LBC")
|
||||
return parser
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import time
|
||||
from asyncio import sleep
|
||||
from bisect import bisect_right
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from struct import pack, unpack
|
||||
|
||||
@@ -72,9 +73,8 @@ class DB:
|
||||
self.header_len = self.dynamic_header_len
|
||||
|
||||
self.logger.info(f'switching current directory to {env.db_dir}')
|
||||
os.chdir(env.db_dir)
|
||||
|
||||
self.db_class = db_class(self.env.db_engine)
|
||||
self.db_class = db_class(env.db_dir, self.env.db_engine)
|
||||
self.history = History()
|
||||
self.utxo_db = None
|
||||
self.tx_counts = None
|
||||
@@ -86,12 +86,13 @@ class DB:
|
||||
self.merkle = Merkle()
|
||||
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
|
||||
|
||||
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
|
||||
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
|
||||
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
|
||||
path = partial(os.path.join, self.env.db_dir)
|
||||
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
|
||||
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
|
||||
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file = util.LogicalFile(
|
||||
'meta/headers_offsets', 2, 16000000)
|
||||
path('meta/headers_offsets'), 2, 16000000)
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
if self.tx_counts is not None:
|
||||
@@ -115,8 +116,9 @@ class DB:
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new database')
|
||||
self.logger.info('creating metadata directory')
|
||||
os.mkdir('meta')
|
||||
with util.open_file('COIN', create=True) as f:
|
||||
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
|
||||
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
|
||||
with util.open_file(coin_path, create=True) as f:
|
||||
f.write(f'ElectrumX databases and metadata for '
|
||||
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
@@ -474,7 +476,7 @@ class DB:
|
||||
return 'meta/block'
|
||||
|
||||
def raw_block_path(self, height):
|
||||
return f'{self.raw_block_prefix()}{height:d}'
|
||||
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
|
||||
|
||||
def read_raw_block(self, height):
|
||||
"""Returns a raw block read from disk. Raises FileNotFoundError
|
||||
|
||||
@@ -80,6 +80,8 @@ class Env:
|
||||
self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
|
||||
@@ -767,6 +767,9 @@ class ElectrumX(SessionBase):
|
||||
'protocol_min': min_str,
|
||||
'protocol_max': max_str,
|
||||
'genesis_hash': env.coin.GENESIS_HASH,
|
||||
'description': env.description,
|
||||
'payment_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
}
|
||||
|
||||
|
||||
@@ -13,20 +13,21 @@ from functools import partial
|
||||
from torba.server import util
|
||||
|
||||
|
||||
def db_class(name):
|
||||
def db_class(db_dir, name):
|
||||
"""Returns a DB engine class."""
|
||||
for db_class in util.subclasses(Storage):
|
||||
if db_class.__name__.lower() == name.lower():
|
||||
db_class.import_module()
|
||||
return db_class
|
||||
return partial(db_class, db_dir)
|
||||
raise RuntimeError('unrecognised DB engine "{}"'.format(name))
|
||||
|
||||
|
||||
class Storage:
|
||||
"""Abstract base class of the DB backend abstraction."""
|
||||
|
||||
def __init__(self, name, for_sync):
|
||||
self.is_new = not os.path.exists(name)
|
||||
def __init__(self, db_dir, name, for_sync):
|
||||
self.db_dir = db_dir
|
||||
self.is_new = not os.path.exists(os.path.join(db_dir, name))
|
||||
self.for_sync = for_sync or self.is_new
|
||||
self.open(name, create=self.is_new)
|
||||
|
||||
@@ -78,8 +79,9 @@ class LevelDB(Storage):
|
||||
|
||||
def open(self, name, create):
|
||||
mof = 512 if self.for_sync else 128
|
||||
path = os.path.join(self.db_dir, name)
|
||||
# Use snappy compression (the default)
|
||||
self.db = self.module.DB(name, create_if_missing=create,
|
||||
self.db = self.module.DB(path, create_if_missing=create,
|
||||
max_open_files=mof)
|
||||
self.close = self.db.close
|
||||
self.get = self.db.get
|
||||
@@ -99,12 +101,13 @@ class RocksDB(Storage):
|
||||
|
||||
def open(self, name, create):
|
||||
mof = 512 if self.for_sync else 128
|
||||
path = os.path.join(self.db_dir, name)
|
||||
# Use snappy compression (the default)
|
||||
options = self.module.Options(create_if_missing=create,
|
||||
use_fsync=True,
|
||||
target_file_size_base=33554432,
|
||||
max_open_files=mof)
|
||||
self.db = self.module.DB(name, options)
|
||||
self.db = self.module.DB(path, options)
|
||||
self.get = self.db.get
|
||||
self.put = self.db.put
|
||||
|
||||
|
||||
@@ -145,8 +145,8 @@ class Stream:
|
||||
def first(self):
|
||||
future = asyncio.get_event_loop().create_future()
|
||||
subscription = self.listen(
|
||||
lambda value: self._cancel_and_callback(subscription, future, value),
|
||||
lambda exception: self._cancel_and_error(subscription, future, exception)
|
||||
lambda value: not future.done() and self._cancel_and_callback(subscription, future, value),
|
||||
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception)
|
||||
)
|
||||
return future
|
||||
|
||||
|
||||
+2
-2
@@ -13,6 +13,6 @@ changedir = {toxinidir}/tests
|
||||
setenv =
|
||||
integration: TORBA_LEDGER={envname}
|
||||
commands =
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.unit
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.unit
|
||||
integration: orchstr8 download
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.integration
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.integration
|
||||
Reference in new issue
Block a user