Compare commits

...
18 Commits
Author SHA1 Message Date
Victor Shyba 5718be1c61 log top memory usage on status if tracemalloc enabled 2019-09-16 15:00:35 -03:00
Lex Berezhny 5c151a3a09 v0.41.0 2019-09-16 13:24:11 -04:00
Lex Berezhny 3c46c6cbb9 fix mypy type issue 2019-09-16 13:21:45 -04:00
Lex Berezhny d31f09c174 client wallet db versioning 2019-09-16 13:21:45 -04:00
Lex Berezhny 063eeb34ec fix channel import test to wait for address subscription tasks to finish before looking up channel 2019-09-16 13:21:45 -04:00
Lex Berezhny f06192601c dont fail when claim is not decodable 2019-09-16 13:21:45 -04:00
Lex Berezhny 091221ba2c added wallet/constants.py 2019-09-16 13:21:45 -04:00
Lex Berezhny 7c22111ae0 channel certificate and channel claim can be different accounts, also JSON RPC responses now include a field showing if the channel certificate is available 2019-09-16 13:21:45 -04:00
Alex Grin daac3eed09 Merge pull request #2458 from lbryio/wallet_server_default_coin
default wallet server to LBC coin
2019-09-16 12:34:54 -04:00
Alex Grintsvayg 32ee7feea8 download wallet snapshot on first run 2019-09-16 12:24:44 -04:00
Alex Grintsvayg 2026848a77 default wallet server to LBC coin 2019-09-16 12:01:38 -04:00
Jack Robison a7b205603c Merge pull request #2447 from lbryio/file_encoder
use json_file_encoder for managed streams (file list)
2019-09-13 11:21:36 -04:00
Victor Shyba 291d14f2d4 move as_dict to json_encoder 2019-09-12 14:08:37 -03:00
Victor Shyba 4a68137cfa remove dead code 2019-09-12 14:08:37 -03:00
Jack Robison b2f0374426 Merge pull request #2452 from lbryio/fix-sqlite-misuse
Fix SQLITE_MISUSE
2019-09-12 11:57:23 -04:00
Jack Robison 5fcca6ed0c feedback
-add comment for the fetchall() that if removed will cause SQLITE_MISUSE
2019-09-10 15:03:17 -04:00
Jack Robison a05c44ad1e fix SQLITE_MISUSE
fixes https://github.com/lbryio/lbry-sdk/issues/2164
2019-09-09 22:38:53 -04:00
Jack Robison f9880c8931 test sqlite binding error 2019-09-09 21:46:04 -04:00
18 changed files with 313 additions and 284 deletions

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.40.1"
__version__ = "0.41.0"
version = tuple(__version__.split('.'))
+57 -20
View File
@@ -9,6 +9,7 @@ import base58
import random
import ecdsa
import hashlib
import tracemalloc
from urllib.parse import urlencode, quote
from typing import Callable, Optional, List
from binascii import hexlify, unhexlify
@@ -145,6 +146,14 @@ DHT_HAS_CONTACTS = "dht_has_contacts"
WALLET_IS_UNLOCKED = "wallet_is_unlocked"
def debug_log_tracemalloc():
snapshot = tracemalloc.take_snapshot()
top_stats = snapshot.statistics('lineno')
log.debug("[ Top 10 memory usage ]")
for stat in top_stats[:10]:
log.debug(stat)
class DHTHasContacts(RequiredCondition):
name = DHT_HAS_CONTACTS
component = DHT_COMPONENT
@@ -795,6 +804,8 @@ class Daemon(metaclass=JSONRPCServerType):
}
}
"""
if tracemalloc.is_tracing():
debug_log_tracemalloc()
connection_code = await self.get_connection_status()
@@ -2291,15 +2302,39 @@ class Daemon(metaclass=JSONRPCServerType):
channel_private_key = ecdsa.SigningKey.from_pem(
data['signing_private_key'], hashfunc=hashlib.sha256
)
account: LBCAccount = await self.ledger.get_account_for_address(data['holding_address'])
if not account:
account = LBCAccount.from_dict(self.ledger, self.default_wallet, {
'name': f"Holding Account For Channel {data['name']}",
'public_key': data['holding_public_key'],
'address_generator': {'name': 'single-address'}
})
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
public_key_der = channel_private_key.get_verifying_key().to_der()
# check that the holding_address hasn't changed since the export was made
holding_address = data['holding_address']
channels, _, _ = await self.ledger.claim_search(
public_key_id=self.ledger.public_key_to_address(public_key_der)
)
if channels and channels[0].get_address(self.ledger) != holding_address:
holding_address = channels[0].get_address(self.ledger)
account: LBCAccount = await self.ledger.get_account_for_address(holding_address)
if account:
# Case 1: channel holding address is in one of the accounts we already have
# simply add the certificate to existing account
pass
else:
# Case 2: channel holding address hasn't changed and thus is in the bundled read-only account
# create a single-address holding account to manage the channel
if holding_address == data['holding_address']:
account = LBCAccount.from_dict(self.ledger, self.default_wallet, {
'name': f"Holding Account For Channel {data['name']}",
'public_key': data['holding_public_key'],
'address_generator': {'name': 'single-address'}
})
if self.ledger.network.is_connected:
await self.ledger.subscribe_account(account)
await self.ledger._update_tasks.done.wait()
# Case 3: the holding address has changed and we can't create or find an account for it
else:
raise Exception(
"Channel owning account has changed since the channel was exported and "
"it is not an account to which you have access."
)
account.add_channel_private_key(channel_private_key)
self.default_wallet.save()
return f"Added channel signing key for {data['name']}."
@@ -3751,17 +3786,19 @@ class Daemon(metaclass=JSONRPCServerType):
key, value = 'name', channel_name
else:
raise ValueError("Couldn't find channel because a channel_id or channel_name was not provided.")
for account in self.get_accounts_or_all(account_ids):
channels = await account.get_channels(**{f'claim_{key}': value}, limit=1)
if len(channels) == 1:
if for_signing and channels[0].private_key is None:
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
return channels[0]
elif len(channels) > 1:
raise ValueError(
f"Multiple channels found with channel_{key} '{value}', "
f"pass a channel_id to narrow it down."
)
channels = await self.ledger.get_channels(
accounts=self.get_accounts_or_all(account_ids),
**{f'claim_{key}': value}
)
if len(channels) == 1:
if for_signing and not channels[0].has_private_key:
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
return channels[0]
elif len(channels) > 1:
raise ValueError(
f"Multiple channels found with channel_{key} '{value}', "
f"pass a channel_id to narrow it down."
)
raise ValueError(f"Couldn't find channel with channel_{key} '{value}'.")
def get_account_or_default(self, account_id: str, argument_name: str = "account", lbc_only=True) -> LBCAccount:
@@ -184,6 +184,8 @@ class JSONResponseEncoder(JSONEncoder):
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.claim.is_channel:
output['has_signing_key'] = txo.has_private_key
if check_signature and txo.claim.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
@@ -218,15 +220,44 @@ class JSONResponseEncoder(JSONEncoder):
return result
def encode_file(self, managed_stream):
file = managed_stream.as_dict()
output_exists = managed_stream.output_file_exists
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
file.update({
return {
'streaming_url': managed_stream.stream_url,
'completed': managed_stream.completed,
'file_name': managed_stream.file_name if output_exists else None,
'download_directory': managed_stream.download_directory if output_exists else None,
'download_path': managed_stream.full_path if output_exists else None,
'points_paid': 0.0,
'stopped': not managed_stream.running,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.descriptor.stream_name,
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
'written_bytes': managed_stream.written_bytes,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'status': managed_stream.status,
'claim_id': managed_stream.claim_id,
'txid': managed_stream.txid,
'nout': managed_stream.nout,
'outpoint': managed_stream.outpoint,
'metadata': managed_stream.metadata,
'protobuf': managed_stream.metadata_protobuf,
'channel_claim_id': managed_stream.channel_claim_id,
'channel_name': managed_stream.channel_name,
'claim_name': managed_stream.claim_name,
'content_fee': managed_stream.content_fee,
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
})
return file
}
def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict()
-119
View File
@@ -90,20 +90,6 @@ def get_claims_from_stream_hashes(transaction: sqlite3.Connection,
}
def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
outpoint: str) -> typing.Optional[StoredStreamClaim]:
query = (
"select content_claim.stream_hash, c.*, case when c.channel_claim_id is not null then "
" (select claim_name from claim where claim_id==c.channel_claim_id) "
" else null end as channel_name "
" from content_claim "
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.claim_outpoint=?"
)
claim_fields = transaction.execute(query, (outpoint, )).fetchone()
if claim_fields:
return StoredStreamClaim(*claim_fields)
def _batched_select(transaction, query, parameters, batch_size=900):
for start_index in range(0, len(parameters), batch_size):
current_batch = parameters[start_index:start_index+batch_size]
@@ -325,31 +311,6 @@ class SQLiteStorage(SQLiteMixin):
"select status from blob where blob_hash=?", blob_hash
)
def should_announce(self, blob_hash: str):
return self.run_and_return_one_or_none(
"select should_announce from blob where blob_hash=?", blob_hash
)
def count_should_announce_blobs(self):
return self.run_and_return_one_or_none(
"select count(*) from blob where should_announce=1 and status='finished'"
)
def get_all_should_announce_blobs(self):
return self.run_and_return_list(
"select blob_hash from blob where should_announce=1 and status='finished'"
)
def get_all_finished_blobs(self):
return self.run_and_return_list(
"select blob_hash from blob where status='finished'"
)
def count_finished_blobs(self):
return self.run_and_return_one_or_none(
"select count(*) from blob where status='finished'"
)
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
def _update_last_announced_blobs(transaction: sqlite3.Connection):
last_announced = self.time_getter()
@@ -427,26 +388,6 @@ class SQLiteStorage(SQLiteMixin):
}
return self.db.run(_sync_blobs)
def sync_files_to_blobs(self):
def _sync_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
transaction.execute(
"select distinct sb.stream_hash from stream_blob sb "
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
).fetchall()
)
return self.db.run(_sync_blobs)
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
def _set_streaming(transaction: sqlite3.Connection):
transaction.executemany(
"update file set file_name=null, download_directory=null where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
return self.db.run(_set_streaming)
# # # # # # # # # stream functions # # # # # # # # #
async def stream_exists(self, sd_hash: str) -> bool:
@@ -459,11 +400,6 @@ class SQLiteStorage(SQLiteMixin):
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
return streams is not None
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
return self.run_and_return_one_or_none(
"select rowid from file where stream_hash=?", stream_hash
)
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
return self.db.run(store_stream, sd_blob, descriptor)
@@ -509,12 +445,6 @@ class SQLiteStorage(SQLiteMixin):
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
)
def get_stream_info_for_sd_hash(self, sd_blob_hash):
return self.run_and_return_one_or_none(
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
sd_blob_hash
)
def delete_stream(self, descriptor: 'StreamDescriptor'):
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
@@ -788,55 +718,6 @@ class SQLiteStorage(SQLiteMixin):
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
return claim
async def get_claims_from_stream_hashes(self, stream_hashes: typing.List[str],
include_supports: typing.Optional[bool] = True):
claims = await self.db.run(get_claims_from_stream_hashes, stream_hashes)
return {stream_hash: claim_info.as_dict() for stream_hash, claim_info in claims.items()}
async def get_claim(self, claim_outpoint, include_supports=True):
claim_info = await self.db.run(get_content_claim_from_outpoint, claim_outpoint)
if not claim_info:
return
result = claim_info.as_dict()
if include_supports:
supports = await self.get_supports(result['claim_id'])
result['supports'] = supports
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
return result
def get_unknown_certificate_ids(self):
def _get_unknown_certificate_claim_ids(transaction):
return [
claim_id for (claim_id,) in transaction.execute(
"select distinct c1.channel_claim_id from claim as c1 "
"where c1.channel_claim_id!='' "
"and c1.channel_claim_id not in "
"(select c2.claim_id from claim as c2)"
).fetchall()
]
return self.db.run(_get_unknown_certificate_claim_ids)
async def get_pending_claim_outpoints(self):
claim_outpoints = await self.run_and_return_list("select claim_outpoint from claim where height=-1")
results = {} # {txid: [nout, ...]}
for outpoint_str in claim_outpoints:
txid, nout = outpoint_str.split(":")
outputs = results.get(txid, [])
outputs.append(int(nout))
results[txid] = outputs
if results:
log.debug("missing transaction heights for %i claims", len(results))
return results
def save_claim_tx_heights(self, claim_tx_heights):
def _save_claim_heights(transaction):
for outpoint, height in claim_tx_heights.items():
transaction.execute(
"update claim set height=? where claim_outpoint=? and height=-1",
(height, outpoint)
)
return self.db.run(_save_claim_heights)
# # # # # # # # # reflector functions # # # # # # # # #
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
+8 -41
View File
@@ -123,6 +123,14 @@ class ManagedStream:
def written_bytes(self) -> int:
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
@property
def completed(self):
return self.written_bytes >= self.descriptor.lower_bound_decrypted_length()
@property
def stream_url(self):
return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}"
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
@@ -203,47 +211,6 @@ class ManagedStream:
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
def as_dict(self) -> typing.Dict:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if not self.output_file_exists:
full_path = None
file_name = None
download_directory = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': self.written_bytes >= self.descriptor.lower_bound_decrypted_length(),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
'suggested_file_name': self.descriptor.suggested_file_name,
'sd_hash': self.descriptor.sd_hash,
'download_path': full_path,
'mime_type': self.mime_type,
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
'written_bytes': self.written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
'status': self.status,
'claim_id': self.claim_id,
'txid': self.txid,
'nout': self.nout,
'outpoint': self.outpoint,
'metadata': self.metadata,
'protobuf': self.metadata_protobuf,
'channel_claim_id': self.channel_claim_id,
'channel_name': self.channel_name,
'claim_name': self.claim_name,
'content_fee': self.content_fee
}
@classmethod
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
+4 -4
View File
@@ -6,6 +6,7 @@ from string import hexdigits
import ecdsa
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.constants import TXO_TYPES
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
@@ -76,7 +77,7 @@ class Account(BaseAccount):
def get_balance(self, confirmations=0, include_claims=False, **constraints):
if not include_claims:
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
constraints.update({'txo_type': 0})
return super().get_balance(confirmations, **constraints)
async def get_granular_balances(self, confirmations=0, reserved_subtotals=False):
@@ -84,7 +85,7 @@ class Account(BaseAccount):
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
total = await get_total_balance()
if reserved_subtotals:
claims_balance = await get_total_balance(claim_type__or={'is_claim': True, 'is_update': True})
claims_balance = await get_total_balance(txo_type__in=[TXO_TYPES['stream'], TXO_TYPES['channel']])
for amount, spent, from_me, to_me, height in await self.get_support_summary():
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
continue
@@ -96,8 +97,7 @@ class Account(BaseAccount):
reserved = claims_balance + supports_balance + tips_balance
else:
reserved = await self.get_balance(
confirmations=confirmations, include_claims=True,
claim_type__or={'is_claim': True, 'is_support': True, 'is_update': True}
confirmations=confirmations, include_claims=True, txo_type__gt=0
)
return {
'total': dewies_to_lbc(total),
+5
View File
@@ -0,0 +1,5 @@
TXO_TYPES = {
"stream": 1,
"channel": 2,
"support": 3
}
+19 -21
View File
@@ -3,6 +3,7 @@ from typing import List
from torba.client.basedatabase import BaseDatabase
from lbry.wallet.transaction import Output
from lbry.wallet.constants import TXO_TYPES
class WalletDatabase(BaseDatabase):
@@ -17,15 +18,12 @@ class WalletDatabase(BaseDatabase):
script blob not null,
is_reserved boolean not null default 0,
txo_type integer not null default 0,
claim_id text,
claim_name text,
is_claim boolean not null default 0,
is_update boolean not null default 0,
is_support boolean not null default 0,
is_buy boolean not null default 0,
is_sell boolean not null default 0
claim_name text
);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_txo_type_idx on txo (txo_type);
"""
CREATE_TABLES_QUERY = (
@@ -40,13 +38,13 @@ class WalletDatabase(BaseDatabase):
def txo_to_row(self, tx, address, txo):
row = super().txo_to_row(tx, address, txo)
row.update({
'is_claim': txo.script.is_claim_name,
'is_update': txo.script.is_update_claim,
'is_support': txo.script.is_support_claim,
'is_buy': txo.script.is_buy_claim,
'is_sell': txo.script.is_sell_claim,
})
if txo.is_claim:
if txo.can_decode_claim:
row['txo_type'] = TXO_TYPES.get(txo.claim.claim_type, TXO_TYPES['stream'])
else:
row['txo_type'] = TXO_TYPES['stream']
elif txo.is_support:
row['txo_type'] = TXO_TYPES['support']
if txo.script.is_claim_involved:
row['claim_id'] = txo.claim_id
row['claim_name'] = txo.claim_name
@@ -87,7 +85,9 @@ class WalletDatabase(BaseDatabase):
@staticmethod
def constrain_claims(constraints):
constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1}
constraints['txo_type__in'] = [
TXO_TYPES['stream'], TXO_TYPES['channel']
]
async def get_claims(self, **constraints) -> List[Output]:
self.constrain_claims(constraints)
@@ -99,8 +99,7 @@ class WalletDatabase(BaseDatabase):
@staticmethod
def constrain_streams(constraints):
if 'claim_name' not in constraints or 'claim_id' not in constraints:
constraints['claim_name__not_like'] = '@%'
constraints['txo_type'] = TXO_TYPES['stream']
def get_streams(self, **constraints):
self.constrain_streams(constraints)
@@ -112,8 +111,7 @@ class WalletDatabase(BaseDatabase):
@staticmethod
def constrain_channels(constraints):
if 'claim_name' not in constraints or 'claim_id' not in constraints:
constraints['claim_name__like'] = '@%'
constraints['txo_type'] = TXO_TYPES['channel']
def get_channels(self, **constraints):
self.constrain_channels(constraints)
@@ -125,7 +123,7 @@ class WalletDatabase(BaseDatabase):
@staticmethod
def constrain_supports(constraints):
constraints['is_support'] = 1
constraints['txo_type'] = TXO_TYPES['support']
def get_supports(self, **constraints):
self.constrain_supports(constraints)
@@ -144,12 +142,12 @@ class WalletDatabase(BaseDatabase):
)
def get_supports_summary(self, account_id):
return self.db.execute_fetchall("""
return self.db.execute_fetchall(f"""
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
(txo.txid in
(select txi.txid from txi join pubkey_address a on txi.address = a.address
where a.account = ?)) as from_me,
(txo.address in (select address from pubkey_address where account=?)) as to_me,
tx.height
from txo join tx using (txid) where is_support=1
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
""", (account_id, account_id))
+1 -1
View File
@@ -110,7 +110,7 @@ class MainNetLedger(BaseLedger):
@staticmethod
def constraint_spending_utxos(constraints):
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
constraints['txo_type'] = 0
def get_utxos(self, **constraints):
self.constraint_spending_utxos(constraints)
+3 -2
View File
@@ -8,7 +8,7 @@ RUN mkdir -p /home/$user/
RUN chown -R $user:$user /home/$user/
# install python, pip, git and clean up
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
# create and chown database dir
ARG db_dir=/database
@@ -53,4 +53,5 @@ ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
ENV MAX_SESSIONS=1000000000
ENV MAX_SEND=1000000000000000000
ENV EVENT_LOOP_POLICY=uvloop
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
COPY ./lbry/scripts/wallet_server_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
# entrypoint for wallet server Docker image
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
wget -O wallet_snapshot.tar.bz2 "$SNAPSHOT_URL"
echo "Extracting snapshot..."
tar xvjf wallet_snapshot.tar.bz2 --directory /database
rm wallet_snapshot.tar.bz2
fi
/home/lbry/.local/bin/torba-server "$@"
@@ -1,5 +1,9 @@
import asyncio
import sqlite3
from lbry.testcase import CommandTestCase
from torba.client.basedatabase import SQLiteMixin
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.account import Account
def extract(d, keys):
@@ -7,6 +11,13 @@ def extract(d, keys):
class AccountManagement(CommandTestCase):
async def test_sqlite_binding_error(self):
tasks = [
self.loop.create_task(self.daemon.jsonrpc_account_create('second account' + str(x))) for x in range(100)
]
await asyncio.wait(tasks)
for result in tasks:
self.assertFalse(isinstance(result.result(), Exception))
async def test_account_list_set_create_remove_add(self):
# check initial account
+10 -36
View File
@@ -499,44 +499,9 @@ class ChannelCommands(CommandTestCase):
claim_id = self.get_claim_id(tx)
channel_private_key = (await self.account.get_channels())[0].private_key
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
daemon2 = await self.add_daemon()
# before importing channel
self.assertEqual(1, len(daemon2.default_wallet.accounts))
# importing channel which will create a new single key account
await daemon2.jsonrpc_channel_import(exported_data)
# after import
self.assertEqual(2, len(daemon2.default_wallet.accounts))
new_account = daemon2.default_wallet.accounts[1]
await daemon2.ledger._update_tasks.done.wait()
channels = await new_account.get_channels()
self.assertEqual(1, len(channels))
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
async def test_channel_export_import_into_existing_account(self):
tx = await self.channel_create('@foo', '1.0')
claim_id = self.get_claim_id(tx)
channel_private_key = (await self.account.get_channels())[0].private_key
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
daemon2 = await self.add_daemon(seed=self.account.seed)
await daemon2.ledger._update_tasks.done.wait() # will sync channel previously created
# before importing channel key, has channel without key
self.assertEqual(1, len(daemon2.default_wallet.accounts))
channels = await daemon2.default_account.get_channels()
self.assertEqual(1, len(channels))
self.assertIsNone(channels[0].private_key)
# importing channel will add it to existing account
await daemon2.jsonrpc_channel_import(exported_data)
# after import, still just one account but with private key now
self.assertEqual(1, len(daemon2.default_wallet.accounts))
channels = await daemon2.default_account.get_channels()
channels = await daemon2.jsonrpc_channel_list()
self.assertEqual(1, len(channels))
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
@@ -653,6 +618,15 @@ class StreamCommands(ClaimTestCase):
'hovercraft5', '0.1', channel_name='@baz', channel_account_id=[account1_id]
)
# signing with channel works even if channel and certificate are in different accounts
await self.channel_update(
baz_id, account_id=account2_id,
claim_address=await self.daemon.jsonrpc_address_unused(account1_id)
)
await self.stream_create(
'hovercraft5', '0.1', channel_id=baz_id
)
async def test_preview_works_with_signed_streams(self):
await self.out(self.channel_create('@spam', '1.0'))
signed = await self.out(self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False))
@@ -43,11 +43,11 @@ class TestManagedStream(BlobExchangeTestBase):
async def test_status_file_completed(self):
await self._test_transfer_stream(10)
self.assertTrue(self.stream.output_file_exists)
self.assertTrue(self.stream.as_dict()['completed'])
self.assertTrue(self.stream.completed)
with open(self.stream.full_path, 'w+b') as outfile:
outfile.truncate(1)
self.assertTrue(self.stream.output_file_exists)
self.assertFalse(self.stream.as_dict()['completed'])
self.assertFalse(self.stream.completed)
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
await self.setup_stream(blob_count)
+78 -1
View File
@@ -1,5 +1,7 @@
import unittest
import sqlite3
import tempfile
import os
from torba.client.wallet import Wallet
from torba.client.constants import COIN
@@ -268,7 +270,6 @@ class TestQueries(AsyncioTestCase):
self.assertEqual(len(tx.outputs), 1)
last_tx = tx
async def test_queries(self):
self.assertEqual(0, await self.ledger.db.get_address_count())
account1 = await self.create_account()
@@ -354,3 +355,79 @@ class TestQueries(AsyncioTestCase):
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
class TestUpgrade(AsyncioTestCase):
def setUp(self) -> None:
self.path = tempfile.mktemp()
def tearDown(self) -> None:
os.remove(self.path)
def get_version(self):
with sqlite3.connect(self.path) as conn:
versions = conn.execute('select version from version').fetchall()
assert len(versions) == 1
return versions[0][0]
def get_tables(self):
with sqlite3.connect(self.path) as conn:
sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
return [col[0] for col in conn.execute(sql).fetchall()]
def add_address(self, address):
with sqlite3.connect(self.path) as conn:
conn.execute("""
INSERT INTO pubkey_address (address, account, chain, position, pubkey)
VALUES (?, 'account1', 0, 0, 'pubkey blob')
""", (address,))
def get_addresses(self):
with sqlite3.connect(self.path) as conn:
sql = "SELECT address FROM pubkey_address ORDER BY address;"
return [col[0] for col in conn.execute(sql).fetchall()]
async def test_reset_on_version_change(self):
self.ledger = ledger_class({
'db': ledger_class.database_class(self.path),
'headers': ledger_class.headers_class(':memory:'),
})
# initial open, pre-version enabled db
self.ledger.db.SCHEMA_VERSION = None
self.assertEqual(self.get_tables(), [])
await self.ledger.db.open()
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo'])
self.assertEqual(self.get_addresses(), [])
self.add_address('address1')
await self.ledger.db.close()
# initial open after version enabled
self.ledger.db.SCHEMA_VERSION = '1.0'
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
self.add_address('address2')
await self.ledger.db.close()
# nothing changes
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.0')
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), ['address2'])
await self.ledger.db.close()
# upgrade version, database reset
self.ledger.db.SCHEMA_VERSION = '1.1'
self.ledger.db.CREATE_TABLES_QUERY += """
create table if not exists foo (bar text);
"""
await self.ledger.db.open()
self.assertEqual(self.get_version(), '1.1')
self.assertEqual(self.get_tables(), ['foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
self.assertEqual(self.get_addresses(), []) # all tables got reset
await self.ledger.db.close()
+59 -30
View File
@@ -1,10 +1,9 @@
import logging
import asyncio
from asyncio import wrap_future
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Optional
import sqlite3
@@ -23,8 +22,10 @@ class AIOSQLite:
@classmethod
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
def _connect():
return sqlite3.connect(path, *args, **kwargs)
db = cls()
db.connection = await wrap_future(db.executor.submit(sqlite3.connect, path, *args, **kwargs))
db.connection = await asyncio.get_event_loop().run_in_executor(db.executor, _connect)
return db
async def close(self):
@@ -38,25 +39,29 @@ class AIOSQLite:
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
def executemany(self, sql: str, params: Iterable):
def __executemany_in_a_transaction(conn: sqlite3.Connection, *args, **kwargs):
return conn.executemany(*args, **kwargs)
return self.run(__executemany_in_a_transaction, sql, params)
params = params if params is not None else []
# this fetchall is needed to prevent SQLITE_MISUSE
return self.run(lambda conn: conn.executemany(sql, params).fetchall())
def executescript(self, script: str) -> Awaitable:
return wrap_future(self.executor.submit(self.connection.executescript, script))
return self.run(lambda conn: conn.executescript(script))
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
def __fetchall(conn: sqlite3.Connection, *args, **kwargs):
return conn.execute(*args, **kwargs).fetchall()
return wrap_future(self.executor.submit(__fetchall, self.connection, sql, parameters))
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
def execute_fetchone(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchone())
def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn, sql, parameters: conn.execute(sql, parameters), sql, parameters)
return self.run(lambda conn: conn.execute(sql, parameters))
def run(self, fun, *args, **kwargs) -> Awaitable:
return wrap_future(self.executor.submit(self.__run_transaction, fun, *args, **kwargs))
return asyncio.get_event_loop().run_in_executor(
self.executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
self.connection.execute('begin')
@@ -64,19 +69,20 @@ class AIOSQLite:
result = fun(self.connection, *args, **kwargs) # type: ignore
self.connection.commit()
return result
except (Exception, OSError): # as e:
#log.exception('Error running transaction:', exc_info=e)
except (Exception, OSError) as e:
log.exception('Error running transaction:', exc_info=e)
self.connection.rollback()
log.warning("rolled back")
raise
def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable:
return wrap_future(
self.executor.submit(self.__run_transaction_with_foreign_keys_disabled, fun, *args, **kwargs)
return asyncio.get_event_loop().run_in_executor(
self.executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
def __run_transaction_with_foreign_keys_disabled(self,
fun: Callable[[sqlite3.Connection, Any, Any], Any],
*args, **kwargs):
args, kwargs):
foreign_keys_enabled, = self.connection.execute("pragma foreign_keys").fetchone()
if not foreign_keys_enabled:
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
@@ -210,9 +216,16 @@ def rows_to_dict(rows, fields):
class SQLiteMixin:
SCHEMA_VERSION: Optional[str] = None
CREATE_TABLES_QUERY: str
MAX_QUERY_VARIABLES = 900
CREATE_VERSION_TABLE = """
create table if not exists version (
version text
);
"""
def __init__(self, path):
self._db_path = path
self.db: AIOSQLite = None
@@ -221,6 +234,20 @@ class SQLiteMixin:
async def open(self):
log.info("connecting to database: %s", self._db_path)
self.db = await AIOSQLite.connect(self._db_path, isolation_level=None)
if self.SCHEMA_VERSION:
tables = [t[0] for t in await self.db.execute_fetchall(
"SELECT name FROM sqlite_master WHERE type='table';"
)]
if tables:
if 'version' in tables:
version = await self.db.execute_fetchone("SELECT version FROM version LIMIT 1;")
if version == (self.SCHEMA_VERSION,):
return
await self.db.executescript('\n'.join(
f"DROP TABLE {table};" for table in tables
))
await self.db.execute(self.CREATE_VERSION_TABLE)
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
await self.db.executescript(self.CREATE_TABLES_QUERY)
async def close(self):
@@ -259,6 +286,8 @@ class SQLiteMixin:
class BaseDatabase(SQLiteMixin):
SCHEMA_VERSION = "1.0"
PRAGMAS = """
pragma journal_mode=WAL;
"""
@@ -337,14 +366,18 @@ class BaseDatabase(SQLiteMixin):
'script': sqlite3.Binary(txo.script.source)
}
async def insert_transaction(self, tx):
await self.db.execute(*self._insert_sql('tx', {
@staticmethod
def tx_to_row(tx):
return {
'txid': tx.id,
'raw': sqlite3.Binary(tx.raw),
'height': tx.height,
'position': tx.position,
'is_verified': tx.is_verified
}))
}
async def insert_transaction(self, tx):
await self.db.execute(*self._insert_sql('tx', self.tx_to_row(tx)))
async def update_transaction(self, tx):
await self.db.execute(*self._update_sql("tx", {
@@ -352,13 +385,7 @@ class BaseDatabase(SQLiteMixin):
}, 'txid = ?', (tx.id,)))
def _transaction_io(self, conn: sqlite3.Connection, tx: BaseTransaction, address, txhash, history):
conn.execute(*self._insert_sql('tx', {
'txid': tx.id,
'raw': sqlite3.Binary(tx.raw),
'height': tx.height,
'position': tx.position,
'is_verified': tx.is_verified
}, replace=True))
conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True))
for txo in tx.outputs:
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
@@ -584,9 +611,11 @@ class BaseDatabase(SQLiteMixin):
async def add_keys(self, account, chain, keys):
await self.db.executemany(
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
((pubkey.address, account.public_key.address, chain,
position, sqlite3.Binary(pubkey.pubkey_bytes))
for position, pubkey in keys)
(
(pubkey.address, account.public_key.address, chain, position,
sqlite3.Binary(pubkey.pubkey_bytes))
for position, pubkey in keys
)
)
async def _set_address_history(self, address, history):
+2
View File
@@ -230,6 +230,8 @@ class BaseLedger(metaclass=LedgerRegistry):
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
def constraint_account_or_all(self, constraints):
if 'accounts' in constraints:
return
account = constraints.pop('account', None)
if account:
constraints['accounts'] = [account]
+2 -1
View File
@@ -10,7 +10,8 @@ def get_argument_parser():
parser = argparse.ArgumentParser(
prog="torba-server"
)
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.")
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
nargs="?", default="lbry.wallet.server.coin.LBC")
return parser