Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5718be1c61 |
No files matched your search
+4
-4
@@ -1,6 +1,6 @@
|
||||
# Installing LBRY
|
||||
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md)!
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
@@ -9,7 +9,7 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
@@ -17,7 +17,7 @@ After installing python 3, you'll need to install some additional libraries depe
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set:
|
||||
These environment variables also need to be set
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
|
||||
@@ -27,7 +27,7 @@ Remaining dependencies can then be installed by running:
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/.
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
|
||||
|
||||
+110
-450
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.43.0"
|
||||
__version__ = "0.41.0"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -37,14 +37,18 @@ class BlobManager:
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs or (
|
||||
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
@@ -78,7 +82,6 @@ class BlobManager:
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
|
||||
@@ -67,7 +67,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("%s started sending blob we didn't request %s instead of %s", self.peer_address,
|
||||
log.warning("%s started sending blob we didnt request %s instead of %s", self.peer_address,
|
||||
blob_response.blob_hash, self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
@@ -83,6 +83,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
self.writer.write(data)
|
||||
@@ -139,7 +141,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob can't be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
+3
-13
@@ -148,12 +148,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
@staticmethod
|
||||
def _parse_list(l):
|
||||
if l == ['null']:
|
||||
return None
|
||||
assert len(l) == 2, (
|
||||
'Max key fee is made up of either two values: '
|
||||
'"AMOUNT CURRENCY", or "null" (to set no limit)'
|
||||
)
|
||||
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
|
||||
try:
|
||||
amount = float(l[0])
|
||||
except ValueError:
|
||||
@@ -181,7 +176,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
nargs='+',
|
||||
nargs=2,
|
||||
metavar=('AMOUNT', 'CURRENCY'),
|
||||
default=NOT_SET
|
||||
)
|
||||
@@ -485,10 +480,6 @@ class Config(CLIConfig):
|
||||
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
wallets = Strings(
|
||||
"Wallet files in 'wallet_dir' to load at startup.",
|
||||
['default_wallet']
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
@@ -536,8 +527,7 @@ class Config(CLIConfig):
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
|
||||
@@ -19,6 +19,8 @@ class ConnectionManager:
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._status = {}
|
||||
self._total_sent = 0
|
||||
self._total_received = 0
|
||||
self._running = False
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@@ -56,29 +58,29 @@ class ConnectionManager:
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbs': 0.0,
|
||||
'total_outgoing_mbs': 0.0,
|
||||
'total_sent': 0,
|
||||
'total_received': 0,
|
||||
'total_sent': self._total_sent,
|
||||
'total_received': self._total_received,
|
||||
'time': self.loop.time()
|
||||
}
|
||||
|
||||
while True:
|
||||
last = self.loop.time()
|
||||
await asyncio.sleep(5, loop=self.loop)
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
now = self.loop.time()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['outgoing_bps'][k] = v / (now - last)
|
||||
self._status['outgoing_bps'][k] = v
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['incoming_bps'][k] = v / (now - last)
|
||||
self._status['incoming_bps'][k] = v
|
||||
now = self.loop.time()
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
)) / 1000000.0
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
)) / 1000000.0
|
||||
self._status['total_sent'] += sum(list(self._status['outgoing_bps'].values()))
|
||||
self._status['total_received'] += sum(list(self._status['incoming_bps'].values()))
|
||||
) / (now - last)) / 1000000.0
|
||||
self._total_sent += sum(list(self._status['outgoing_bps'].values()))
|
||||
self._total_received += sum(list(self._status['incoming_bps'].values()))
|
||||
|
||||
self._status['time'] = now
|
||||
|
||||
|
||||
@@ -4,14 +4,13 @@ import typing
|
||||
import binascii
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import PeerManager
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -37,7 +36,7 @@ class Node:
|
||||
total_peers: typing.List['KademliaPeer'] = []
|
||||
# add all peers in the routing table
|
||||
total_peers.extend(self.protocol.routing_table.get_peers())
|
||||
# add all the peers who have announced blobs to us
|
||||
# add all the peers who have announed blobs to us
|
||||
total_peers.extend(self.protocol.data_store.get_storing_contacts())
|
||||
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
@@ -142,7 +141,7 @@ class Node:
|
||||
|
||||
if known_node_addresses:
|
||||
peers = [
|
||||
make_kademlia_peer(None, address, port)
|
||||
KademliaPeer(self.loop, address, udp_port=port)
|
||||
for (address, port) in known_node_addresses
|
||||
]
|
||||
while True:
|
||||
@@ -233,7 +232,7 @@ class Node:
|
||||
if not peer.udp_port:
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if not peer.udp_port:
|
||||
peer = make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
peer.update_udp_port(udp_port_to_try)
|
||||
self.loop.create_task(ping(peer))
|
||||
else:
|
||||
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
|
||||
|
||||
+46
-28
@@ -3,7 +3,6 @@ import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from lbry.dht import constants
|
||||
@@ -12,13 +11,6 @@ from lbry.dht.serialization.datagram import make_compact_address, make_compact_i
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(1024)
|
||||
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
|
||||
udp_port: typing.Optional[int] = None,
|
||||
tcp_port: typing.Optional[int] = None) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port)
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
@@ -89,6 +81,10 @@ class PeerManager:
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
@lru_cache(maxsize=400)
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
@@ -112,16 +108,14 @@ class PeerManager:
|
||||
delay = self._loop.time() - constants.check_refresh_interval
|
||||
|
||||
# fixme: find a way to re-enable that without breaking other parts
|
||||
# if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
# return
|
||||
# addr_tup = (address, udp_port)
|
||||
# if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
#addr_tup = (address, udp_port)
|
||||
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
# return
|
||||
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
|
||||
last_requested = self._last_requested.get((address, udp_port))
|
||||
last_replied = self._last_replied.get((address, udp_port))
|
||||
if node_id is None:
|
||||
return None
|
||||
if most_recent_failure and last_replied:
|
||||
if delay < last_replied > most_recent_failure:
|
||||
return True
|
||||
@@ -141,31 +135,47 @@ class PeerManager:
|
||||
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
|
||||
return KademliaPeer(self._loop, address, node_id, tcp_port=tcp_port)
|
||||
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
class KademliaPeer:
|
||||
address: str = field(hash=True)
|
||||
_node_id: typing.Optional[bytes] = field(hash=True)
|
||||
udp_port: typing.Optional[int] = field(hash=True)
|
||||
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
|
||||
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'_node_id',
|
||||
'address',
|
||||
'udp_port',
|
||||
'tcp_port',
|
||||
'protocol_version',
|
||||
]
|
||||
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.hash_length:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
if not len(node_id) == constants.hash_length:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(node_id).decode()))
|
||||
if udp_port is not None and not 0 <= udp_port <= 65536:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
if tcp_port and not 0 <= tcp_port <= 65536:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_ipv4(self.address):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
if not is_valid_ipv4(address):
|
||||
raise ValueError("invalid ip address")
|
||||
self.loop = loop
|
||||
self._node_id = node_id
|
||||
self.address = address
|
||||
self.udp_port = udp_port
|
||||
self.tcp_port = tcp_port
|
||||
self.protocol_version = 1
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
self.tcp_port = tcp_port
|
||||
|
||||
def update_udp_port(self, udp_port: int):
|
||||
self.udp_port = udp_port
|
||||
|
||||
def set_id(self, node_id):
|
||||
if not self._node_id:
|
||||
self._node_id = node_id
|
||||
|
||||
@property
|
||||
def node_id(self) -> bytes:
|
||||
return self._node_id
|
||||
@@ -178,3 +188,11 @@ class KademliaPeer:
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, KademliaPeer):
|
||||
raise TypeError("invalid type to compare with Peer: %s" % str(type(other)))
|
||||
return (self.node_id, self.address, self.udp_port) == (other.node_id, other.address, other.udp_port)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.node_id, self.address, self.udp_port))
|
||||
@@ -7,7 +7,6 @@ import logging
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
@@ -151,7 +150,7 @@ class IterativeFinder:
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
|
||||
@@ -15,7 +15,6 @@ from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbry.dht.protocol.data_store import DictDataStore
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
@@ -323,9 +322,6 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
if not peer.node_id:
|
||||
log.warning("Tried adding a peer with no node id!")
|
||||
return False
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
@@ -451,7 +447,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
@@ -498,7 +494,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer = make_kademlia_peer(response_datagram.node_id, address[0], address[1])
|
||||
peer.set_id(response_datagram.node_id)
|
||||
peer.update_udp_port(address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
@@ -532,7 +529,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
@@ -542,7 +539,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
f"pending request: {str(remote_exception)}"
|
||||
log.warning(msg)
|
||||
else:
|
||||
log.debug("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], old_protocol_errors[error_datagram.response])
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
|
||||
@@ -646,8 +643,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return False
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer',
|
||||
retry: bool = True) -> typing.Tuple[bytes, bool]:
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
|
||||
async def __store():
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
@@ -659,15 +655,34 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, False
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
return peer.node_id, False
|
||||
except RemoteException as err:
|
||||
if 'Invalid token' not in str(err):
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
try:
|
||||
return await __store()
|
||||
except (ValueError, asyncio.TimeoutError, RemoteException):
|
||||
return peer.node_id, False
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
if not retry:
|
||||
return peer.node_id, False
|
||||
return await self.store_to_peer(hash_value, peer, retry=False)
|
||||
return peer.node_id, False
|
||||
|
||||
def _write(self, data: bytes, address: typing.Tuple[str, int]):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.sendto(data, address)
|
||||
except OSError as err:
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
# elif err.errno == socket.ENETUNREACH:
|
||||
# # this should probably try to retransmit when the network connection is back
|
||||
# log.error("Network is unreachable")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), address[0], address[1], str(err), err.errno)
|
||||
raise err
|
||||
else:
|
||||
raise TransportNotConnected()
|
||||
+11
-16
@@ -45,7 +45,7 @@ async def execute_command(conf, method, params, callback=display):
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
@@ -65,10 +65,10 @@ def remove_brackets(key):
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
k = None
|
||||
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
@@ -189,7 +189,7 @@ def get_argument_parser():
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_to_argparse(start)
|
||||
@@ -221,7 +221,7 @@ def ensure_directory_exists(path: str):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def setup_logging(args: argparse.Namespace, conf: Config, loop: asyncio.AbstractEventLoop):
|
||||
def run_daemon(args: list, conf: Config):
|
||||
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
conf.log_file_path, maxBytes=2097152, backupCount=5
|
||||
@@ -240,24 +240,19 @@ def setup_logging(args: argparse.Namespace, conf: Config, loop: asyncio.Abstract
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
log.setLevel(logging.INFO)
|
||||
if args.verbose is not None:
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
loop.set_debug(True)
|
||||
if len(args.verbose) > 0:
|
||||
for module in args.verbose:
|
||||
logging.getLogger(module).setLevel(logging.DEBUG)
|
||||
else:
|
||||
log.setLevel(logging.DEBUG)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
setup_logging(args, conf, loop)
|
||||
daemon = Daemon(conf)
|
||||
|
||||
def __exit():
|
||||
|
||||
@@ -252,6 +252,8 @@ class WalletComponent(Component):
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
|
||||
+498
-740
File diff suppressed because it is too large.
Load diff
@@ -6,8 +6,6 @@ from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.bip32 import PubKey
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.ledger import MainNetLedger, Account
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
@@ -69,13 +67,6 @@ def encode_account_doc():
|
||||
}
|
||||
|
||||
|
||||
def encode_wallet_doc():
|
||||
return {
|
||||
'id': 'wallet_id',
|
||||
'name': 'optional wallet name',
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
@@ -119,8 +110,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
@@ -129,8 +118,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -232,12 +219,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_wallet(self, wallet):
|
||||
return {
|
||||
'id': wallet.id,
|
||||
'name': wallet.name
|
||||
}
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
|
||||
@@ -3,7 +3,6 @@ from aiohttp.client_exceptions import ClientError
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
|
||||
|
||||
@@ -43,8 +42,6 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
@@ -52,17 +49,13 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record, retry=True):
|
||||
data = self.format(record).encode()
|
||||
async def _emit(self, record):
|
||||
try:
|
||||
async with self._session.post(self.url, data=data,
|
||||
cookies=self.cookies) as response:
|
||||
async with utils.aiohttp_request('post', self.url, data=self.format(record).encode(),
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
pass
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -52,6 +51,7 @@ def migrate_db(conf, start, end):
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
|
||||
@@ -125,9 +125,10 @@ def do_migration(conf):
|
||||
name_metadata_cursor.execute("select txid, n, sd_hash from name_metadata").fetchall()
|
||||
}
|
||||
|
||||
sd_hash_to_stream_hash = dict(
|
||||
sd_hash_to_stream_hash = {
|
||||
sd_hash: stream_hash for (sd_hash, stream_hash) in
|
||||
lbryfile_cursor.execute("select sd_blob_hash, stream_hash from lbry_file_descriptors").fetchall()
|
||||
)
|
||||
}
|
||||
|
||||
stream_hash_to_stream_blobs = {}
|
||||
|
||||
|
||||
@@ -101,9 +101,6 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
stream_hashes = tuple(
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
|
||||
)
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
@@ -111,7 +108,9 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", stream_hashes):
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -138,7 +137,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
tuple(signed_claims.keys())):
|
||||
list(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
@@ -148,35 +147,35 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs)
|
||||
).fetchall()
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
).fetchall()
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
@@ -192,7 +191,7 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
).fetchall()
|
||||
)
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
@@ -294,17 +293,17 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
[
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
]
|
||||
)
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
]
|
||||
)
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
@@ -318,9 +317,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
((int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
[(int(last_announced + (data_expiration / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes]
|
||||
)
|
||||
return self.db.run(_update_last_announced_blobs)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
@@ -331,11 +330,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1, next_announce_time=? "
|
||||
"where blob_hash=? and status='finished'", (int(now), blob_hash)
|
||||
).fetchall()
|
||||
)
|
||||
else:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
@@ -348,22 +347,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
).fetchall()
|
||||
)
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
).fetchall()
|
||||
return [b[0] for b in r]
|
||||
)
|
||||
return [b[0] for b in r.fetchall()]
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
@@ -371,18 +370,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
finished_blob_hashes = tuple(
|
||||
blob_hash for (blob_hash, ) in transaction.execute(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
).fetchall()
|
||||
)
|
||||
finished_blobs_set = set(finished_blob_hashes)
|
||||
to_update_set = finished_blobs_set.difference(blob_files)
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
((blob_hash, ) for blob_hash in to_update_set)
|
||||
).fetchall()
|
||||
return blob_files.intersection(finished_blobs_set)
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
@@ -475,12 +478,13 @@ class SQLiteStorage(SQLiteMixin):
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
removed.append((stream_hash,))
|
||||
else:
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
).fetchall()
|
||||
)
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
@@ -488,7 +492,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
@@ -497,22 +501,22 @@ class SQLiteStorage(SQLiteMixin):
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
@@ -533,13 +537,13 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
((stream_hash, ) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
@@ -551,16 +555,14 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(
|
||||
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
|
||||
).fetchall()
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -579,7 +581,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for support_info in _batched_select(
|
||||
transaction,
|
||||
"select * from support where claim_id in {}",
|
||||
claim_ids
|
||||
tuple(claim_ids)
|
||||
)
|
||||
]
|
||||
|
||||
@@ -610,7 +612,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
).fetchall()
|
||||
)
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
@@ -697,9 +699,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
# update the claim associated to the file
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
|
||||
).fetchall()
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
@@ -722,11 +722,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.execute_fetchall(
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
)
|
||||
return self.db.execute_fetchall(
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ def get_platform() -> dict:
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
import distro # pylint: disable=import-outside-toplevel
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
|
||||
@@ -139,15 +139,12 @@ types_map = {
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.flac': ('audio/flac', 'audio'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mkv': ('video/x-matroska', 'video'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
|
||||
@@ -4,10 +4,7 @@ from typing import NamedTuple, Tuple
|
||||
|
||||
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
# see https://spec.lbry.com/
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
@@ -20,7 +17,7 @@ def _create_url_regex():
|
||||
|
||||
def _claim(name, prefix=""):
|
||||
return _group(
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
|
||||
_oneof(
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
|
||||
@@ -71,7 +71,7 @@ class StreamDescriptor:
|
||||
self.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def length(self) -> int:
|
||||
def length(self):
|
||||
return len(self.as_json())
|
||||
|
||||
def get_stream_hash(self) -> str:
|
||||
@@ -114,7 +114,7 @@ class StreamDescriptor:
|
||||
])
|
||||
).encode()
|
||||
|
||||
def calculate_old_sort_sd_hash(self) -> str:
|
||||
def calculate_old_sort_sd_hash(self):
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
@@ -2,12 +2,11 @@ import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import DownloadSDTimeout
|
||||
from lbry.utils import resolve_host, lru_cache_concurrent
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.blob_exchange.downloader import BlobDownloader
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.dht.node import Node
|
||||
@@ -51,7 +50,7 @@ class StreamDownloader:
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1)
|
||||
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ from lbry.stream.downloader import StreamDownloader
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredStreamClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.schema.claim import Claim
|
||||
@@ -217,9 +215,6 @@ class ManagedStream:
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
"""
|
||||
Generate a stream from a file and save it to the db
|
||||
"""
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
@@ -294,13 +289,13 @@ class ManagedStream:
|
||||
await response.prepare(request)
|
||||
self.streaming_responses.append((request, response))
|
||||
self.streaming.set()
|
||||
wrote = 0
|
||||
try:
|
||||
wrote = 0
|
||||
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
|
||||
if not wrote:
|
||||
decrypted = decrypted[first_blob_start_offset:]
|
||||
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
|
||||
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * (MAX_BLOB_SIZE - 1))))
|
||||
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
|
||||
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
|
||||
len(self.descriptor.blobs) - 1)
|
||||
await response.write_eof(decrypted)
|
||||
@@ -313,9 +308,6 @@ class ManagedStream:
|
||||
if response._eof_sent:
|
||||
break
|
||||
return response
|
||||
except ConnectionResetError:
|
||||
log.warning("connection was reset after sending browser %i blob bytes", wrote)
|
||||
raise asyncio.CancelledError("range request transport was reset")
|
||||
finally:
|
||||
response.force_close()
|
||||
if (request, response) in self.streaming_responses:
|
||||
@@ -491,8 +483,8 @@ class ManagedStream:
|
||||
if end >= size:
|
||||
raise HTTPRequestRangeNotSatisfiable()
|
||||
|
||||
skip_blobs = start // (MAX_BLOB_SIZE - 2) # -2 because ... dont remember
|
||||
skip = skip_blobs * (MAX_BLOB_SIZE - 1) # -1 because
|
||||
skip_blobs = start // 2097150
|
||||
skip = skip_blobs * 2097151
|
||||
skip_first_blob = start - skip
|
||||
start = skip_first_blob + skip
|
||||
final_size = end - start + 1
|
||||
|
||||
@@ -401,8 +401,8 @@ class StreamManager:
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
|
||||
), 5) if self.config.max_key_fee else None
|
||||
if max_fee_amount and fee_amount > max_fee_amount:
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
@@ -442,7 +442,7 @@ class StreamManager:
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeout(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
except Exception as err: # forgive data timeout, dont delete stream
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
|
||||
@@ -151,11 +151,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
wallet_node.manager.old_db = daemon.storage
|
||||
return daemon
|
||||
|
||||
async def confirm_tx(self, txid, ledger=None):
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
await self.on_transaction_id(txid)
|
||||
return txid
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
|
||||
+2
-2
@@ -19,6 +19,7 @@ import collections
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -191,8 +192,7 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
return (await loop.getaddrinfo(
|
||||
url, port,
|
||||
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
|
||||
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM,
|
||||
family=socket.AF_INET
|
||||
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
|
||||
+28
-20
@@ -5,6 +5,7 @@ from hashlib import sha256
|
||||
from string import hexdigits
|
||||
|
||||
import ecdsa
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.constants import TXO_TYPES
|
||||
|
||||
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
|
||||
@@ -27,6 +28,7 @@ class Account(BaseAccount):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.channel_keys = {}
|
||||
self.preferences = {}
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
@@ -35,9 +37,10 @@ class Account(BaseAccount):
|
||||
h.update(cert.encode())
|
||||
return h.digest()
|
||||
|
||||
def merge(self, d: dict):
|
||||
super().merge(d)
|
||||
def apply(self, d: dict):
|
||||
super().apply(d)
|
||||
self.channel_keys.update(d.get('certificates', {}))
|
||||
self.preferences.update(d.get('preferences', {}))
|
||||
|
||||
def add_channel_private_key(self, private_key):
|
||||
public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
@@ -77,7 +80,7 @@ class Account(BaseAccount):
|
||||
constraints.update({'txo_type': 0})
|
||||
return super().get_balance(confirmations, **constraints)
|
||||
|
||||
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
|
||||
async def get_granular_balances(self, confirmations=0, reserved_subtotals=False):
|
||||
tips_balance, supports_balance, claims_balance = 0, 0, 0
|
||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
||||
total = await get_total_balance()
|
||||
@@ -97,13 +100,13 @@ class Account(BaseAccount):
|
||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
||||
)
|
||||
return {
|
||||
'total': total,
|
||||
'available': total - reserved,
|
||||
'reserved': reserved,
|
||||
'total': dewies_to_lbc(total),
|
||||
'available': dewies_to_lbc(total - reserved),
|
||||
'reserved': dewies_to_lbc(reserved),
|
||||
'reserved_subtotals': {
|
||||
'claims': claims_balance,
|
||||
'supports': supports_balance,
|
||||
'tips': tips_balance
|
||||
'claims': dewies_to_lbc(claims_balance),
|
||||
'supports': dewies_to_lbc(supports_balance),
|
||||
'tips': dewies_to_lbc(tips_balance)
|
||||
} if reserved_subtotals else None
|
||||
}
|
||||
|
||||
@@ -117,48 +120,53 @@ class Account(BaseAccount):
|
||||
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
|
||||
account = super().from_dict(ledger, wallet, d)
|
||||
account.channel_keys = d.get('certificates', {})
|
||||
account.preferences = d.get('preferences', {})
|
||||
return account
|
||||
|
||||
def to_dict(self, include_channel_keys=True):
|
||||
def to_dict(self, include_channel_keys=True, include_preferences=True):
|
||||
d = super().to_dict()
|
||||
if include_channel_keys:
|
||||
d['certificates'] = self.channel_keys
|
||||
if include_preferences and self.preferences:
|
||||
d['preferences'] = self.preferences
|
||||
return d
|
||||
|
||||
async def get_details(self, **kwargs):
|
||||
details = await super().get_details(**kwargs)
|
||||
details['certificates'] = len(self.channel_keys)
|
||||
if self.preferences:
|
||||
details['preferences'] = self.preferences
|
||||
return details
|
||||
|
||||
def get_transaction_history(self, **constraints):
|
||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_transaction_history(account=self, **constraints)
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_transaction_history_count(account=self, **constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_claims(account=self, **constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
return self.ledger.get_claim_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_claim_count(account=self, **constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
return self.ledger.get_streams(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_streams(account=self, **constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
return self.ledger.get_stream_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_stream_count(account=self, **constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
return self.ledger.get_channels(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_channels(account=self, **constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_channel_count(account=self, **constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_supports(account=self, **constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_support_count(account=self, **constraints)
|
||||
|
||||
def get_support_summary(self):
|
||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
||||
|
||||
@@ -22,18 +22,18 @@ class WalletDatabase(BaseDatabase):
|
||||
claim_id text,
|
||||
claim_name text
|
||||
);
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
create index if not exists txo_claim_id_idx on txo (claim_id);
|
||||
create index if not exists txo_txo_type_idx on txo (txo_type);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
BaseDatabase.PRAGMAS +
|
||||
BaseDatabase.CREATE_ACCOUNT_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXI_TABLE
|
||||
BaseDatabase.CREATE_TX_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
BaseDatabase.CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TXO_TABLE +
|
||||
BaseDatabase.CREATE_TXO_INDEX +
|
||||
BaseDatabase.CREATE_TXI_TABLE +
|
||||
BaseDatabase.CREATE_TXI_INDEX
|
||||
)
|
||||
|
||||
def txo_to_row(self, tx, address, txo):
|
||||
@@ -50,16 +50,18 @@ class WalletDatabase(BaseDatabase):
|
||||
row['claim_name'] = txo.claim_name
|
||||
return row
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints) -> List[Output]:
|
||||
txos = await super().get_txos(wallet=wallet, no_tx=no_tx, **constraints)
|
||||
async def get_txos(self, **constraints) -> List[Output]:
|
||||
my_accounts = constraints.get('my_accounts', constraints.get('accounts', []))
|
||||
|
||||
txos = await super().get_txos(**constraints)
|
||||
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and wallet:
|
||||
for account in wallet.accounts:
|
||||
if txo.claim.is_channel and my_accounts:
|
||||
for account in my_accounts:
|
||||
private_key = account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
@@ -71,7 +73,7 @@ class WalletDatabase(BaseDatabase):
|
||||
channels = {
|
||||
txo.claim_id: txo for txo in
|
||||
(await self.get_claims(
|
||||
wallet=wallet,
|
||||
my_accounts=my_accounts,
|
||||
claim_id__in=channel_ids
|
||||
))
|
||||
}
|
||||
@@ -132,20 +134,20 @@ class WalletDatabase(BaseDatabase):
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute_fetchall(
|
||||
await self.db.execute(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from account_address WHERE account = ?"
|
||||
" )", (account.public_key.address, )
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
)
|
||||
|
||||
def get_supports_summary(self, account_id):
|
||||
return self.db.execute_fetchall(f"""
|
||||
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
|
||||
(txo.txid in
|
||||
(select txi.txid from txi join account_address a on txi.address = a.address
|
||||
(select txi.txid from txi join pubkey_address a on txi.address = a.address
|
||||
where a.account = ?)) as from_me,
|
||||
(txo.address in (select address from account_address where account=?)) as to_me,
|
||||
(txo.address in (select address from pubkey_address where account=?)) as to_me,
|
||||
tx.height
|
||||
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
|
||||
""", (account_id, account_id))
|
||||
@@ -31,12 +31,3 @@ def lbc_to_dewies(lbc: str) -> int:
|
||||
|
||||
def dewies_to_lbc(dewies) -> str:
|
||||
return satoshis_to_coins(dewies)
|
||||
|
||||
|
||||
def dict_values_to_lbc(d):
|
||||
for key, value in d.items():
|
||||
if isinstance(value, int):
|
||||
d[key] = dewies_to_lbc(value)
|
||||
elif isinstance(value, dict):
|
||||
dict_values_to_lbc(value)
|
||||
return d
|
||||
+11
-20
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from functools import partial
|
||||
from typing import Tuple, List
|
||||
from datetime import datetime
|
||||
|
||||
@@ -61,8 +60,7 @@ class MainNetLedger(BaseLedger):
|
||||
return outputs.inflate(txs), outputs.offset, outputs.total
|
||||
|
||||
async def resolve(self, urls):
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
txos = (await self._inflate_outputs(resolve(urls)))[0]
|
||||
txos = (await self._inflate_outputs(self.network.resolve(urls)))[0]
|
||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||
result = {}
|
||||
for url, txo in zip(urls, txos):
|
||||
@@ -123,30 +121,39 @@ class MainNetLedger(BaseLedger):
|
||||
return super().get_utxo_count(**constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claims(**constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_claim_count(**constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_streams(**constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_stream_count(**constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channels(**constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_channel_count(**constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_supports(**constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_support_count(**constraints)
|
||||
|
||||
async def get_transaction_history(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
txs = await self.db.get_transactions(**constraints)
|
||||
headers = self.headers
|
||||
history = []
|
||||
@@ -242,25 +249,9 @@ class MainNetLedger(BaseLedger):
|
||||
return history
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
@staticmethod
|
||||
async def get_detailed_balance(accounts, confirmations=0, reserved_subtotals=False):
|
||||
result = {}
|
||||
for account in accounts:
|
||||
balance = await account.get_detailed_balance(confirmations, reserved_subtotals)
|
||||
if result:
|
||||
for key, value in balance.items():
|
||||
if key == 'reserved_subtotals':
|
||||
if value is not None:
|
||||
for subkey, subvalue in value.items():
|
||||
result['reserved_subtotals'][subkey] += subvalue
|
||||
else:
|
||||
result[key] += value
|
||||
else:
|
||||
result = balance
|
||||
return result
|
||||
|
||||
|
||||
class TestNetLedger(MainNetLedger):
|
||||
network_name = 'testnet'
|
||||
|
||||
+47
-12
@@ -26,8 +26,42 @@ class LbryWalletManager(BaseWalletManager):
|
||||
def db(self) -> WalletDatabase:
|
||||
return self.ledger.db
|
||||
|
||||
@property
|
||||
def use_encryption(self):
|
||||
return self.default_account.serialize_encrypted
|
||||
|
||||
@property
|
||||
def is_wallet_unlocked(self):
|
||||
return not self.default_account.encrypted
|
||||
|
||||
def check_locked(self):
|
||||
return self.default_wallet.is_locked
|
||||
return self.default_account.encrypted
|
||||
|
||||
def decrypt_account(self, account):
|
||||
assert account.password is not None, "account is not unlocked"
|
||||
assert not account.encrypted, "account is not unlocked"
|
||||
account.serialize_encrypted = False
|
||||
self.save()
|
||||
return not account.encrypted and not account.serialize_encrypted
|
||||
|
||||
def encrypt_account(self, password, account):
|
||||
assert not account.encrypted, "account is already encrypted"
|
||||
account.encrypt(password)
|
||||
account.serialize_encrypted = True
|
||||
self.save()
|
||||
self.unlock_account(password, account)
|
||||
return account.serialize_encrypted
|
||||
|
||||
def unlock_account(self, password, account):
|
||||
assert account.encrypted, "account is not locked"
|
||||
account.decrypt(password)
|
||||
return not account.encrypted
|
||||
|
||||
def lock_account(self, account):
|
||||
assert account.password is not None, "account is already locked"
|
||||
assert not account.encrypted and account.serialize_encrypted, "account is not encrypted"
|
||||
account.encrypt(account.password)
|
||||
return account.encrypted
|
||||
|
||||
@staticmethod
|
||||
def migrate_lbryum_to_torba(path):
|
||||
@@ -104,23 +138,20 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if not os.path.exists(wallets_directory):
|
||||
os.mkdir(wallets_directory)
|
||||
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(
|
||||
os.path.join(wallets_directory, 'default_wallet')
|
||||
)
|
||||
wallet_file_path = os.path.join(wallets_directory, 'default_wallet')
|
||||
|
||||
receiving_addresses, change_addresses = cls.migrate_lbryum_to_torba(wallet_file_path)
|
||||
|
||||
manager = cls.from_config({
|
||||
'ledgers': {ledger_id: ledger_config},
|
||||
'wallets': [
|
||||
os.path.join(wallets_directory, wallet_file) for wallet_file in settings.wallets
|
||||
]
|
||||
'wallets': [wallet_file_path]
|
||||
})
|
||||
ledger = manager.get_or_create_ledger(ledger_id)
|
||||
ledger.coin_selection_strategy = settings.coin_selection_strategy
|
||||
default_wallet = manager.default_wallet
|
||||
if default_wallet.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', default_wallet.id)
|
||||
default_wallet.generate_account(ledger)
|
||||
default_wallet.save()
|
||||
if manager.default_account is None:
|
||||
log.info('Wallet at %s is empty, generating a default account.', wallet_file_path)
|
||||
manager.default_wallet.generate_account(ledger)
|
||||
manager.default_wallet.save()
|
||||
if receiving_addresses or change_addresses:
|
||||
if not os.path.exists(ledger.path):
|
||||
os.mkdir(ledger.path)
|
||||
@@ -176,3 +207,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
tx = self.ledger.transaction_class(unhexlify(raw))
|
||||
await self.ledger.maybe_verify_transaction(tx, height)
|
||||
return tx
|
||||
|
||||
def save(self):
|
||||
for wallet in self.wallets:
|
||||
wallet.save()
|
||||
@@ -79,7 +79,7 @@ class LBC(Coin):
|
||||
|
||||
@classmethod
|
||||
def address_from_script(cls, script):
|
||||
'''Given a pk_script, return the address it pays to, or None.'''
|
||||
'''Given a pk_script, return the adddress it pays to, or None.'''
|
||||
return ScriptPubKey.pay_to(cls.address_handlers, script)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -2,7 +2,6 @@ import time
|
||||
import struct
|
||||
import sqlite3
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from typing import Tuple, List, Dict, Union, Type, Optional
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
@@ -337,7 +336,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
from math import sqrt
|
||||
|
||||
# TRENDING_WINDOW is the number of blocks in ~6hr period (21600 seconds / 161 seconds per block)
|
||||
TRENDING_WINDOW = 134
|
||||
|
||||
# TRENDING_DATA_POINTS says how many samples to use for the trending algorithm
|
||||
# i.e. only consider claims from the most recent (TRENDING_WINDOW * TRENDING_DATA_POINTS) blocks
|
||||
TRENDING_DATA_POINTS = 7
|
||||
TRENDING_WINDOW = 650 # number of blocks, ~24hr period
|
||||
TRENDING_DATA_POINTS = 7 # WINDOW * DATA_POINTS = ~1 week worth of trending data
|
||||
|
||||
CREATE_TREND_TABLE = """
|
||||
create table if not exists trend (
|
||||
@@ -30,7 +26,7 @@ class ZScore:
|
||||
if self.last is not None:
|
||||
self.count += 1
|
||||
self.total += self.last
|
||||
self.power += self.last ** 2
|
||||
self.power += self.last**2
|
||||
self.last = value
|
||||
|
||||
@property
|
||||
@@ -39,7 +35,7 @@ class ZScore:
|
||||
|
||||
@property
|
||||
def standard_deviation(self):
|
||||
return sqrt((self.power / self.count) - self.mean ** 2)
|
||||
return sqrt((self.power / self.count) - self.mean**2)
|
||||
|
||||
def finalize(self):
|
||||
if self.count == 0:
|
||||
@@ -51,19 +47,19 @@ def register_trending_functions(connection):
|
||||
connection.create_aggregate("zscore", 1, ZScore)
|
||||
|
||||
|
||||
def calculate_trending(db, height, final_height):
|
||||
def calculate_trending(db, height, is_first_sync, final_height):
|
||||
# don't start tracking until we're at the end of initial sync
|
||||
if height < (final_height - (TRENDING_WINDOW * TRENDING_DATA_POINTS)):
|
||||
if is_first_sync and height < (final_height - (TRENDING_WINDOW*TRENDING_DATA_POINTS)):
|
||||
return
|
||||
|
||||
if height % TRENDING_WINDOW != 0:
|
||||
return
|
||||
|
||||
db.execute(f"""
|
||||
DELETE FROM trend WHERE height < {height - (TRENDING_WINDOW * TRENDING_DATA_POINTS)}
|
||||
DELETE FROM trend WHERE height < {height-(TRENDING_WINDOW*TRENDING_DATA_POINTS)}
|
||||
""")
|
||||
|
||||
start = (height - TRENDING_WINDOW) + 1
|
||||
start = (height-TRENDING_WINDOW)+1
|
||||
db.execute(f"""
|
||||
INSERT OR IGNORE INTO trend (claim_hash, height, amount)
|
||||
SELECT claim_hash, {start}, COALESCE(
|
||||
|
||||
@@ -699,7 +699,7 @@ class SQLDB:
|
||||
update_claims, delete_claim_hashes, affected_channels, forward_timer=True)
|
||||
r(self.insert_supports, insert_supports)
|
||||
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
|
||||
r(calculate_trending, self.db, height, daemon_height)
|
||||
r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
|
||||
|
||||
|
||||
class LBRYDB(DB):
|
||||
|
||||
@@ -203,7 +203,7 @@ class Transaction(BaseTransaction):
|
||||
|
||||
@classmethod
|
||||
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
output = Output.pay_pubkey_hash(amount, ledger.address_to_hash160(address))
|
||||
return cls.create([], [output], funding_accounts, change_account)
|
||||
|
||||
@@ -211,7 +211,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_create(
|
||||
cls, name: str, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
claim_output = Output.pay_claim_name_pubkey_hash(
|
||||
amount, name, claim, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -223,7 +223,7 @@ class Transaction(BaseTransaction):
|
||||
def claim_update(
|
||||
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
claim, ledger.address_to_hash160(holding_address)
|
||||
@@ -239,7 +239,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
@@ -248,7 +248,7 @@ class Transaction(BaseTransaction):
|
||||
@classmethod
|
||||
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger, wallet = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
claim_output = Output.purchase_claim_pubkey_hash(
|
||||
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
|
||||
)
|
||||
|
||||
@@ -13,15 +13,13 @@ from lbry.extras.daemon.Daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from lbry.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
|
||||
encode_wallet_doc
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
|
||||
)
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
RETURN_DOCS = {
|
||||
'Account': encode_account_doc(),
|
||||
'Wallet': encode_wallet_doc(),
|
||||
'File': encode_file_doc(),
|
||||
'Transaction': encode_tx_doc(),
|
||||
'Output': encode_txo_doc(),
|
||||
@@ -105,13 +103,6 @@ class Examples(CommandTestCase):
|
||||
'preference', 'get'
|
||||
)
|
||||
|
||||
# wallets
|
||||
|
||||
await r(
|
||||
'List your wallets',
|
||||
'wallet', 'list'
|
||||
)
|
||||
|
||||
# accounts
|
||||
|
||||
await r(
|
||||
|
||||
@@ -58,7 +58,7 @@ async def main(cmd_args=None):
|
||||
start = time.perf_counter()
|
||||
kwargs = {
|
||||
'page': page,
|
||||
'claim_type': 'stream',
|
||||
# 'claim_type': 'stream',
|
||||
'order_by': ['trending_global', 'trending_mixed'],
|
||||
'no_totals': True
|
||||
}
|
||||
@@ -153,7 +153,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--allow_fees", action='store_true')
|
||||
parser.add_argument("--exit_on_error", action='store_true')
|
||||
parser.add_argument("--stall_download_timeout", default=5, type=int)
|
||||
parser.add_argument("--stall_download_timeout", default=0, type=int)
|
||||
parser.add_argument("--keep_files", action='store_true')
|
||||
parser.add_argument("--head_blob_only", action='store_true')
|
||||
parser.add_argument("--download_pages", type=int, default=10)
|
||||
|
||||
@@ -11,6 +11,14 @@ def extract(d, keys):
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
async def test_sqlite_binding_error(self):
|
||||
tasks = [
|
||||
self.loop.create_task(self.daemon.jsonrpc_account_create('second account' + str(x))) for x in range(100)
|
||||
]
|
||||
await asyncio.wait(tasks)
|
||||
for result in tasks:
|
||||
self.assertFalse(isinstance(result.result(), Exception))
|
||||
|
||||
async def test_account_list_set_create_remove_add(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
@@ -85,7 +93,7 @@ class AccountManagement(CommandTestCase):
|
||||
stream_list = self.daemon.jsonrpc_stream_list
|
||||
support_list = self.daemon.jsonrpc_support_list
|
||||
utxo_list = self.daemon.jsonrpc_utxo_list
|
||||
default_account = self.wallet.default_account
|
||||
default_account = self.daemon.default_account
|
||||
second_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
|
||||
tx = await self.daemon.jsonrpc_account_send(
|
||||
@@ -171,9 +179,3 @@ class AccountManagement(CommandTestCase):
|
||||
})
|
||||
self.assertEqual(history[6]['value'], '0.0')
|
||||
self.assertEqual(history[7]['value'], '10.0')
|
||||
|
||||
async def test_address_validation(self):
|
||||
address = await self.daemon.jsonrpc_address_unused()
|
||||
bad_address = address[0:20] + '9999999' + address[27:]
|
||||
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
|
||||
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
|
||||
@@ -466,63 +466,45 @@ class ChannelCommands(CommandTestCase):
|
||||
'title': 'foo', 'email': 'new@email.com'}
|
||||
)
|
||||
|
||||
# move channel to another account
|
||||
# send channel to someone else
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
# before moving
|
||||
# before sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 0)
|
||||
|
||||
other_address = await account2.receiving.get_or_create_usable_address()
|
||||
tx = await self.out(self.channel_update(claim_id, claim_address=other_address))
|
||||
|
||||
# after moving
|
||||
# after sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=self.account.id)), 2)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_channel_list(account_id=account2_id)), 1)
|
||||
|
||||
async def test_channel_export_import_before_sending_channel(self):
|
||||
# export
|
||||
# shoud not have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNone(txo.private_key)
|
||||
|
||||
# send the private key too
|
||||
private_key = self.account.get_channel_private_key(unhexlify(channel['public_key']))
|
||||
account2.add_channel_private_key(private_key)
|
||||
|
||||
# now should have private key
|
||||
txo = (await account2.get_channels())[0]
|
||||
self.assertIsNotNone(txo.private_key)
|
||||
|
||||
async def test_channel_export_import_into_new_account(self):
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
channel_private_key = (await self.account.get_channels())[0].private_key
|
||||
exported_data = await self.out(self.daemon.jsonrpc_channel_export(claim_id))
|
||||
|
||||
# import
|
||||
daemon2 = await self.add_daemon()
|
||||
self.assertEqual(0, len(await daemon2.jsonrpc_channel_list()))
|
||||
await daemon2.jsonrpc_channel_import(exported_data)
|
||||
channels = await daemon2.jsonrpc_channel_list()
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
# second wallet can't update until channel is sent to it
|
||||
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
# now send the channel as well
|
||||
await self.channel_update(claim_id, claim_address=await daemon2.jsonrpc_address_unused())
|
||||
|
||||
# second wallet should be able to update now
|
||||
await daemon2.jsonrpc_channel_update(claim_id, bid='0.5')
|
||||
|
||||
async def test_channel_update_across_accounts(self):
|
||||
account2 = await self.daemon.jsonrpc_account_create('second account')
|
||||
channel = await self.out(self.channel_create('@spam', '1.0', account_id=account2.id))
|
||||
# channel not in account1
|
||||
with self.assertRaisesRegex(Exception, "Can't find the channel"):
|
||||
await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=self.account.id)
|
||||
# channel is in account2
|
||||
await self.channel_update(self.get_claim_id(channel), bid='2.0', account_id=account2.id)
|
||||
result = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(result[0]['amount'], '2.0')
|
||||
# check all accounts for channel
|
||||
await self.channel_update(self.get_claim_id(channel), bid='3.0')
|
||||
result = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(result[0]['amount'], '3.0')
|
||||
await self.channel_abandon(self.get_claim_id(channel))
|
||||
|
||||
|
||||
class StreamCommands(ClaimTestCase):
|
||||
|
||||
@@ -580,26 +562,10 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(len(tx['outputs']), 1) # no change
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
|
||||
|
||||
async def test_stream_update_and_abandon_across_accounts(self):
|
||||
account2 = await self.daemon.jsonrpc_account_create('second account')
|
||||
stream = await self.out(self.stream_create('spam', '1.0', account_id=account2.id))
|
||||
# stream not in account1
|
||||
with self.assertRaisesRegex(Exception, "Can't find the stream"):
|
||||
await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=self.account.id)
|
||||
# stream is in account2
|
||||
await self.stream_update(self.get_claim_id(stream), bid='2.0', account_id=account2.id)
|
||||
result = await self.out(self.daemon.jsonrpc_stream_list())
|
||||
self.assertEqual(result[0]['amount'], '2.0')
|
||||
# check all accounts for stream
|
||||
await self.stream_update(self.get_claim_id(stream), bid='3.0')
|
||||
result = await self.out(self.daemon.jsonrpc_stream_list())
|
||||
self.assertEqual(result[0]['amount'], '3.0')
|
||||
await self.stream_abandon(self.get_claim_id(stream))
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_channel(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
|
||||
@@ -816,7 +782,7 @@ class StreamCommands(ClaimTestCase):
|
||||
|
||||
# send claim to someone else
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.wallet.get_account_or_error(new_account['id'])
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
|
||||
@@ -1113,12 +1079,13 @@ class StreamCommands(ClaimTestCase):
|
||||
class SupportCommands(CommandTestCase):
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
|
||||
account2 = wallet2.accounts[0]
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
|
||||
account2 = self.daemon.get_account_or_error(account2_id)
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(wallet_id='wallet2')
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.on_transaction_dict(result)
|
||||
|
||||
@@ -1136,7 +1103,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
claim_id, '1.0', True, account2_id, funding_account_ids=[account2_id])
|
||||
)
|
||||
await self.confirm_tx(tip['txid'])
|
||||
|
||||
@@ -1155,7 +1122,7 @@ class SupportCommands(CommandTestCase):
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
@@ -1167,7 +1134,7 @@ class SupportCommands(CommandTestCase):
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
claim_id, '2.0', False, account2_id, funding_account_ids=[account2_id])
|
||||
)
|
||||
await self.confirm_tx(support['txid'])
|
||||
|
||||
@@ -1176,7 +1143,7 @@ class SupportCommands(CommandTestCase):
|
||||
await self.assertBalance(account2, '1.999717')
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2'))
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import contextlib
|
||||
import asyncio
|
||||
import logging
|
||||
from io import StringIO
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
@@ -39,27 +37,3 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
cli.main(["--api", "localhost:5299", "status"])
|
||||
actual_output = actual_output.getvalue()
|
||||
self.assertIn("connection_status", actual_output)
|
||||
|
||||
def test_setup_logging(self):
|
||||
def setup(argv):
|
||||
parser = cli.get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
loop = asyncio.get_event_loop()
|
||||
conf = Config.create_from_arguments(args)
|
||||
cli.setup_logging(args, conf, loop)
|
||||
|
||||
setup(["start"])
|
||||
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
|
||||
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
|
||||
|
||||
setup(["start", "--verbose"])
|
||||
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
|
||||
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
|
||||
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
|
||||
|
||||
setup(["start", "--verbose", "lbry.extras", "lbry.wallet", "torba.client"])
|
||||
self.assertTrue(logging.getLogger("lbry.extras").isEnabledFor(logging.DEBUG))
|
||||
self.assertTrue(logging.getLogger("lbry.wallet").isEnabledFor(logging.DEBUG))
|
||||
self.assertTrue(logging.getLogger("torba.client").isEnabledFor(logging.DEBUG))
|
||||
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
|
||||
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
|
||||
@@ -3,7 +3,7 @@ from binascii import hexlify
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
bad_peers = []
|
||||
for candidate in self.nodes[1:10]:
|
||||
address, port, node_id = candidate.protocol.external_ip, candidate.protocol.udp_port, candidate.protocol.node_id
|
||||
peer = make_kademlia_peer(node_id, address, udp_port=port)
|
||||
peer = KademliaPeer(self.loop, address, node_id, port)
|
||||
bad_peers.append(peer)
|
||||
node.protocol.add_peer(peer)
|
||||
candidate.stop()
|
||||
@@ -70,7 +70,7 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
await asyncio.sleep(.1)
|
||||
timeout -= 1
|
||||
if not timeout:
|
||||
self.fail("node didn't join back after 2 seconds")
|
||||
self.fail("node didnt join back after 2 seconds")
|
||||
|
||||
async def test_announce_no_peers(self):
|
||||
await self.setup_network(1)
|
||||
@@ -100,9 +100,9 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
await self.setup_network(2, seed_nodes=2)
|
||||
node1, node2 = self.nodes
|
||||
node2.stop()
|
||||
# forcefully make it a bad peer but don't remove it from routing table
|
||||
# forcefully make it a bad peer but dont remove it from routing table
|
||||
address, port, node_id = node2.protocol.external_ip, node2.protocol.udp_port, node2.protocol.node_id
|
||||
peer = make_kademlia_peer(node_id, address, udp_port=port)
|
||||
peer = KademliaPeer(self.loop, address, node_id, port)
|
||||
self.assertTrue(node1.protocol.peer_manager.peer_is_good(peer))
|
||||
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
|
||||
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
|
||||
|
||||
@@ -98,7 +98,7 @@ class FileCommands(CommandTestCase):
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didn't create a file")
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didnt create a file")
|
||||
await self.server.blob_manager.delete_blobs([sd_hash])
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
@@ -118,7 +118,7 @@ class FileCommands(CommandTestCase):
|
||||
handle.write(b'some other stuff was there instead')
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
|
||||
file_info = self.sout(self.daemon.jsonrpc_file_list()[0])
|
||||
@@ -277,33 +277,6 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['amount'], '2.0')
|
||||
self.assertEqual(response['content_fee']['outputs'][0]['address'], target_address)
|
||||
|
||||
async def test_null_max_key_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
self.daemon.conf.max_key_fee = None
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
await self.assertBalance(self.account, '9.483893')
|
||||
response = await self.daemon.jsonrpc_get('lbry://somename')
|
||||
await self.ledger.wait(response.content_fee)
|
||||
await self.assertBalance(self.account, '8.483769')
|
||||
|
||||
# Assert the file downloads
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
# Assert the transaction is recorded to the blockchain
|
||||
starting_balance = await self.blockchain.get_balance()
|
||||
await self.generate(1)
|
||||
block_reward_and_claim_fee = 2.0
|
||||
self.assertEqual(
|
||||
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
|
||||
)
|
||||
|
||||
async def test_null_fee(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
tx = await self.stream_create(
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AddressManagement(CommandTestCase):
|
||||
|
||||
async def test_address_list(self):
|
||||
addresses = await self.out(self.daemon.jsonrpc_address_list())
|
||||
self.assertEqual(27, len(addresses))
|
||||
|
||||
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
|
||||
self.assertEqual(1, len(single))
|
||||
self.assertEqual(single[0], addresses[11])
|
||||
@@ -86,12 +86,6 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
self.assertEqual(claim['name'], 'gibberish')
|
||||
self.assertNotIn('value', claim)
|
||||
|
||||
# resolve retries
|
||||
await self.conductor.spv_node.stop()
|
||||
resolving_future = asyncio.ensure_future(self.resolve('foo'))
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
|
||||
|
||||
async def test_winning_by_effective_amount(self):
|
||||
# first one remains winner unless something else changes
|
||||
claim_id1 = self.get_claim_id(
|
||||
@@ -178,7 +172,7 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
|
||||
orphan_claim_id = self.get_claim_id(orphan_claim)
|
||||
|
||||
# Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.resolve('lbry://@abc/on-channel-claim')
|
||||
self.assertEqual(response, {
|
||||
|
||||
@@ -413,7 +413,7 @@ class RangeRequestsLRUCache(CommandTestCase):
|
||||
self.server.stop_server()
|
||||
|
||||
# running with cache size 0 gets through without errors without
|
||||
# this since the server doesn't stop immediately
|
||||
# this since the server doesnt stop immediately
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
await self._request_stream()
|
||||
@@ -1,72 +1,114 @@
|
||||
import asyncio
|
||||
from lbry.testcase import CommandTestCase
|
||||
from binascii import unhexlify
|
||||
from unittest import mock
|
||||
|
||||
from torba.orchstr8.node import WalletNode, SPVNode
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet import LbryWalletManager, RegTestLedger
|
||||
from lbry.extras.daemon.Daemon import Daemon
|
||||
from lbry.extras.daemon.Components import WalletComponent
|
||||
from lbry.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
|
||||
class WalletSynchronization(CommandTestCase):
|
||||
SEED = "carbon smart garage balance margin twelve chest sword toast envelope bottom stomach absent"
|
||||
class AccountSynchronization(AsyncioTestCase):
|
||||
|
||||
async def test_sync(self):
|
||||
daemon = self.daemon
|
||||
daemon2 = await self.add_daemon(
|
||||
seed="chest sword toast envelope bottom stomach absent "
|
||||
"carbon smart garage balance margin twelve"
|
||||
async def asyncSetUp(self):
|
||||
self.wallet_node = WalletNode(LbryWalletManager, RegTestLedger)
|
||||
await self.wallet_node.start(
|
||||
SPVNode(None),
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach absent",
|
||||
False
|
||||
)
|
||||
address = (await daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 1)
|
||||
await self.confirm_tx(sendtxid, daemon2.ledger)
|
||||
self.account = self.wallet_node.account
|
||||
|
||||
# Preferences
|
||||
self.assertFalse(daemon.jsonrpc_preference_get())
|
||||
self.assertFalse(daemon2.jsonrpc_preference_get())
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.reflector_servers = []
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
daemon.jsonrpc_preference_set("one", "1")
|
||||
daemon.jsonrpc_preference_set("conflict", "1")
|
||||
daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]')
|
||||
await asyncio.sleep(1)
|
||||
daemon2.jsonrpc_preference_set("two", "2")
|
||||
daemon2.jsonrpc_preference_set("conflict", "2")
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.wallet_node.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
self.assertDictEqual(daemon.jsonrpc_preference_get(), {
|
||||
"one": "1", "conflict": "1", "fruit": ["peach", "apricot"]
|
||||
})
|
||||
self.assertDictEqual(daemon2.jsonrpc_preference_get(), {"two": "2", "conflict": "2"})
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
|
||||
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 1)
|
||||
async def asyncTearDown(self):
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop(shutdown_runner=False)
|
||||
|
||||
daemon2.jsonrpc_wallet_encrypt('password')
|
||||
daemon2.jsonrpc_wallet_lock()
|
||||
with self.assertRaises(AssertionError):
|
||||
await daemon2.jsonrpc_sync_apply('password')
|
||||
@mock.patch('time.time', mock.Mock(return_value=12345))
|
||||
def test_sync(self):
|
||||
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
|
||||
self.account.modified_on = 123.456
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
|
||||
self.assertFalse(self.account.channel_keys)
|
||||
|
||||
daemon2.jsonrpc_wallet_unlock('password')
|
||||
data = await daemon2.jsonrpc_sync_apply('password')
|
||||
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
|
||||
|
||||
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 2)
|
||||
self.assertDictEqual(
|
||||
# "two" key added and "conflict" value changed to "2"
|
||||
daemon.jsonrpc_preference_get(),
|
||||
{"one": "1", "two": "2", "conflict": "2", "fruit": ["peach", "apricot"]}
|
||||
hash_w_cert = '974721f42dab42657b5911b7caf4af98ce4d3879eea6ac23d50c1d79bc5020ef'
|
||||
add_cert = (
|
||||
'czo4MTkyOjE2OjE6qs3JRvS/bhX8p1JD68sqyA2Qhx3EVTqskhqEAwtfAUfUsQqeJ1rtMdRf40vkGKnpt4NT0b'
|
||||
'XEqb5O+lba4nkLF7vZENhc2zuOrjobCPVbiVHNwOfH56Ayrh1ts5LMcnl5+Mk1BUyGCwXcqEg2KiUkd3YZpiHQ'
|
||||
'T7WfcODcU6l7IRivb8iawCebZJx9waVyQoqEDKwZUY1i5HA0VLC+s5cV7it1AWbewiyWOQtZdEPzNY44oXLJex'
|
||||
'SirElQqDqNZyl3Hjy8YqacBbSYoejIRnmXpC9y25keP6hep3f9i1K2HDNwhwns1W1vhuzuO2Gy9+a0JlVm5mwc'
|
||||
'N2pqO4tCZr6tE3aym2FaSAunOi7QYVFMI6arb9Gvn9P+T+WRiFYfzwDFVR+j5ZPmUDXxHisy5OF163jH61wbBY'
|
||||
'pPienjlVtDOxoZmA8+AwWXKRdINsRcull9pu7EVCq5yQmrmxoPbLxNh5pRGrBB0JwCCOMIf+KPwS+7Z6dDbiwO'
|
||||
'2NUpk8USJMTmXmFDCr2B0PJiG6Od2dD2oGN0F7aYZvUuKbqj8eDrJMe/zLbhq47jUjkJFCvtxUioo63ORk1pzH'
|
||||
'S0/X4/6/95PRSMaXm4DcZ9BdyxR2E/AKc8UN6AL5rrn6quXkC6R3ZhKgN3Si2S9y6EGFsL7dgzX331U08ZviLj'
|
||||
'NsrG0EKUnf+TGQ42MqnLQBOiO/ZoAwleOzNZnCYOQQ14Mm8y17xUpmdWRDiRKpAOJU22jKnxtqQ='
|
||||
)
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
|
||||
self.assertEqual(self.account.channel_keys, {'abcdefg1234:0': '---PRIVATE KEY---'})
|
||||
|
||||
# Channel Certificate
|
||||
channel = await daemon2.jsonrpc_channel_create('@foo', '0.1')
|
||||
await daemon2.ledger.wait(channel)
|
||||
await self.generate(1)
|
||||
await daemon2.ledger.wait(channel)
|
||||
# applying the same diff is idempotent
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
|
||||
self.assertEqual(self.account.channel_keys, {'abcdefg1234:0': '---PRIVATE KEY---'})
|
||||
|
||||
# both daemons will have the channel but only one has the cert so far
|
||||
self.assertEqual(len(await daemon.jsonrpc_channel_list()), 1)
|
||||
self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
|
||||
self.assertEqual(len(await daemon2.jsonrpc_channel_list()), 1)
|
||||
self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
|
||||
@mock.patch('time.time', mock.Mock(return_value=12345))
|
||||
def test_account_preferences_syncing(self):
|
||||
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
|
||||
self.account.modified_on = 123.456
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
|
||||
self.assertFalse(self.daemon.jsonrpc_preference_get())
|
||||
|
||||
data = await daemon2.jsonrpc_sync_apply('password')
|
||||
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
|
||||
|
||||
# both daemons have the cert after sync'ing
|
||||
self.assertEqual(
|
||||
daemon2.wallet_manager.default_account.channel_keys,
|
||||
daemon.wallet_manager.default_wallet.accounts[1].channel_keys
|
||||
hash_w_pref = '2fe43f0b2f8bbf1fbb55537f862d8bcb0823791019a7151c848bd5f5bd32d336'
|
||||
add_pref = (
|
||||
'czo4MTkyOjE2OjE6Jgn3nAGrfYP2usMA4KQ/73+YHAwMyiGdSWuxCmgZKlpwSpnfQv8R7R0tum/n2oTSBQxjdL'
|
||||
'OlTW+tv/G5L2GfQ5op3xaT89gN+F/JJnvf3cdWvYH7Nc+uTUMb7cKhJP7hQvFW5bb1Y3jX3EBBY00Jkqyj9RCR'
|
||||
'XPtbLVu71KbVRvCAR/oAnMEsgD+ITsC3WkXMwE3BS2LjJDQmeqbH4YXNdcjJN/JzQ6fxOmr3Uk1GqnpuhFsta8'
|
||||
'H14ViRilq1pLKOSZIN80rrm5cKq45nFO5kFeoqBCEaal4u2/OkX9nOnpQlO3E95wD8hkCmZ3i20aSte6nqwqXx'
|
||||
'ZKVRZqR2a0TjwVWB8kPXPA2ewKvPILaj190bXPl8EVu+TAnTCQwMgytinYjtcKNZmMz3ENJyI2mCANwpWlX7xl'
|
||||
'y/J+qLi5b9N+agghTxggs5rVJ/hkaue7GS542dXDrwMrw9nwGqNw3dS/lcU+1wRUQ0fnHwb/85XbbwyO2aDj2i'
|
||||
'DFNkdyLyUIiIUvB1JfWAnWqX3vQcL1REK1ePgUei7dCHJ3WyWdsRx3cVXzlK8yOPkf0N6d3AKrZQWVebwDC7Nd'
|
||||
'eL4sDW8AkaXuBIrbuZw6XUHd6WI0NvU/q10j2qMm0YoXSu+dExou1/1THwx5g86MxcX5nwodKUEVCOTzKMyrLz'
|
||||
'CRsitH/+dAXhZNRp/FbnDCGBMyD3MOYCjZvAFbCZUasoRwqponxILw=='
|
||||
)
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_pref)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
|
||||
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["apple", "orange"]})
|
||||
|
||||
self.daemon.jsonrpc_preference_set("fruit", ["peach", "apricot"])
|
||||
self.assertEqual(self.daemon.jsonrpc_preference_get(), {"fruit": ["peach", "apricot"]})
|
||||
self.assertNotEqual(self.daemon.jsonrpc_sync_hash(), hash_w_pref)
|
||||
@@ -38,42 +38,34 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
|
||||
async def test_granular_balances(self):
|
||||
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
|
||||
|
||||
account_balance = self.daemon.jsonrpc_account_balance
|
||||
wallet_balance = self.daemon.jsonrpc_wallet_balance
|
||||
|
||||
expected = {
|
||||
self.assertEqual(await account_balance(reserved_subtotals=False), {
|
||||
'total': '10.0',
|
||||
'available': '10.0',
|
||||
'reserved': '0.0',
|
||||
'reserved_subtotals': None
|
||||
}
|
||||
self.assertEqual(await account_balance(reserved_subtotals=False), expected)
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=False), expected)
|
||||
})
|
||||
|
||||
expected = {
|
||||
self.assertEqual(await account_balance(reserved_subtotals=True), {
|
||||
'total': '10.0',
|
||||
'available': '10.0',
|
||||
'reserved': '0.0',
|
||||
'reserved_subtotals': {'claims': '0.0', 'supports': '0.0', 'tips': '0.0'}
|
||||
}
|
||||
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
|
||||
})
|
||||
|
||||
# claim with update + supporting our own claim
|
||||
stream1 = await self.stream_create('granularity', '3.0')
|
||||
await self.stream_update(self.get_claim_id(stream1), data=b'news', bid='1.0')
|
||||
await self.support_create(self.get_claim_id(stream1), '2.0')
|
||||
expected = {
|
||||
self.assertEqual(await account_balance(reserved_subtotals=True), {
|
||||
'total': '9.977534',
|
||||
'available': '6.977534',
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
}
|
||||
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
|
||||
})
|
||||
|
||||
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
|
||||
address2 = await self.daemon.jsonrpc_address_unused(account2.id)
|
||||
|
||||
# send lbc to someone else
|
||||
@@ -85,12 +77,6 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
})
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
|
||||
'total': '9.97741',
|
||||
'available': '6.97741',
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
})
|
||||
|
||||
# tip received
|
||||
support1 = await self.support_create(
|
||||
@@ -102,12 +88,6 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
'reserved': '3.3',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
|
||||
})
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
|
||||
'total': '9.977268',
|
||||
'available': '6.677268',
|
||||
'reserved': '3.3',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
|
||||
})
|
||||
|
||||
# tip claimed
|
||||
tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0)
|
||||
@@ -118,12 +98,6 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
})
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
|
||||
'total': '9.977161',
|
||||
'available': '6.977161',
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
})
|
||||
|
||||
stream2 = await self.stream_create(
|
||||
'granularity-is-cool', '0.1', account_id=account2.id, funding_account_ids=[account2.id]
|
||||
@@ -131,17 +105,10 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
# tip another claim
|
||||
await self.support_create(
|
||||
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id]
|
||||
)
|
||||
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id])
|
||||
self.assertEqual(await account_balance(reserved_subtotals=True), {
|
||||
'total': '9.077157',
|
||||
'available': '6.077157',
|
||||
'reserved': '3.0',
|
||||
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
|
||||
})
|
||||
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
|
||||
'total': '9.938908',
|
||||
'available': '6.638908',
|
||||
'reserved': '3.3',
|
||||
'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'}
|
||||
})
|
||||
@@ -55,7 +55,7 @@ class TestHeadersComponent(CommandTestCase):
|
||||
async def test_cant_reach_host(self):
|
||||
HeadersComponent.HEADERS_URL = 'notthere/'
|
||||
os.unlink(self.headers_component.headers.path)
|
||||
# test is that this doesn't raise
|
||||
# test is that this doesnt raise
|
||||
await self.headers_component.start()
|
||||
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
|
||||
self.assertEqual(await self.headers_component.get_status(), {})
|
||||
@@ -12,8 +12,7 @@ from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbry.blob_exchange.client import request_blob
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
|
||||
from lbry.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
# import logging
|
||||
# logging.getLogger("lbry").setLevel(logging.DEBUG)
|
||||
@@ -44,7 +43,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
await self.client_storage.open()
|
||||
await self.server_storage.open()
|
||||
@@ -103,7 +102,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
await second_client_storage.open()
|
||||
await second_client_blob_manager.setup()
|
||||
@@ -194,7 +193,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
await second_client_storage.open()
|
||||
await second_client_blob_manager.setup()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
from unittest import mock, TestCase
|
||||
from lbry.dht.protocol.data_store import DictDataStore
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
|
||||
class DataStoreTests(TestCase):
|
||||
@@ -13,7 +13,7 @@ class DataStoreTests(TestCase):
|
||||
|
||||
def _test_add_peer_to_blob(self, blob=b'2' * 48, node_id=b'1' * 48, address='1.2.3.4', tcp_port=3333,
|
||||
udp_port=4444):
|
||||
peer = make_kademlia_peer(node_id, address, udp_port)
|
||||
peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
peer.update_tcp_port(tcp_port)
|
||||
before = self.data_store.get_peers_for_blob(blob)
|
||||
self.data_store.add_peer_to_blob(peer, blob)
|
||||
|
||||
@@ -2,7 +2,7 @@ import struct
|
||||
import asyncio
|
||||
from lbry.utils import generate_id
|
||||
from lbry.dht.protocol.routing_table import KBucket
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
from lbry.dht import constants
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
@@ -29,8 +29,8 @@ class TestKBucket(AsyncioTestCase):
|
||||
self.kbucket = KBucket(self.peer_manager, 0, 2**constants.hash_bits, generate_id())
|
||||
|
||||
def test_add_peer(self):
|
||||
peer = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4444)
|
||||
peer_update2 = make_kademlia_peer(constants.generate_id(2), "1.2.3.4", udp_port=4445)
|
||||
peer = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4444)
|
||||
peer_update2 = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4445)
|
||||
|
||||
self.assertListEqual([], self.kbucket.peers)
|
||||
|
||||
@@ -59,12 +59,12 @@ class TestKBucket(AsyncioTestCase):
|
||||
# Test if contacts can be added to empty list
|
||||
# Add k contacts to bucket
|
||||
for i in range(constants.k):
|
||||
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
self.assertTrue(self.kbucket.add_peer(peer))
|
||||
self.assertEqual(peer, self.kbucket.peers[i])
|
||||
|
||||
# Test if contact is not added to full list
|
||||
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
self.assertFalse(self.kbucket.add_peer(peer))
|
||||
|
||||
# Test if an existing contact is updated correctly if added again
|
||||
@@ -125,13 +125,13 @@ class TestKBucket(AsyncioTestCase):
|
||||
|
||||
def test_remove_peer(self):
|
||||
# try remove contact from empty list
|
||||
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
self.assertRaises(ValueError, self.kbucket.remove_peer, peer)
|
||||
|
||||
added = []
|
||||
# Add couple contacts
|
||||
for i in range(constants.k-2):
|
||||
peer = make_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
peer = self.peer_manager.get_kademlia_peer(generate_id(), next(self.address_generator), 4444)
|
||||
self.assertTrue(self.kbucket.add_peer(peer))
|
||||
added.append(peer)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from tests import dht_mocks
|
||||
from lbry.dht.serialization.bencoding import bencode, bdecode
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
|
||||
class TestProtocol(AsyncioTestCase):
|
||||
@@ -22,7 +22,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
|
||||
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
|
||||
|
||||
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
result = await peer2.get_rpc_peer(peer).ping()
|
||||
self.assertEqual(result, b'pong')
|
||||
peer1.stop()
|
||||
@@ -43,7 +43,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
|
||||
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
|
||||
|
||||
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
self.assertEqual(None, peer2.peer_manager.get_node_token(peer.node_id))
|
||||
await peer2.get_rpc_peer(peer).find_value(b'1' * 48)
|
||||
self.assertNotEqual(None, peer2.peer_manager.get_node_token(peer.node_id))
|
||||
@@ -65,12 +65,12 @@ class TestProtocol(AsyncioTestCase):
|
||||
await loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
|
||||
await loop.create_datagram_endpoint(lambda: peer2, ('1.2.3.5', 4444))
|
||||
|
||||
peer = make_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
peer2_from_peer1 = make_kademlia_peer(
|
||||
peer = peer2.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', udp_port=4444)
|
||||
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
|
||||
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
|
||||
)
|
||||
peer2_from_peer1.update_tcp_port(3333)
|
||||
peer3 = make_kademlia_peer(
|
||||
peer3 = peer1.peer_manager.get_kademlia_peer(
|
||||
constants.generate_id(), '1.2.3.6', udp_port=4444
|
||||
)
|
||||
store_result = await peer2.store_to_peer(b'2' * 48, peer)
|
||||
@@ -103,7 +103,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
)
|
||||
await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444))
|
||||
proto.start()
|
||||
return proto, make_kademlia_peer(node_id, address, udp_port=udp_port)
|
||||
return proto, other_peer.peer_manager.get_kademlia_peer(node_id, address, udp_port=udp_port)
|
||||
|
||||
async def test_add_peer_after_handle_request(self):
|
||||
with dht_mocks.mock_network_loop(self.loop):
|
||||
@@ -129,7 +129,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer1.routing_table.remove_peer(peer_2_from_peer_1)
|
||||
|
||||
# peers not known by be good/bad should be enqueued to maybe-ping
|
||||
peer1_from_peer3 = peer3.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1_from_peer3 = peer3.get_rpc_peer(peer3.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
pong = await peer1_from_peer3.ping()
|
||||
self.assertEqual(b'pong', pong)
|
||||
@@ -137,7 +137,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer1.ping_queue._pending_contacts.clear()
|
||||
|
||||
# peers who are already good should be added
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_last_replied('1.2.3.7', 4444)
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
@@ -149,7 +149,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer1.routing_table.buckets[0].peers.clear()
|
||||
|
||||
# peers who are known to be bad recently should not be added or maybe-pinged
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(make_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_failure('1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_failure('1.2.3.7', 4444)
|
||||
|
||||
@@ -3,7 +3,8 @@ from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
|
||||
expected_ranges = [
|
||||
(
|
||||
@@ -28,7 +29,6 @@ expected_ranges = [
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestRouting(AsyncioTestCase):
|
||||
async def test_fill_one_bucket(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
@@ -53,7 +53,7 @@ class TestRouting(AsyncioTestCase):
|
||||
for i in range(1, len(peer_addresses)):
|
||||
self.assertEqual(len(node_1.protocol.routing_table.get_peers()), contact_cnt)
|
||||
node = nodes[i]
|
||||
peer = make_kademlia_peer(
|
||||
peer = node_1.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip,
|
||||
udp_port=node.protocol.udp_port
|
||||
)
|
||||
@@ -66,16 +66,6 @@ class TestRouting(AsyncioTestCase):
|
||||
for node in nodes.values():
|
||||
node.protocol.stop()
|
||||
|
||||
async def test_cant_add_peer_without_a_node_id_gracefully(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
node = Node(loop, PeerManager(loop), constants.generate_id(), 4444, 4444, 3333, '1.2.3.4')
|
||||
bad_peer = make_kademlia_peer(None, '1.2.3.4', 5555)
|
||||
with self.assertLogs(level='WARNING') as logged:
|
||||
self.assertFalse(await node.protocol._add_peer(bad_peer))
|
||||
self.assertEqual(1, len(logged.output))
|
||||
self.assertTrue(logged.output[0].endswith('Tried adding a peer with no node id!'))
|
||||
|
||||
|
||||
async def test_split_buckets(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
peer_addresses = [
|
||||
@@ -91,7 +81,7 @@ class TestRouting(AsyncioTestCase):
|
||||
node_1 = nodes[0]
|
||||
for i in range(1, len(peer_addresses)):
|
||||
node = nodes[i]
|
||||
peer = make_kademlia_peer(
|
||||
peer = node_1.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip,
|
||||
udp_port=node.protocol.udp_port
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ from tests import dht_mocks
|
||||
from lbry.conf import Config
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
@@ -35,7 +35,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
self.nodes.update({len(self.nodes): n})
|
||||
if add_to_routing_table:
|
||||
self.node.protocol.add_peer(
|
||||
make_kademlia_peer(
|
||||
self.peer_manager.get_kademlia_peer(
|
||||
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
|
||||
)
|
||||
)
|
||||
@@ -67,7 +67,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
await self.add_peer(node_id, address, False)
|
||||
last_node = self.nodes[len(self.nodes) - 1]
|
||||
peer = last_node.protocol.get_rpc_peer(
|
||||
make_kademlia_peer(
|
||||
last_node.protocol.peer_manager.get_kademlia_peer(
|
||||
previous_last_node.protocol.node_id, previous_last_node.protocol.external_ip,
|
||||
previous_last_node.protocol.udp_port
|
||||
)
|
||||
@@ -129,12 +129,12 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
announced_to = self.nodes[0]
|
||||
for i in range(1, peer_count):
|
||||
node = self.nodes[i]
|
||||
kad_peer = make_kademlia_peer(
|
||||
kad_peer = announced_to.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
|
||||
)
|
||||
await announced_to.protocol._add_peer(kad_peer)
|
||||
peer = node.protocol.get_rpc_peer(
|
||||
make_kademlia_peer(
|
||||
node.protocol.peer_manager.get_kademlia_peer(
|
||||
announced_to.protocol.node_id,
|
||||
announced_to.protocol.external_ip,
|
||||
announced_to.protocol.udp_port
|
||||
|
||||
@@ -4,7 +4,7 @@ from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
|
||||
class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
@@ -42,7 +42,7 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
for i in range(1, len(peer_addresses)):
|
||||
node = nodes[i]
|
||||
assert node.protocol.node_id != node_1.protocol.node_id
|
||||
peer = make_kademlia_peer(
|
||||
peer = node_1.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip, udp_port=node.protocol.udp_port
|
||||
)
|
||||
futs.append(node_1.protocol.get_rpc_peer(peer).ping())
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
from lbry.utils import generate_id
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.peer import PeerManager
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
|
||||
@@ -10,30 +10,20 @@ class PeerTest(AsyncioTestCase):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.peer_manager = PeerManager(self.loop)
|
||||
self.node_ids = [generate_id(), generate_id(), generate_id()]
|
||||
self.first_contact = make_kademlia_peer(self.node_ids[1], '127.0.0.1', udp_port=1000)
|
||||
self.second_contact = make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
|
||||
def test_peer_is_good_unknown_peer(self):
|
||||
# Scenario: peer replied, but caller doesn't know the node_id.
|
||||
# Outcome: We can't say it's good or bad.
|
||||
# (yes, we COULD tell the node id, but not here. It would be
|
||||
# a side effect and the caller is responsible to discover it)
|
||||
peer = make_kademlia_peer(None, '1.2.3.4', 4444)
|
||||
self.peer_manager.report_last_requested('1.2.3.4', 4444)
|
||||
self.peer_manager.report_last_replied('1.2.3.4', 4444)
|
||||
self.assertIsNone(self.peer_manager.peer_is_good(peer))
|
||||
self.first_contact = self.peer_manager.get_kademlia_peer(self.node_ids[1], '127.0.0.1', udp_port=1000)
|
||||
self.second_contact = self.peer_manager.get_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
|
||||
def test_make_contact_error_cases(self):
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', 100000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], 'this is not an ip', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', -1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, b'not valid node id', '192.168.1.20', 1000)
|
||||
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20', 100000)
|
||||
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20.1', 1000)
|
||||
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], 'this is not an ip', 1000)
|
||||
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, self.node_ids[1], '192.168.1.20', -1000)
|
||||
self.assertRaises(ValueError, self.peer_manager.get_kademlia_peer, b'not valid node id', '192.168.1.20', 1000)
|
||||
|
||||
def test_boolean(self):
|
||||
self.assertNotEqual(self.first_contact, self.second_contact)
|
||||
self.assertEqual(
|
||||
self.second_contact, make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
self.second_contact, self.peer_manager.get_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
)
|
||||
|
||||
def test_compact_ip(self):
|
||||
|
||||
@@ -55,41 +55,10 @@ class TestURLParsing(unittest.TestCase):
|
||||
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
|
||||
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
|
||||
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
|
||||
# unicode regex edges
|
||||
_url = lambda name: url(name, stream_name=name)
|
||||
_url('\u0009')
|
||||
_url('\u000a')
|
||||
_url('\u000d')
|
||||
_url('\u0020')
|
||||
_url('\uD799')
|
||||
_url('\uE000')
|
||||
_url('\uFFFD')
|
||||
|
||||
def test_parser_invalid_urls(self):
|
||||
fail = self._fail_url
|
||||
fail("lbry://")
|
||||
fail("lbry://\u0000")
|
||||
fail("lbry://\u0008")
|
||||
fail("lbry://\u000b")
|
||||
fail("lbry://\u000c")
|
||||
fail("lbry://\u000e")
|
||||
fail("lbry://\u001f")
|
||||
fail("lbry://\uD800")
|
||||
fail("lbry://\uDFFF")
|
||||
fail("lbry://\uDFFE")
|
||||
fail("lbry://\uFFFF")
|
||||
fail("lbry://;")
|
||||
fail("lbry://\"")
|
||||
fail("lbry://\\")
|
||||
fail("lbry:///")
|
||||
fail("lbry://<") and fail("lbry://>")
|
||||
fail("lbry://{") and fail("lbry://}")
|
||||
fail("lbry://[") and fail("lbry://]")
|
||||
fail("lbry://%")
|
||||
fail("lbry://|")
|
||||
fail("lbry://^")
|
||||
fail("lbry://~")
|
||||
fail("lbry://`")
|
||||
fail("lbry://test:3$1")
|
||||
fail("lbry://test$1:1")
|
||||
fail("lbry://test#x")
|
||||
|
||||
@@ -7,7 +7,8 @@ from lbry.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbry.blob_exchange.serialization import BlobResponse
|
||||
from lbry.blob_exchange.server import BlobServerProtocol
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
from lbry.extras.daemon.storage import StoredStreamClaim
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
@@ -94,7 +95,7 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334)
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
|
||||
@@ -126,5 +126,5 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
loop, tmp_dir, blob
|
||||
)
|
||||
self.assertFalse(blob.file_exists)
|
||||
# fixme: this is an emergency PR, please move this to blob_file tests later
|
||||
# fixme: this is an emergency PR, plase move this to blob_file tests later
|
||||
self.assertIsNone(blob.length)
|
||||
@@ -51,7 +51,6 @@ class CLITest(AsyncioTestCase):
|
||||
self.assertEqual('3', normalize_value('3', key="name"))
|
||||
self.assertEqual('3', normalize_value('3', key="download_directory"))
|
||||
self.assertEqual('3', normalize_value('3', key="channel_name"))
|
||||
self.assertEqual('3', normalize_value('3', key="claim_name"))
|
||||
|
||||
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
|
||||
|
||||
|
||||
@@ -222,10 +222,6 @@ class ConfigurationTests(unittest.TestCase):
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
args = parser.parse_args(['--max-key-fee', 'null'])
|
||||
c = Config.create_from_arguments(args)
|
||||
self.assertEqual(c.max_key_fee, None)
|
||||
|
||||
# set
|
||||
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
|
||||
c = Config.create_from_arguments(args)
|
||||
|
||||
@@ -61,17 +61,13 @@ class TestAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address('bCqJrLHdoiRqEZ1whFZ3WHNb33bP34SuGx')
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9vwXVierUTT4hmoe3dtTeBfbNv1ph2mm8RWXARU6HsZjBaAoFaS2FRQu4fptR'
|
||||
'AyJWhJW42dmsEaC1nKnVKKTMhq3TVEHsNj1ca3ciZMKktT'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
self.assertIsNone(private_key)
|
||||
|
||||
def test_load_and_save_account(self):
|
||||
|
||||
@@ -11,7 +11,6 @@ ignore_missing_imports = True
|
||||
|
||||
[pylint]
|
||||
ignore=words,server,workbench,rpc
|
||||
max-parents=10
|
||||
max-args=10
|
||||
max-line-length=110
|
||||
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -17,38 +16,6 @@ class NetworkTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_header.first
|
||||
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
|
||||
|
||||
async def test_server_features(self):
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
'daily_fee': 0,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
address = (await self.account.get_addresses(limit=1))[0]
|
||||
os.environ.update({
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'DONATION_ADDRESS': address,
|
||||
'DAILY_FEE': '42'})
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
'hosts': {},
|
||||
'protocol_max': '1.4',
|
||||
'protocol_min': '1.1',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
'payment_address': address,
|
||||
'daily_fee': 42,
|
||||
'server_version': '0.5.7'}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
@@ -100,7 +67,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
# * goes to pick some water outside... * time passes by and another donation comes in
|
||||
sendtxid = await self.blockchain.send_to_address(address1, 42)
|
||||
await self.blockchain.generate(1)
|
||||
# (this is just so the test doesn't hang forever if it doesn't reconnect)
|
||||
# (this is just so the test doesnt hang forever if it doesnt reconnect)
|
||||
if not self.ledger.network.is_connected:
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
|
||||
# omg, the burned cable still works! torba is fire proof!
|
||||
@@ -115,13 +82,6 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_online_but_still_unavailable(self):
|
||||
# Edge case. See issue #2445 for context
|
||||
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.response_time = None
|
||||
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
@@ -162,7 +122,7 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
|
||||
# ensure we are connected to all of them after a while
|
||||
await asyncio.sleep(1)
|
||||
|
||||
@@ -88,7 +88,7 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
await self.assertBalance(account1, '0.0')
|
||||
await self.assertBalance(account2, '0.0')
|
||||
|
||||
addresses = await account1.receiving.get_addresses()
|
||||
addresses = await self.account.receiving.get_addresses()
|
||||
txids = await asyncio.gather(*(
|
||||
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
|
||||
))
|
||||
@@ -148,7 +148,7 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
return summary
|
||||
self.conductor.spv_node.server.mempool.transaction_summaries = random_summary
|
||||
# 10 unconfirmed txs, all from blockchain wallet
|
||||
sends = [self.blockchain.send_to_address(address, 10) for _ in range(10)]
|
||||
sends = list(self.blockchain.send_to_address(address, 10) for _ in range(10))
|
||||
# use batching to reduce issues with send_to_address on cli
|
||||
for batch in range(0, len(sends), 10):
|
||||
txids = await asyncio.gather(*sends[batch:batch + 10])
|
||||
|
||||
@@ -47,7 +47,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
async with self.account.receiving.address_generator_lock:
|
||||
await self.account.receiving._generate_keys(0, 200)
|
||||
records = await self.account.receiving.get_address_records()
|
||||
self.assertEqual(len(records), 201)
|
||||
self.assertEqual(201, len(records))
|
||||
|
||||
async def test_ensure_address_gap(self):
|
||||
account = self.account
|
||||
@@ -59,8 +59,8 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
await account.receiving._generate_keys(0, 3)
|
||||
await account.receiving._generate_keys(8, 11)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertListEqual(
|
||||
[r['pubkey'].n for r in records],
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
||||
)
|
||||
|
||||
@@ -68,8 +68,8 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
new_keys = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(len(new_keys), 8)
|
||||
records = await account.receiving.get_address_records()
|
||||
self.assertListEqual(
|
||||
[r['pubkey'].n for r in records],
|
||||
self.assertEqual(
|
||||
[r['position'] for r in records],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
|
||||
)
|
||||
|
||||
@@ -125,18 +125,14 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, '1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J'
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address('1CDLuMfwmPqJiNk5C2Bvew6tpgjAGgUk8J')
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9xV7rhbg6M4yWrdTeLorz3Q1GrQb4aQzzGWboP3du7W7UUztzNTUrEYTnDfz7o'
|
||||
'ptBygDxXYRppyiuenJpoBTgYP2C26E1Ah5FEALM24CsWi'
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
@@ -178,7 +174,7 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
account_data['ledger'] = 'btc_mainnet'
|
||||
self.assertDictEqual(account_data, account.to_dict())
|
||||
|
||||
def test_merge_diff(self):
|
||||
def test_apply_diff(self):
|
||||
account_data = {
|
||||
'name': 'My Account',
|
||||
'modified_on': 123.456,
|
||||
@@ -213,13 +209,13 @@ class TestHierarchicalDeterministicAccount(AsyncioTestCase):
|
||||
account_data['address_generator']['receiving']['gap'] = 8
|
||||
account_data['address_generator']['receiving']['maximum_uses_per_address'] = 9
|
||||
|
||||
account.merge(account_data)
|
||||
account.apply(account_data)
|
||||
# no change because modified_on is not newer
|
||||
self.assertEqual(account.name, 'My Account')
|
||||
|
||||
account_data['modified_on'] = 200.00
|
||||
|
||||
account.merge(account_data)
|
||||
account.apply(account_data)
|
||||
self.assertEqual(account.name, 'Changed Name')
|
||||
self.assertEqual(account.change.gap, 6)
|
||||
self.assertEqual(account.change.maximum_uses_per_address, 7)
|
||||
@@ -272,25 +268,19 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
|
||||
self.assertIsInstance(account.receiving, SingleKey)
|
||||
addresses = await account.receiving.get_addresses()
|
||||
self.assertListEqual(addresses, [])
|
||||
self.assertEqual(addresses, [])
|
||||
|
||||
# we have 12, but default gap is 20
|
||||
new_keys = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(len(new_keys), 1)
|
||||
self.assertEqual(new_keys[0], account.public_key.address)
|
||||
records = await account.receiving.get_address_records()
|
||||
pubkey = records[0].pop('pubkey')
|
||||
self.assertListEqual(records, [{
|
||||
'chain': 0,
|
||||
self.assertEqual(records, [{
|
||||
'position': 0, 'chain': 0,
|
||||
'account': account.public_key.address,
|
||||
'address': account.public_key.address,
|
||||
'history': None,
|
||||
'used_times': 0
|
||||
}])
|
||||
self.assertEqual(
|
||||
pubkey.extended_key_string(),
|
||||
account.public_key.extended_key_string()
|
||||
)
|
||||
|
||||
# case #1: no new addresses needed
|
||||
empty = await account.receiving.ensure_address_gap()
|
||||
@@ -343,18 +333,14 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
address = await account.receiving.ensure_address_gap()
|
||||
self.assertEqual(address[0], account.public_key.address)
|
||||
|
||||
private_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, address[0]
|
||||
)
|
||||
private_key = await self.ledger.get_private_key_for_address(address[0])
|
||||
self.assertEqual(
|
||||
private_key.extended_key_string(),
|
||||
'xprv9s21ZrQH143K3TsAz5efNV8K93g3Ms3FXcjaWB9fVUsMwAoE3ZT4vYymkp'
|
||||
'5BxKKfnpz8J6sHDFriX1SnpvjNkzcks8XBnxjGLS83BTyfpna',
|
||||
)
|
||||
|
||||
invalid_key = await self.ledger.get_private_key_for_address(
|
||||
account.wallet, 'BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX'
|
||||
)
|
||||
invalid_key = await self.ledger.get_private_key_for_address('BcQjRlhDOIrQez1WHfz3whnB33Bp34sUgX')
|
||||
self.assertIsNone(invalid_key)
|
||||
|
||||
self.assertEqual(
|
||||
|
||||
@@ -37,13 +37,13 @@ class BaseSelectionTestCase(AsyncioTestCase):
|
||||
class TestCoinSelectionTests(BaseSelectionTestCase):
|
||||
|
||||
def test_empty_coins(self):
|
||||
self.assertListEqual(CoinSelector(0, 0).select([]), [])
|
||||
self.assertEqual(CoinSelector(0, 0).select([]), [])
|
||||
|
||||
def test_skip_binary_search_if_total_not_enough(self):
|
||||
fee = utxo(CENT).get_estimator(self.ledger).fee
|
||||
big_pool = self.estimates(utxo(CENT+fee) for _ in range(100))
|
||||
selector = CoinSelector(101 * CENT, 0)
|
||||
self.assertListEqual(selector.select(big_pool), [])
|
||||
self.assertEqual(selector.select(big_pool), [])
|
||||
self.assertEqual(selector.tries, 0) # Never tried.
|
||||
# check happy path
|
||||
selector = CoinSelector(100 * CENT, 0)
|
||||
@@ -59,7 +59,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
|
||||
)
|
||||
selector = CoinSelector(CENT, 0)
|
||||
match = selector.select(utxo_pool)
|
||||
self.assertListEqual([CENT + fee], [c.txo.amount for c in match])
|
||||
self.assertEqual([CENT + fee], [c.txo.amount for c in match])
|
||||
self.assertTrue(selector.exact_match)
|
||||
|
||||
def test_random_draw(self):
|
||||
@@ -70,7 +70,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
|
||||
)
|
||||
selector = CoinSelector(CENT, 0, '\x00')
|
||||
match = selector.select(utxo_pool)
|
||||
self.assertListEqual([2 * CENT], [c.txo.amount for c in match])
|
||||
self.assertEqual([2 * CENT], [c.txo.amount for c in match])
|
||||
self.assertFalse(selector.exact_match)
|
||||
|
||||
def test_pick(self):
|
||||
@@ -83,7 +83,7 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
|
||||
)
|
||||
selector = CoinSelector(3*CENT, 0)
|
||||
match = selector.select(utxo_pool)
|
||||
self.assertListEqual([5*CENT], [c.txo.amount for c in match])
|
||||
self.assertEqual([5*CENT], [c.txo.amount for c in match])
|
||||
|
||||
def test_confirmed_strategies(self):
|
||||
utxo_pool = self.estimates(
|
||||
@@ -94,14 +94,14 @@ class TestCoinSelectionTests(BaseSelectionTestCase):
|
||||
)
|
||||
|
||||
match = CoinSelector(20*CENT, 0).select(utxo_pool, "only_confirmed")
|
||||
self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match])
|
||||
self.assertEqual([5, 5], [c.txo.tx_ref.height for c in match])
|
||||
match = CoinSelector(25*CENT, 0).select(utxo_pool, "only_confirmed")
|
||||
self.assertListEqual([], [c.txo.tx_ref.height for c in match])
|
||||
self.assertEqual([], [c.txo.tx_ref.height for c in match])
|
||||
|
||||
match = CoinSelector(20*CENT, 0).select(utxo_pool, "prefer_confirmed")
|
||||
self.assertListEqual([5, 5], [c.txo.tx_ref.height for c in match])
|
||||
self.assertEqual([5, 5], [c.txo.tx_ref.height for c in match])
|
||||
match = CoinSelector(25*CENT, 0, '\x00').select(utxo_pool, "prefer_confirmed")
|
||||
self.assertListEqual([5, 0, -2], [c.txo.tx_ref.height for c in match])
|
||||
self.assertEqual([5, 0, -2], [c.txo.tx_ref.height for c in match])
|
||||
|
||||
|
||||
class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
|
||||
@@ -136,20 +136,20 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
|
||||
)
|
||||
|
||||
# Select 1 Cent
|
||||
self.assertListEqual([1 * CENT], search(utxo_pool, 1 * CENT, 0.5 * CENT))
|
||||
self.assertEqual([1 * CENT], search(utxo_pool, 1 * CENT, 0.5 * CENT))
|
||||
|
||||
# Select 2 Cent
|
||||
self.assertListEqual([2 * CENT], search(utxo_pool, 2 * CENT, 0.5 * CENT))
|
||||
self.assertEqual([2 * CENT], search(utxo_pool, 2 * CENT, 0.5 * CENT))
|
||||
|
||||
# Select 5 Cent
|
||||
self.assertListEqual([3 * CENT, 2 * CENT], search(utxo_pool, 5 * CENT, 0.5 * CENT))
|
||||
self.assertEqual([3 * CENT, 2 * CENT], search(utxo_pool, 5 * CENT, 0.5 * CENT))
|
||||
|
||||
# Select 11 Cent, not possible
|
||||
self.assertListEqual([], search(utxo_pool, 11 * CENT, 0.5 * CENT))
|
||||
self.assertEqual([], search(utxo_pool, 11 * CENT, 0.5 * CENT))
|
||||
|
||||
# Select 10 Cent
|
||||
utxo_pool += self.estimates(utxo(5 * CENT))
|
||||
self.assertListEqual(
|
||||
self.assertEqual(
|
||||
[4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT],
|
||||
search(utxo_pool, 10 * CENT, 0.5 * CENT)
|
||||
)
|
||||
@@ -157,18 +157,18 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
|
||||
# Negative effective value
|
||||
# Select 10 Cent but have 1 Cent not be possible because too small
|
||||
# TODO: bitcoin has [5, 3, 2]
|
||||
self.assertListEqual(
|
||||
self.assertEqual(
|
||||
[4 * CENT, 3 * CENT, 2 * CENT, 1 * CENT],
|
||||
search(utxo_pool, 10 * CENT, 5000)
|
||||
)
|
||||
|
||||
# Select 0.25 Cent, not possible
|
||||
self.assertListEqual(search(utxo_pool, 0.25 * CENT, 0.5 * CENT), [])
|
||||
self.assertEqual(search(utxo_pool, 0.25 * CENT, 0.5 * CENT), [])
|
||||
|
||||
# Iteration exhaustion test
|
||||
utxo_pool, target = self.make_hard_case(17)
|
||||
selector = CoinSelector(target, 0)
|
||||
self.assertListEqual(selector.select(utxo_pool, 'branch_and_bound'), [])
|
||||
self.assertEqual(selector.select(utxo_pool, 'branch_and_bound'), [])
|
||||
self.assertEqual(selector.tries, MAXIMUM_TRIES) # Should exhaust
|
||||
utxo_pool, target = self.make_hard_case(14)
|
||||
self.assertIsNotNone(search(utxo_pool, target, 0)) # Should not exhaust
|
||||
@@ -181,7 +181,7 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
|
||||
utxo(7 * CENT),
|
||||
utxo(2 * CENT)
|
||||
] + [utxo(5 * CENT)]*50000)
|
||||
self.assertListEqual(
|
||||
self.assertEqual(
|
||||
[7 * CENT, 7 * CENT, 7 * CENT, 7 * CENT, 2 * CENT],
|
||||
search(utxo_pool, 30 * CENT, 5000)
|
||||
)
|
||||
@@ -189,4 +189,4 @@ class TestOfficialBitcoinCoinSelectionTests(BaseSelectionTestCase):
|
||||
# Select 1 Cent with pool of only greater than 5 Cent
|
||||
utxo_pool = self.estimates(utxo(i * CENT) for i in range(5, 21))
|
||||
for _ in range(100):
|
||||
self.assertListEqual(search(utxo_pool, 1 * CENT, 2 * CENT), [])
|
||||
self.assertEqual(search(utxo_pool, 1 * CENT, 2 * CENT), [])
|
||||
@@ -1,10 +1,7 @@
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import asyncio
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import os
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
@@ -60,17 +57,17 @@ class TestAIOSQLite(AsyncioTestCase):
|
||||
class TestQueryBuilder(unittest.TestCase):
|
||||
|
||||
def test_dot(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.position': 18}),
|
||||
('txo.position = :txo_position0', {'txo_position0': 18})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.position#6': 18}),
|
||||
('txo.position = :txo_position6', {'txo_position6': 18})
|
||||
)
|
||||
|
||||
def test_any(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({
|
||||
'ages__any': {
|
||||
'txo.age__gt': 18,
|
||||
@@ -84,41 +81,41 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_in(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.age__in#2': [18, 38]}),
|
||||
('txo.age IN (:txo_age__in2_0, :txo_age__in2_1)', {
|
||||
'txo_age__in2_0': 18,
|
||||
'txo_age__in2_1': 38
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.name__in': ('abc123', 'def456')}),
|
||||
('txo.name IN (:txo_name__in0_0, :txo_name__in0_1)', {
|
||||
'txo_name__in0_0': 'abc123',
|
||||
'txo_name__in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
|
||||
('txo.age IN (SELECT age from ages_table)', {})
|
||||
)
|
||||
|
||||
def test_not_in(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.age__not_in': [18, 38]}),
|
||||
('txo.age NOT IN (:txo_age__not_in0_0, :txo_age__not_in0_1)', {
|
||||
'txo_age__not_in0_0': 18,
|
||||
'txo_age__not_in0_1': 38
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.name__not_in': ('abc123', 'def456')}),
|
||||
('txo.name NOT IN (:txo_name__not_in0_0, :txo_name__not_in0_1)', {
|
||||
'txo_name__not_in0_0': 'abc123',
|
||||
'txo_name__not_in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
|
||||
('txo.age NOT IN (SELECT age from ages_table)', {})
|
||||
)
|
||||
@@ -128,11 +125,11 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
constraints_to_sql({'ages__in': 9})
|
||||
|
||||
def test_query(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo"),
|
||||
("select * from foo", {})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query(
|
||||
"select * from foo",
|
||||
a__not='b', b__in='select * from blah where c=:$c',
|
||||
@@ -146,11 +143,11 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_query_order_by(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo", order_by='foo'),
|
||||
("select * from foo ORDER BY foo", {})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo", order_by=['foo', 'bar']),
|
||||
("select * from foo ORDER BY foo, bar", {})
|
||||
)
|
||||
@@ -158,15 +155,15 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
query("select * from foo", order_by={'foo': 'bar'})
|
||||
|
||||
def test_query_limit_offset(self):
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo", limit=10),
|
||||
("select * from foo LIMIT 10", {})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo", offset=10),
|
||||
("select * from foo OFFSET 10", {})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
query("select * from foo", limit=20, offset=10),
|
||||
("select * from foo LIMIT 20 OFFSET 10", {})
|
||||
)
|
||||
@@ -188,7 +185,7 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
"(one LIKE 'o' OR two = 2) AND "
|
||||
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
|
||||
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
|
||||
"ORDER BY b LIMIT 10"
|
||||
"ORDER BY b LIMIT 10",
|
||||
)
|
||||
|
||||
|
||||
@@ -199,14 +196,13 @@ class TestQueries(AsyncioTestCase):
|
||||
'db': ledger_class.database_class(':memory:'),
|
||||
'headers': ledger_class.headers_class(':memory:'),
|
||||
})
|
||||
self.wallet = Wallet()
|
||||
await self.ledger.db.open()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.ledger.db.close()
|
||||
|
||||
async def create_account(self, wallet=None):
|
||||
account = self.ledger.account_class.generate(self.ledger, wallet or self.wallet)
|
||||
async def create_account(self):
|
||||
account = self.ledger.account_class.generate(self.ledger, Wallet())
|
||||
await account.ensure_address_gap()
|
||||
return account
|
||||
|
||||
@@ -264,9 +260,8 @@ class TestQueries(AsyncioTestCase):
|
||||
for height in range(1, 1200):
|
||||
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
|
||||
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
|
||||
for limit in range(variable_limit - 2, variable_limit + 2):
|
||||
txs = await self.ledger.get_transactions(
|
||||
accounts=self.wallet.accounts, limit=limit, order_by='height asc')
|
||||
for limit in range(variable_limit-2, variable_limit+2):
|
||||
txs = await self.ledger.get_transactions(limit=limit, order_by='height asc')
|
||||
self.assertEqual(len(txs), limit)
|
||||
inputs, outputs, last_tx = set(), set(), txs[0]
|
||||
for tx in txs[1:]:
|
||||
@@ -276,23 +271,19 @@ class TestQueries(AsyncioTestCase):
|
||||
last_tx = tx
|
||||
|
||||
async def test_queries(self):
|
||||
wallet1 = Wallet()
|
||||
account1 = await self.create_account(wallet1)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account1]))
|
||||
wallet2 = Wallet()
|
||||
account2 = await self.create_account(wallet2)
|
||||
account3 = await self.create_account(wallet2)
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_address_count())
|
||||
account1 = await self.create_account()
|
||||
self.assertEqual(26, await self.ledger.db.get_address_count())
|
||||
account2 = await self.create_account()
|
||||
self.assertEqual(52, await self.ledger.db.get_address_count())
|
||||
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2, account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_transaction_count(accounts=[account1, account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count())
|
||||
self.assertListEqual([], await self.ledger.db.get_utxos())
|
||||
self.assertEqual([], await self.ledger.db.get_utxos())
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx1 = await self.create_tx_from_nothing(account1, 1)
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -300,28 +291,20 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx2 = await self.create_tx_from_txo(tx1.outputs[0], account2, 2)
|
||||
tx2b = await self.create_tx_from_nothing(account3, 2)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_transaction_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_utxo_count(accounts=[account3]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account3]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8+10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
tx3 = await self.create_tx_to_nowhere(tx2.outputs[0], 3)
|
||||
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account1]))
|
||||
@@ -330,25 +313,23 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_utxo_count(accounts=[account2]))
|
||||
self.assertEqual(1, await self.ledger.db.get_txo_count(accounts=[account2]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(wallet=wallet1))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(wallet=wallet2))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance())
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account1]))
|
||||
self.assertEqual(0, await self.ledger.db.get_balance(accounts=[account2]))
|
||||
self.assertEqual(10**8, await self.ledger.db.get_balance(accounts=[account3]))
|
||||
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertListEqual([3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual([3, 2, 1], [tx.height for tx in txs])
|
||||
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
|
||||
self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1])
|
||||
self.assertEqual([tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].inputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].outputs[0].is_my_account, True)
|
||||
|
||||
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
|
||||
self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs])
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account2])
|
||||
self.assertEqual([tx3.id, tx2.id], [tx.id for tx in txs])
|
||||
self.assertEqual(txs[0].inputs[0].is_my_account, True)
|
||||
self.assertEqual(txs[0].outputs[0].is_my_account, False)
|
||||
self.assertEqual(txs[1].inputs[0].is_my_account, False)
|
||||
@@ -359,21 +340,21 @@ class TestQueries(AsyncioTestCase):
|
||||
self.assertEqual(tx.id, tx2.id)
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account1])
|
||||
self.assertEqual(tx.inputs[0].is_my_account, True)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, False)
|
||||
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
|
||||
tx = await self.ledger.db.get_transaction(txid=tx2.id, accounts=[account2])
|
||||
self.assertEqual(tx.inputs[0].is_my_account, False)
|
||||
self.assertEqual(tx.outputs[0].is_my_account, True)
|
||||
|
||||
# height 0 sorted to the top with the rest in descending order
|
||||
tx4 = await self.create_tx_from_nothing(account1, 0)
|
||||
txos = await self.ledger.db.get_txos()
|
||||
self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
self.assertEqual([0, 2, 1], [txo.tx_ref.height for txo in txos])
|
||||
self.assertEqual([tx4.id, tx2.id, tx1.id], [txo.tx_ref.id for txo in txos])
|
||||
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
|
||||
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
self.assertEqual([0, 3, 2, 1], [tx.height for tx in txs])
|
||||
self.assertEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
|
||||
|
||||
|
||||
class TestUpgrade(AsyncioTestCase):
|
||||
@@ -398,13 +379,13 @@ class TestUpgrade(AsyncioTestCase):
|
||||
def add_address(self, address):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
conn.execute("""
|
||||
INSERT INTO account_address (address, account, chain, n, pubkey, chain_code, depth)
|
||||
VALUES (?, 'account1', 0, 0, 'pubkey', 'chain_code', 0)
|
||||
INSERT INTO pubkey_address (address, account, chain, position, pubkey)
|
||||
VALUES (?, 'account1', 0, 0, 'pubkey blob')
|
||||
""", (address,))
|
||||
|
||||
def get_addresses(self):
|
||||
with sqlite3.connect(self.path) as conn:
|
||||
sql = "SELECT address FROM account_address ORDER BY address;"
|
||||
sql = "SELECT address FROM pubkey_address ORDER BY address;"
|
||||
return [col[0] for col in conn.execute(sql).fetchall()]
|
||||
|
||||
async def test_reset_on_version_change(self):
|
||||
@@ -415,10 +396,10 @@ class TestUpgrade(AsyncioTestCase):
|
||||
|
||||
# initial open, pre-version enabled db
|
||||
self.ledger.db.SCHEMA_VERSION = None
|
||||
self.assertListEqual(self.get_tables(), [])
|
||||
self.assertEqual(self.get_tables(), [])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo'])
|
||||
self.assertListEqual(self.get_addresses(), [])
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo'])
|
||||
self.assertEqual(self.get_addresses(), [])
|
||||
self.add_address('address1')
|
||||
await self.ledger.db.close()
|
||||
|
||||
@@ -426,18 +407,18 @@ class TestUpgrade(AsyncioTestCase):
|
||||
self.ledger.db.SCHEMA_VERSION = '1.0'
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertListEqual(self.get_addresses(), []) # address1 deleted during version upgrade
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # address1 deleted during version upgrade
|
||||
self.add_address('address2')
|
||||
await self.ledger.db.close()
|
||||
|
||||
# nothing changes
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.0')
|
||||
self.assertListEqual(self.get_tables(), ['account_address', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertListEqual(self.get_addresses(), ['address2'])
|
||||
self.assertEqual(self.get_tables(), ['pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), ['address2'])
|
||||
await self.ledger.db.close()
|
||||
|
||||
# upgrade version, database reset
|
||||
@@ -447,101 +428,6 @@ class TestUpgrade(AsyncioTestCase):
|
||||
"""
|
||||
await self.ledger.db.open()
|
||||
self.assertEqual(self.get_version(), '1.1')
|
||||
self.assertListEqual(self.get_tables(), ['account_address', 'foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertListEqual(self.get_addresses(), []) # all tables got reset
|
||||
self.assertEqual(self.get_tables(), ['foo', 'pubkey_address', 'tx', 'txi', 'txo', 'version'])
|
||||
self.assertEqual(self.get_addresses(), []) # all tables got reset
|
||||
await self.ledger.db.close()
|
||||
|
||||
|
||||
class TestSQLiteRace(AsyncioTestCase):
|
||||
max_misuse_attempts = 40000
|
||||
|
||||
def setup_db(self):
|
||||
self.db = sqlite3.connect(":memory:", isolation_level=None)
|
||||
self.db.executescript(
|
||||
"create table test1 (id text primary key not null, val text);\n" +
|
||||
"create table test2 (id text primary key not null, val text);\n" +
|
||||
"\n".join(f"insert into test1 values ({v}, NULL);" for v in range(1000))
|
||||
)
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
await self.loop.run_in_executor(self.executor, self.setup_db)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.loop.run_in_executor(self.executor, self.db.close)
|
||||
self.executor.shutdown()
|
||||
|
||||
async def test_binding_param_0_error(self):
|
||||
# test real param 0 binding errors
|
||||
|
||||
for supported_type in [str, int, bytes]:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 values (?, NULL)",
|
||||
[(supported_type(1), ), (supported_type(2), )]
|
||||
)
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.execute, "delete from test2 where id in (1, 2)"
|
||||
)
|
||||
for unsupported_type in [lambda x: (x, ), lambda x: [x], lambda x: {x}]:
|
||||
try:
|
||||
await self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "insert into test2 (id, val) values (?, NULL)",
|
||||
[(unsupported_type(1), ), (unsupported_type(2), )]
|
||||
)
|
||||
self.assertTrue(False)
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
|
||||
async def test_unhandled_sqlite_misuse(self):
|
||||
# test SQLITE_MISUSE being incorrectly raised as a param 0 binding error
|
||||
attempts = 0
|
||||
python_version = sys.version.split('\n')[0].rstrip(' ')
|
||||
|
||||
try:
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, self.db.executemany, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} "
|
||||
f"did not raise SQLITE_MISUSE within {attempts} attempts of the race condition")
|
||||
self.assertTrue(False, 'this test failing means either the sqlite race conditions '
|
||||
'have been fixed in cpython or the test max_attempts needs to be increased')
|
||||
except sqlite3.InterfaceError as err:
|
||||
self.assertEqual(str(err), "Error binding parameter 0 - probably unsupported type.")
|
||||
print(f"\nsqlite3 {sqlite3.version}/python {python_version} raised SQLITE_MISUSE "
|
||||
f"after {attempts} attempts of the race condition")
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_fetchall_prevents_sqlite_misuse(self):
|
||||
# test that calling fetchall sufficiently avoids the race
|
||||
attempts = 0
|
||||
|
||||
def executemany_fetchall(query, params):
|
||||
self.db.executemany(query, params).fetchall()
|
||||
|
||||
while attempts < self.max_misuse_attempts:
|
||||
f1 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test1 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
f2 = asyncio.wrap_future(
|
||||
self.loop.run_in_executor(
|
||||
self.executor, executemany_fetchall, "update test2 set val='derp' where id=?",
|
||||
((str(i),) for i in range(2))
|
||||
)
|
||||
)
|
||||
attempts += 1
|
||||
await asyncio.gather(f1, f2)
|
||||
@@ -23,7 +23,7 @@ class TestAESEncryptDecrypt(TestCase):
|
||||
'ZmZmZmZmZmZmZmZmZmZmZjlrKptoKD+MFwDxcg3XtCD9qz8UWhEhq/TVJT5+Mtp2a8sE'
|
||||
'CaO6WQj7fYsWGu2Hvbc0qYqxdN0HeTsiO+cZRo3eJISgr3F+rXFYi5oSBlD2'
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
self.assertEqual(
|
||||
aes_decrypt(self.password, self.expected),
|
||||
(self.message, b'f' * 16)
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
|
||||
async def test_serialization(self):
|
||||
h = await self.get_headers()
|
||||
self.assertDictEqual(h[0], {
|
||||
self.assertEqual(h[0], {
|
||||
'bits': 486604799,
|
||||
'block_height': 0,
|
||||
'merkle_root': b'4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
|
||||
@@ -44,7 +44,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
'timestamp': 1231006505,
|
||||
'version': 1
|
||||
})
|
||||
self.assertDictEqual(h[self.RETARGET_BLOCK-1], {
|
||||
self.assertEqual(h[self.RETARGET_BLOCK-1], {
|
||||
'bits': 486604799,
|
||||
'block_height': 32255,
|
||||
'merkle_root': b'89b4f223789e40b5b475af6483bb05bceda54059e17d2053334b358f6bb310ac',
|
||||
@@ -53,7 +53,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
'timestamp': 1262152739,
|
||||
'version': 1
|
||||
})
|
||||
self.assertDictEqual(h[self.RETARGET_BLOCK], {
|
||||
self.assertEqual(h[self.RETARGET_BLOCK], {
|
||||
'bits': 486594666,
|
||||
'block_height': 32256,
|
||||
'merkle_root': b'64b5e5f5a262f47af443a0120609206a3305877693edfe03e994f20a024ab627',
|
||||
@@ -62,7 +62,7 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
'timestamp': 1262153464,
|
||||
'version': 1
|
||||
})
|
||||
self.assertDictEqual(h[self.RETARGET_BLOCK+1], {
|
||||
self.assertEqual(h[self.RETARGET_BLOCK+1], {
|
||||
'bits': 486594666,
|
||||
'block_height': 32257,
|
||||
'merkle_root': b'4d1488981f08b3037878193297dbac701a2054e0f803d4424fe6a4d763d62334',
|
||||
|
||||
@@ -90,8 +90,8 @@ class TestSynchronization(LedgerTestCase):
|
||||
'abcd03': hexlify(get_transaction(get_output(3)).raw),
|
||||
})
|
||||
await self.ledger.update_history(address, '')
|
||||
self.assertListEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
|
||||
self.assertEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
|
||||
|
||||
address_details = await self.ledger.db.get_address(address=address)
|
||||
self.assertEqual(
|
||||
@@ -104,16 +104,16 @@ class TestSynchronization(LedgerTestCase):
|
||||
self.ledger.network.get_history_called = []
|
||||
self.ledger.network.get_transaction_called = []
|
||||
await self.ledger.update_history(address, '')
|
||||
self.assertListEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertListEqual(self.ledger.network.get_transaction_called, [])
|
||||
self.assertEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertEqual(self.ledger.network.get_transaction_called, [])
|
||||
|
||||
self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 3})
|
||||
self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw)
|
||||
self.ledger.network.get_history_called = []
|
||||
self.ledger.network.get_transaction_called = []
|
||||
await self.ledger.update_history(address, '')
|
||||
self.assertListEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd04'])
|
||||
self.assertEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertEqual(self.ledger.network.get_transaction_called, ['abcd04'])
|
||||
address_details = await self.ledger.db.get_address(address=address)
|
||||
self.assertEqual(
|
||||
address_details['history'],
|
||||
|
||||
@@ -24,14 +24,14 @@ def parse(opcodes, source):
|
||||
class TestScriptTemplates(unittest.TestCase):
|
||||
|
||||
def test_push_data(self):
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_SINGLE('script_hash'),),
|
||||
(b'abcdef',)
|
||||
), {
|
||||
'script_hash': b'abcdef'
|
||||
}
|
||||
)
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_SINGLE('first'), PUSH_INTEGER('rating')),
|
||||
(b'Satoshi', (1000).to_bytes(2, 'little'))
|
||||
), {
|
||||
@@ -39,7 +39,7 @@ class TestScriptTemplates(unittest.TestCase):
|
||||
'rating': 1000,
|
||||
}
|
||||
)
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL),
|
||||
(OP_HASH160, b'abcdef', OP_EQUAL)
|
||||
), {
|
||||
@@ -48,21 +48,21 @@ class TestScriptTemplates(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_push_data_many(self):
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_MANY('names'),),
|
||||
(b'amit',)
|
||||
), {
|
||||
'names': [b'amit']
|
||||
}
|
||||
)
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_MANY('names'),),
|
||||
(b'jeremy', b'amit', b'victor')
|
||||
), {
|
||||
'names': [b'jeremy', b'amit', b'victor']
|
||||
}
|
||||
)
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(OP_HASH160, PUSH_MANY('names'), OP_EQUAL),
|
||||
(OP_HASH160, b'grin', b'jack', OP_EQUAL)
|
||||
), {
|
||||
@@ -71,7 +71,7 @@ class TestScriptTemplates(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_push_data_mixed(self):
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_SINGLE('CEO'), PUSH_MANY('Devs'), PUSH_SINGLE('CTO'), PUSH_SINGLE('State')),
|
||||
(b'jeremy', b'lex', b'amit', b'victor', b'jack', b'grin', b'NH')
|
||||
), {
|
||||
@@ -83,7 +83,7 @@ class TestScriptTemplates(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_push_data_many_separated(self):
|
||||
self.assertDictEqual(parse(
|
||||
self.assertEqual(parse(
|
||||
(PUSH_MANY('Chiefs'), OP_HASH160, PUSH_MANY('Devs')),
|
||||
(b'jeremy', b'grin', OP_HASH160, b'lex', b'jack')
|
||||
), {
|
||||
@@ -135,16 +135,16 @@ class TestRedeemScriptHash(unittest.TestCase):
|
||||
)
|
||||
subscript1 = src1.values['script']
|
||||
self.assertEqual(src1.template.name, 'script_hash')
|
||||
self.assertListEqual([hexlify(v) for v in src1.values['signatures']], sigs)
|
||||
self.assertListEqual([hexlify(p) for p in subscript1.values['pubkeys']], pubkeys)
|
||||
self.assertEqual([hexlify(v) for v in src1.values['signatures']], sigs)
|
||||
self.assertEqual([hexlify(p) for p in subscript1.values['pubkeys']], pubkeys)
|
||||
self.assertEqual(subscript1.values['signatures_count'], len(sigs))
|
||||
self.assertEqual(subscript1.values['pubkeys_count'], len(pubkeys))
|
||||
# now we test that it will round trip
|
||||
src2 = BaseInputScript(src1.source)
|
||||
subscript2 = src2.values['script']
|
||||
self.assertEqual(src2.template.name, 'script_hash')
|
||||
self.assertListEqual([hexlify(v) for v in src2.values['signatures']], sigs)
|
||||
self.assertListEqual([hexlify(p) for p in subscript2.values['pubkeys']], pubkeys)
|
||||
self.assertEqual([hexlify(v) for v in src2.values['signatures']], sigs)
|
||||
self.assertEqual([hexlify(p) for p in subscript2.values['pubkeys']], pubkeys)
|
||||
self.assertEqual(subscript2.values['signatures_count'], len(sigs))
|
||||
self.assertEqual(subscript2.values['pubkeys_count'], len(pubkeys))
|
||||
return hexlify(src1.source)
|
||||
|
||||
@@ -9,7 +9,7 @@ class StreamControllerTestCase(AsyncioTestCase):
|
||||
controller.stream.listen(on_data=events.append)
|
||||
controller.add("yo")
|
||||
controller.add("yo")
|
||||
self.assertListEqual(events, ["yo", "yo"])
|
||||
self.assertEqual(events, ["yo", "yo"])
|
||||
|
||||
def test_unique_events(self):
|
||||
events = []
|
||||
@@ -17,4 +17,4 @@ class StreamControllerTestCase(AsyncioTestCase):
|
||||
controller.stream.listen(on_data=events.append)
|
||||
controller.add("yo")
|
||||
controller.add("yo")
|
||||
self.assertListEqual(events, ["yo"])
|
||||
self.assertEqual(events, ["yo"])
|
||||
@@ -294,9 +294,9 @@ class TransactionIOBalancing(AsyncioTestCase):
|
||||
[self.txo(3)] # outputs
|
||||
)
|
||||
# best UTXO match is 5 (as UTXO 3 will be short 0.02 to cover fees)
|
||||
self.assertListEqual(self.inputs(tx), [5])
|
||||
self.assertEqual(self.inputs(tx), [5])
|
||||
# a change of 1.98 is added to reach balance
|
||||
self.assertListEqual(self.outputs(tx), [3, 1.98])
|
||||
self.assertEqual(self.outputs(tx), [3, 1.98])
|
||||
|
||||
await self.ledger.release_outputs(utxos)
|
||||
|
||||
@@ -306,8 +306,8 @@ class TransactionIOBalancing(AsyncioTestCase):
|
||||
[self.txo(2.98)] # outputs
|
||||
)
|
||||
# best UTXO match is 3 and no change is needed
|
||||
self.assertListEqual(self.inputs(tx), [3])
|
||||
self.assertListEqual(self.outputs(tx), [2.98])
|
||||
self.assertEqual(self.inputs(tx), [3])
|
||||
self.assertEqual(self.outputs(tx), [2.98])
|
||||
|
||||
await self.ledger.release_outputs(utxos)
|
||||
|
||||
@@ -317,9 +317,9 @@ class TransactionIOBalancing(AsyncioTestCase):
|
||||
[self.txo(11)] # outputs
|
||||
)
|
||||
# additional input is chosen (UTXO 3)
|
||||
self.assertListEqual([10, 3], self.inputs(tx))
|
||||
self.assertEqual([10, 3], self.inputs(tx))
|
||||
# change is now needed to consume extra input
|
||||
self.assertListEqual([11, 1.96], self.outputs(tx))
|
||||
self.assertEqual([11, 1.96], self.outputs(tx))
|
||||
|
||||
await self.ledger.release_outputs(utxos)
|
||||
|
||||
@@ -328,9 +328,9 @@ class TransactionIOBalancing(AsyncioTestCase):
|
||||
[self.txi(self.txo(10))], # inputs
|
||||
[] # outputs
|
||||
)
|
||||
self.assertListEqual([10], self.inputs(tx))
|
||||
self.assertEqual([10], self.inputs(tx))
|
||||
# missing change added to consume the amount
|
||||
self.assertListEqual([9.98], self.outputs(tx))
|
||||
self.assertEqual([9.98], self.outputs(tx))
|
||||
|
||||
await self.ledger.release_outputs(utxos)
|
||||
|
||||
@@ -340,6 +340,6 @@ class TransactionIOBalancing(AsyncioTestCase):
|
||||
[] # outputs
|
||||
)
|
||||
# UTXO 1 is added to cover some of the fee
|
||||
self.assertListEqual([0.01, 1], self.inputs(tx))
|
||||
self.assertEqual([0.01, 1], self.inputs(tx))
|
||||
# change is now needed to consume extra input
|
||||
self.assertListEqual([0.97], self.outputs(tx))
|
||||
self.assertEqual([0.97], self.outputs(tx))
|
||||
@@ -1,13 +1,12 @@
|
||||
import tempfile
|
||||
from binascii import hexlify
|
||||
|
||||
from unittest import TestCase, mock
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
from torba.coin.bitcoinsegwit import MainNetLedger as BTCLedger
|
||||
from torba.coin.bitcoincash import MainNetLedger as BCHLedger
|
||||
from torba.client.basemanager import BaseWalletManager
|
||||
from torba.client.wallet import Wallet, WalletStorage, TimestampedPreferences
|
||||
from torba.client.wallet import Wallet, WalletStorage
|
||||
|
||||
|
||||
class TestWalletCreation(AsyncioTestCase):
|
||||
@@ -21,7 +20,7 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
def test_create_wallet_and_accounts(self):
|
||||
wallet = Wallet()
|
||||
self.assertEqual(wallet.name, 'Wallet')
|
||||
self.assertListEqual(wallet.accounts, [])
|
||||
self.assertEqual(wallet.accounts, [])
|
||||
|
||||
account1 = wallet.generate_account(self.btc_ledger)
|
||||
wallet.generate_account(self.btc_ledger)
|
||||
@@ -33,7 +32,6 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
wallet_dict = {
|
||||
'version': 1,
|
||||
'name': 'Main Wallet',
|
||||
'preferences': {},
|
||||
'accounts': [
|
||||
{
|
||||
'name': 'An Account',
|
||||
@@ -62,7 +60,7 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
wallet = Wallet.from_storage(storage, self.manager)
|
||||
self.assertEqual(wallet.name, 'Main Wallet')
|
||||
self.assertEqual(
|
||||
hexlify(wallet.hash), b'1bd61fbe18875cb7828c466022af576104ed861c8a1fdb1dadf5e39417a68483'
|
||||
hexlify(wallet.hash), b'9f462b8dd802eb8c913e54f09a09827ebc14abbc13f33baa90d8aec5ae920fc7'
|
||||
)
|
||||
self.assertEqual(len(wallet.accounts), 1)
|
||||
account = wallet.default_account
|
||||
@@ -93,56 +91,3 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
wallet = Wallet.from_storage(wallet_storage, manager)
|
||||
|
||||
self.assertEqual(account.public_key.address, wallet.default_account.public_key.address)
|
||||
|
||||
def test_merge(self):
|
||||
wallet1 = Wallet()
|
||||
wallet1.preferences['one'] = 1
|
||||
wallet1.preferences['conflict'] = 1
|
||||
wallet1.generate_account(self.btc_ledger)
|
||||
wallet2 = Wallet()
|
||||
wallet2.preferences['two'] = 2
|
||||
wallet2.preferences['conflict'] = 2 # will be more recent
|
||||
wallet2.generate_account(self.btc_ledger)
|
||||
|
||||
self.assertEqual(len(wallet1.accounts), 1)
|
||||
self.assertEqual(wallet1.preferences, {'one': 1, 'conflict': 1})
|
||||
|
||||
added = wallet1.merge(self.manager, 'password', wallet2.pack('password'))
|
||||
self.assertEqual(added[0].id, wallet2.default_account.id)
|
||||
self.assertEqual(len(wallet1.accounts), 2)
|
||||
self.assertEqual(wallet1.accounts[1].id, wallet2.default_account.id)
|
||||
self.assertEqual(wallet1.preferences, {'one': 1, 'two': 2, 'conflict': 2})
|
||||
|
||||
|
||||
class TestTimestampedPreferences(TestCase):
|
||||
|
||||
def test_hash(self):
|
||||
p = TimestampedPreferences()
|
||||
self.assertEqual(
|
||||
hexlify(p.hash), b'44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a'
|
||||
)
|
||||
with mock.patch('time.time', mock.Mock(return_value=12345)):
|
||||
p['one'] = 1
|
||||
self.assertEqual(
|
||||
hexlify(p.hash), b'c9e82bf4cb099dd0125f78fa381b21a8131af601917eb531e1f5f980f8f3da66'
|
||||
)
|
||||
|
||||
def test_merge(self):
|
||||
p1 = TimestampedPreferences()
|
||||
p2 = TimestampedPreferences()
|
||||
with mock.patch('time.time', mock.Mock(return_value=10)):
|
||||
p1['one'] = 1
|
||||
p1['conflict'] = 1
|
||||
with mock.patch('time.time', mock.Mock(return_value=20)):
|
||||
p2['two'] = 2
|
||||
p2['conflict'] = 2
|
||||
|
||||
# conflict in p2 overrides conflict in p1
|
||||
p1.merge(p2.data)
|
||||
self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 2})
|
||||
|
||||
# have a newer conflict in p1 so it is not overridden this time
|
||||
with mock.patch('time.time', mock.Mock(return_value=21)):
|
||||
p1['conflict'] = 1
|
||||
p1.merge(p2.data)
|
||||
self.assertEqual(p1, {'one': 1, 'two': 2, 'conflict': 1})
|
||||
@@ -42,7 +42,7 @@ class AddressManager:
|
||||
d['change'] = change_dict
|
||||
return d
|
||||
|
||||
def merge(self, d: dict):
|
||||
def apply(self, d: dict):
|
||||
pass
|
||||
|
||||
def to_dict_instance(self) -> Optional[dict]:
|
||||
@@ -85,7 +85,7 @@ class AddressManager:
|
||||
class HierarchicalDeterministic(AddressManager):
|
||||
""" Implements simple version of Bitcoin Hierarchical Deterministic key management. """
|
||||
|
||||
name: str = "deterministic-chain"
|
||||
name = "deterministic-chain"
|
||||
|
||||
__slots__ = 'gap', 'maximum_uses_per_address'
|
||||
|
||||
@@ -101,7 +101,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
|
||||
)
|
||||
|
||||
def merge(self, d: dict):
|
||||
def apply(self, d: dict):
|
||||
self.gap = d.get('gap', self.gap)
|
||||
self.maximum_uses_per_address = d.get('maximum_uses_per_address', self.maximum_uses_per_address)
|
||||
|
||||
@@ -115,7 +115,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
return self.account.public_key.child(self.chain_number).child(index)
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
addresses = await self._query_addresses(order_by="n asc")
|
||||
addresses = await self._query_addresses(order_by="position ASC")
|
||||
max_gap = 0
|
||||
current_gap = 0
|
||||
for address in addresses:
|
||||
@@ -128,7 +128,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
|
||||
async def ensure_address_gap(self) -> List[str]:
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="n desc")
|
||||
addresses = await self._query_addresses(limit=self.gap, order_by="position DESC")
|
||||
|
||||
existing_gap = 0
|
||||
for address in addresses:
|
||||
@@ -140,7 +140,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
if existing_gap == self.gap:
|
||||
return []
|
||||
|
||||
start = addresses[0]['pubkey'].n+1 if addresses else 0
|
||||
start = addresses[0]['position']+1 if addresses else 0
|
||||
end = start + (self.gap - existing_gap)
|
||||
new_keys = await self._generate_keys(start, end-1)
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
@@ -149,22 +149,22 @@ class HierarchicalDeterministic(AddressManager):
|
||||
async def _generate_keys(self, start: int, end: int) -> List[str]:
|
||||
if not self.address_generator_lock.locked():
|
||||
raise RuntimeError('Should not be called outside of address_generator_lock.')
|
||||
keys = [self.public_key.child(index) for index in range(start, end+1)]
|
||||
keys = [(index, self.public_key.child(index)) for index in range(start, end+1)]
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, keys)
|
||||
return [key.address for key in keys]
|
||||
return [key[1].address for key in keys]
|
||||
|
||||
def get_address_records(self, only_usable: bool = False, **constraints):
|
||||
if only_usable:
|
||||
constraints['used_times__lt'] = self.maximum_uses_per_address
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = "used_times asc, n asc"
|
||||
constraints['order_by'] = "used_times ASC, position ASC"
|
||||
return self._query_addresses(**constraints)
|
||||
|
||||
|
||||
class SingleKey(AddressManager):
|
||||
""" Single Key address manager always returns the same address for all operations. """
|
||||
|
||||
name: str = "single-address"
|
||||
name = "single-address"
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@@ -190,7 +190,9 @@ class SingleKey(AddressManager):
|
||||
async with self.address_generator_lock:
|
||||
exists = await self.get_address_records()
|
||||
if not exists:
|
||||
await self.account.ledger.db.add_keys(self.account, self.chain_number, [self.public_key])
|
||||
await self.account.ledger.db.add_keys(
|
||||
self.account, self.chain_number, [(0, self.public_key)]
|
||||
)
|
||||
new_keys = [self.public_key.address]
|
||||
await self.account.ledger.announce_addresses(self, new_keys)
|
||||
return new_keys
|
||||
@@ -310,7 +312,7 @@ class BaseAccount:
|
||||
'modified_on': self.modified_on
|
||||
}
|
||||
|
||||
def merge(self, d: dict):
|
||||
def apply(self, d: dict):
|
||||
if d.get('modified_on', 0) > self.modified_on:
|
||||
self.name = d['name']
|
||||
self.modified_on = d.get('modified_on', time.time())
|
||||
@@ -318,7 +320,7 @@ class BaseAccount:
|
||||
for chain_name in ('change', 'receiving'):
|
||||
if chain_name in d['address_generator']:
|
||||
chain_object = getattr(self, chain_name)
|
||||
chain_object.merge(d['address_generator'][chain_name])
|
||||
chain_object.apply(d['address_generator'][chain_name])
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
@@ -415,16 +417,16 @@ class BaseAccount:
|
||||
}
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_utxos(account=self, **constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
return self.ledger.get_utxo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_utxo_count(account=self, **constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
return self.ledger.get_transactions(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_transactions(account=self, **constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
return self.ledger.get_transaction_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
return self.ledger.get_transaction_count(account=self, **constraints)
|
||||
|
||||
async def fund(self, to_account, amount=None, everything=False,
|
||||
outputs=1, broadcast=False, **constraints):
|
||||
|
||||
@@ -3,12 +3,12 @@ import asyncio
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
from torba.client.basetransaction import BaseTransaction, TXRefImmutable
|
||||
from torba.client.bip32 import PubKey
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,7 +19,6 @@ class AIOSQLite:
|
||||
# has to be single threaded as there is no mapping of thread:connection
|
||||
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||
self.connection: sqlite3.Connection = None
|
||||
self._closing = False
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
@@ -30,12 +29,14 @@ class AIOSQLite:
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
if self._closing:
|
||||
def __close(conn):
|
||||
self.executor.submit(conn.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
conn = self.connection
|
||||
if not conn:
|
||||
return
|
||||
self._closing = True
|
||||
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
|
||||
self.executor.shutdown(wait=True)
|
||||
self.connection = None
|
||||
return asyncio.get_event_loop_policy().get_event_loop().call_later(0.01, __close, conn)
|
||||
|
||||
def executemany(self, sql: str, params: Iterable):
|
||||
params = params if params is not None else []
|
||||
@@ -86,10 +87,10 @@ class AIOSQLite:
|
||||
if not foreign_keys_enabled:
|
||||
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
|
||||
try:
|
||||
self.connection.execute('pragma foreign_keys=off').fetchone()
|
||||
self.connection.execute('pragma foreign_keys=off')
|
||||
return self.__run_transaction(fun, *args, **kwargs)
|
||||
finally:
|
||||
self.connection.execute('pragma foreign_keys=on').fetchone()
|
||||
self.connection.execute('pragma foreign_keys=on')
|
||||
|
||||
|
||||
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
@@ -102,20 +103,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
if not key:
|
||||
sql.append(constraint)
|
||||
continue
|
||||
if key.startswith('$'):
|
||||
elif key.startswith('$'):
|
||||
values[key] = constraint
|
||||
continue
|
||||
if key.endswith('__not'):
|
||||
elif key.endswith('__not'):
|
||||
col, op = col[:-len('__not')], '!='
|
||||
elif key.endswith('__is_null'):
|
||||
col = col[:-len('__is_null')]
|
||||
sql.append(f'{col} IS NULL')
|
||||
continue
|
||||
if key.endswith('__is_not_null'):
|
||||
elif key.endswith('__is_not_null'):
|
||||
col = col[:-len('__is_not_null')]
|
||||
sql.append(f'{col} IS NOT NULL')
|
||||
continue
|
||||
if key.endswith('__lt'):
|
||||
elif key.endswith('__lt'):
|
||||
col, op = col[:-len('__lt')], '<'
|
||||
elif key.endswith('__lte'):
|
||||
col, op = col[:-len('__lte')], '<='
|
||||
@@ -149,7 +150,7 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
sql.append(f'({where})')
|
||||
values.update(subvalues)
|
||||
continue
|
||||
if key.endswith('__and'):
|
||||
elif key.endswith('__and'):
|
||||
where, subvalues = constraints_to_sql(constraint, ' AND ', key+tag+'_')
|
||||
sql.append(f'({where})')
|
||||
values.update(subvalues)
|
||||
@@ -159,14 +160,15 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
return joiner.join(sql) if sql else '', values
|
||||
|
||||
|
||||
def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
|
||||
def query(select, **constraints):
|
||||
sql = [select]
|
||||
limit = constraints.pop('limit', None)
|
||||
offset = constraints.pop('offset', None)
|
||||
order_by = constraints.pop('order_by', None)
|
||||
|
||||
accounts = constraints.pop('accounts', [])
|
||||
if accounts:
|
||||
constraints.pop('my_accounts', None)
|
||||
accounts = constraints.pop('accounts', None)
|
||||
if accounts is not None:
|
||||
constraints['account__in'] = [a.public_key.address for a in accounts]
|
||||
|
||||
where, values = constraints_to_sql(constraints)
|
||||
@@ -284,33 +286,26 @@ class SQLiteMixin:
|
||||
|
||||
class BaseDatabase(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.1"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
"""
|
||||
|
||||
CREATE_ACCOUNT_TABLE = """
|
||||
create table if not exists account_address (
|
||||
account text not null,
|
||||
address text not null,
|
||||
chain integer not null,
|
||||
pubkey blob not null,
|
||||
chain_code blob not null,
|
||||
n integer not null,
|
||||
depth integer not null,
|
||||
primary key (account, address)
|
||||
);
|
||||
create index if not exists address_account_idx on account_address (address, account);
|
||||
"""
|
||||
|
||||
CREATE_PUBKEY_ADDRESS_TABLE = """
|
||||
create table if not exists pubkey_address (
|
||||
address text primary key,
|
||||
account text not null,
|
||||
chain integer not null,
|
||||
position integer not null,
|
||||
pubkey blob not null,
|
||||
history text,
|
||||
used_times integer not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_PUBKEY_ADDRESS_INDEX = """
|
||||
create index if not exists pubkey_address_account_idx on pubkey_address (account);
|
||||
"""
|
||||
|
||||
CREATE_TX_TABLE = """
|
||||
create table if not exists tx (
|
||||
@@ -332,6 +327,8 @@ class BaseDatabase(SQLiteMixin):
|
||||
script blob not null,
|
||||
is_reserved boolean not null default 0
|
||||
);
|
||||
"""
|
||||
CREATE_TXO_INDEX = """
|
||||
create index if not exists txo_address_idx on txo (address);
|
||||
"""
|
||||
|
||||
@@ -341,17 +338,21 @@ class BaseDatabase(SQLiteMixin):
|
||||
txoid text references txo,
|
||||
address text references pubkey_address
|
||||
);
|
||||
"""
|
||||
CREATE_TXI_INDEX = """
|
||||
create index if not exists txi_address_idx on txi (address);
|
||||
create index if not exists txi_txoid_idx on txi (txoid);
|
||||
"""
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_ACCOUNT_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
CREATE_TX_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_TABLE +
|
||||
CREATE_PUBKEY_ADDRESS_INDEX +
|
||||
CREATE_TXO_TABLE +
|
||||
CREATE_TXI_TABLE
|
||||
CREATE_TXO_INDEX +
|
||||
CREATE_TXI_TABLE +
|
||||
CREATE_TXI_INDEX
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -376,10 +377,10 @@ class BaseDatabase(SQLiteMixin):
|
||||
}
|
||||
|
||||
async def insert_transaction(self, tx):
|
||||
await self.db.execute_fetchall(*self._insert_sql('tx', self.tx_to_row(tx)))
|
||||
await self.db.execute(*self._insert_sql('tx', self.tx_to_row(tx)))
|
||||
|
||||
async def update_transaction(self, tx):
|
||||
await self.db.execute_fetchall(*self._update_sql("tx", {
|
||||
await self.db.execute(*self._update_sql("tx", {
|
||||
'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
|
||||
}, 'txid = ?', (tx.id,)))
|
||||
|
||||
@@ -390,7 +391,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
|
||||
)).fetchall()
|
||||
))
|
||||
elif txo.script.is_pay_script_hash:
|
||||
# TODO: implement script hash payments
|
||||
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
|
||||
@@ -403,7 +404,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
'txid': tx.id,
|
||||
'txoid': txo.id,
|
||||
'address': address,
|
||||
}, ignore_duplicate=True)).fetchall()
|
||||
}, ignore_duplicate=True))
|
||||
|
||||
conn.execute(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
@@ -434,22 +435,25 @@ class BaseDatabase(SQLiteMixin):
|
||||
|
||||
async def select_transactions(self, cols, accounts=None, **constraints):
|
||||
if not set(constraints) & {'txid', 'txid__in'}:
|
||||
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
|
||||
assert accounts is not None, "'accounts' argument required when no 'txid' constraint"
|
||||
constraints.update({
|
||||
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
|
||||
})
|
||||
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
|
||||
where = f" WHERE account_address.account IN ({account_values})"
|
||||
constraints['txid__in'] = f"""
|
||||
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
|
||||
SELECT txo.txid FROM txo
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
UNION
|
||||
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
|
||||
SELECT txi.txid FROM txi
|
||||
INNER JOIN pubkey_address USING (address) WHERE pubkey_address.account IN ({account_values})
|
||||
"""
|
||||
return await self.db.execute_fetchall(
|
||||
*query("SELECT {} FROM tx".format(cols), **constraints)
|
||||
)
|
||||
|
||||
async def get_transactions(self, wallet=None, **constraints):
|
||||
async def get_transactions(self, **constraints):
|
||||
accounts = constraints.get('accounts', None)
|
||||
|
||||
tx_rows = await self.select_transactions(
|
||||
'txid, raw, height, position, is_verified',
|
||||
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
|
||||
@@ -474,7 +478,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
annotated_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
wallet=wallet,
|
||||
my_accounts=accounts,
|
||||
txid__in=txids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -484,7 +488,7 @@ class BaseDatabase(SQLiteMixin):
|
||||
referenced_txos.update({
|
||||
txo.id: txo for txo in
|
||||
(await self.get_txos(
|
||||
wallet=wallet,
|
||||
my_accounts=accounts,
|
||||
txoid__in=txi_txoids[offset:offset+step],
|
||||
))
|
||||
})
|
||||
@@ -504,7 +508,6 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs
|
||||
|
||||
async def get_transaction_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -517,24 +520,23 @@ class BaseDatabase(SQLiteMixin):
|
||||
return txs[0]
|
||||
|
||||
async def select_txos(self, cols, **constraints):
|
||||
sql = "SELECT {} FROM txo JOIN tx USING (txid)"
|
||||
if 'accounts' in constraints:
|
||||
sql += " JOIN account_address USING (address)"
|
||||
return await self.db.execute_fetchall(*query(sql.format(cols), **constraints))
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM txo"
|
||||
" JOIN pubkey_address USING (address)"
|
||||
" JOIN tx USING (txid)".format(cols), **constraints
|
||||
))
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, **constraints):
|
||||
my_accounts = set(a.public_key.address for a in wallet.accounts) if wallet else set()
|
||||
async def get_txos(self, my_accounts=None, no_tx=False, **constraints):
|
||||
my_accounts = [
|
||||
(a.public_key.address if isinstance(a, BaseAccount) else a)
|
||||
for a in (my_accounts or constraints.get('accounts', []))
|
||||
]
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"]
|
||||
rows = await self.select_txos(
|
||||
"""
|
||||
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
|
||||
select group_concat(account||"|"||chain) from account_address
|
||||
where account_address.address=txo.address
|
||||
)
|
||||
""",
|
||||
"tx.txid, raw, tx.height, tx.position, tx.is_verified, "
|
||||
"txo.position, chain, account, amount, script",
|
||||
**constraints
|
||||
)
|
||||
txos = []
|
||||
@@ -543,8 +545,8 @@ class BaseDatabase(SQLiteMixin):
|
||||
for row in rows:
|
||||
if no_tx:
|
||||
txo = output_class(
|
||||
amount=row[6],
|
||||
script=output_class.script_class(row[7]),
|
||||
amount=row[8],
|
||||
script=output_class.script_class(row[9]),
|
||||
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
|
||||
position=row[5]
|
||||
)
|
||||
@@ -554,18 +556,12 @@ class BaseDatabase(SQLiteMixin):
|
||||
row[1], height=row[2], position=row[3], is_verified=row[4]
|
||||
)
|
||||
txo = txs[row[0]].outputs[row[5]]
|
||||
row_accounts = dict(a.split('|') for a in row[8].split(','))
|
||||
account_match = set(row_accounts) & my_accounts
|
||||
if account_match:
|
||||
txo.is_my_account = True
|
||||
txo.is_change = row_accounts[account_match.pop()] == '1'
|
||||
else:
|
||||
txo.is_change = txo.is_my_account = False
|
||||
txo.is_change = row[6] == 1
|
||||
txo.is_my_account = row[7] in my_accounts
|
||||
txos.append(txo)
|
||||
return txos
|
||||
|
||||
async def get_txo_count(self, **constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -585,60 +581,45 @@ class BaseDatabase(SQLiteMixin):
|
||||
self.constrain_utxo(constraints)
|
||||
return self.get_txo_count(**constraints)
|
||||
|
||||
async def get_balance(self, wallet=None, accounts=None, **constraints):
|
||||
assert wallet or accounts, \
|
||||
"'wallet' or 'accounts' constraints required to calculate balance"
|
||||
constraints['accounts'] = accounts or wallet.accounts
|
||||
async def get_balance(self, **constraints):
|
||||
self.constrain_utxo(constraints)
|
||||
balance = await self.select_txos('SUM(amount)', **constraints)
|
||||
return balance[0][0] or 0
|
||||
|
||||
async def select_addresses(self, cols, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
"SELECT {} FROM pubkey_address JOIN account_address USING (address)".format(cols),
|
||||
**constraints
|
||||
"SELECT {} FROM pubkey_address".format(cols), **constraints
|
||||
))
|
||||
|
||||
async def get_addresses(self, cols=None, **constraints):
|
||||
cols = cols or (
|
||||
'address', 'account', 'chain', 'history', 'used_times',
|
||||
'pubkey', 'chain_code', 'n', 'depth'
|
||||
)
|
||||
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
|
||||
for address in addresses:
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
return addresses
|
||||
async def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'),
|
||||
**constraints):
|
||||
addresses = await self.select_addresses(', '.join(cols), **constraints)
|
||||
return rows_to_dict(addresses, cols)
|
||||
|
||||
async def get_address_count(self, **constraints):
|
||||
count = await self.select_addresses('count(*)', **constraints)
|
||||
return count[0][0]
|
||||
|
||||
async def get_address(self, **constraints):
|
||||
addresses = await self.get_addresses(limit=1, **constraints)
|
||||
addresses = await self.get_addresses(
|
||||
cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
|
||||
limit=1, **constraints
|
||||
)
|
||||
if addresses:
|
||||
return addresses[0]
|
||||
|
||||
async def add_keys(self, account, chain, pubkeys):
|
||||
async def add_keys(self, account, chain, keys):
|
||||
await self.db.executemany(
|
||||
"insert or ignore into account_address "
|
||||
"(account, address, chain, pubkey, chain_code, n, depth) values "
|
||||
"(?, ?, ?, ?, ?, ?, ?)", ((
|
||||
account.id, k.address, chain,
|
||||
sqlite3.Binary(k.pubkey_bytes),
|
||||
sqlite3.Binary(k.chain_code),
|
||||
k.n, k.depth
|
||||
) for k in pubkeys)
|
||||
)
|
||||
await self.db.executemany(
|
||||
"insert or ignore into pubkey_address (address) values (?)",
|
||||
((pubkey.address,) for pubkey in pubkeys)
|
||||
"insert into pubkey_address (address, account, chain, position, pubkey) values (?, ?, ?, ?, ?)",
|
||||
(
|
||||
(pubkey.address, account.public_key.address, chain, position,
|
||||
sqlite3.Binary(pubkey.pubkey_bytes))
|
||||
for position, pubkey in keys
|
||||
)
|
||||
)
|
||||
|
||||
async def _set_address_history(self, address, history):
|
||||
await self.db.execute_fetchall(
|
||||
await self.db.execute(
|
||||
"UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
|
||||
(history, history.count(':')//2, address)
|
||||
)
|
||||
|
||||
@@ -19,7 +19,6 @@ from torba.client.coinselection import CoinSelector
|
||||
from torba.client.constants import COIN, NULL_HASH32
|
||||
from torba.stream import StreamController
|
||||
from torba.client.hash import hash160, double_sha256, sha256, Base58
|
||||
from torba.client.bip32 import PubKey, PrivateKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -161,7 +160,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
decoded = Base58.decode_check(address)
|
||||
decoded = Base58.decode(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
@@ -179,29 +178,27 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def add_account(self, account: baseaccount.BaseAccount):
|
||||
self.accounts.append(account)
|
||||
|
||||
async def _get_account_and_address_info_for_address(self, wallet, address):
|
||||
match = await self.db.get_address(accounts=wallet.accounts, address=address)
|
||||
async def _get_account_and_address_info_for_address(self, address):
|
||||
match = await self.db.get_address(address=address)
|
||||
if match:
|
||||
for account in wallet.accounts:
|
||||
for account in self.accounts:
|
||||
if match['account'] == account.public_key.address:
|
||||
return account, match
|
||||
|
||||
async def get_private_key_for_address(self, wallet, address) -> Optional[PrivateKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
async def get_private_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
if match:
|
||||
account, address_info = match
|
||||
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
|
||||
return None
|
||||
return account.get_private_key(address_info['chain'], address_info['position'])
|
||||
|
||||
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
async def get_public_key_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
if match:
|
||||
_, address_info = match
|
||||
return address_info['pubkey']
|
||||
return None
|
||||
account, address_info = match
|
||||
return account.get_public_key(address_info['chain'], address_info['position'])
|
||||
|
||||
async def get_account_for_address(self, wallet, address):
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
async def get_account_for_address(self, address):
|
||||
match = await self._get_account_and_address_info_for_address(address)
|
||||
if match:
|
||||
return match[0]
|
||||
|
||||
@@ -213,12 +210,6 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
estimators.append(utxo.get_estimator(self))
|
||||
return estimators
|
||||
|
||||
async def get_addresses(self, **constraints):
|
||||
return await self.db.get_addresses(**constraints)
|
||||
|
||||
def get_address_count(self, **constraints):
|
||||
return self.db.get_address_count(**constraints)
|
||||
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts):
|
||||
async with self._utxo_reservation_lock:
|
||||
txos = await self.get_effective_amount_estimators(funding_accounts)
|
||||
@@ -238,16 +229,29 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
def release_tx(self, tx):
|
||||
return self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
|
||||
|
||||
def constraint_account_or_all(self, constraints):
|
||||
if 'accounts' in constraints:
|
||||
return
|
||||
account = constraints.pop('account', None)
|
||||
if account:
|
||||
constraints['accounts'] = [account]
|
||||
else:
|
||||
constraints['accounts'] = self.accounts
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxos(**constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_utxo_count(**constraints)
|
||||
|
||||
def get_transactions(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transactions(**constraints)
|
||||
|
||||
def get_transaction_count(self, **constraints):
|
||||
self.constraint_account_or_all(constraints)
|
||||
return self.db.get_transaction_count(**constraints)
|
||||
|
||||
async def get_local_status_and_history(self, address, history=None):
|
||||
@@ -544,7 +548,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
return None
|
||||
|
||||
def broadcast(self, tx):
|
||||
# broadcast can't be a retriable call yet
|
||||
# broadcast cant be a retriable call yet
|
||||
return self.network.broadcast(hexlify(tx.raw).decode())
|
||||
|
||||
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
|
||||
@@ -558,7 +562,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
addresses.add(
|
||||
self.hash160_to_address(txo.script.values['pubkey_hash'])
|
||||
)
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
records = await self.db.get_addresses(cols=('address',), address__in=addresses)
|
||||
_, pending = await asyncio.wait([
|
||||
self.on_transaction.where(partial(
|
||||
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Type, MutableSequence, MutableMapping, Optional
|
||||
from typing import Type, MutableSequence, MutableMapping
|
||||
|
||||
from torba.client.baseledger import BaseLedger, LedgerRegistry
|
||||
from torba.client.wallet import Wallet, WalletStorage
|
||||
@@ -41,6 +41,16 @@ class BaseWalletManager:
|
||||
self.wallets.append(wallet)
|
||||
return wallet
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@property
|
||||
def default_wallet(self):
|
||||
for wallet in self.wallets:
|
||||
@@ -68,14 +78,3 @@ class BaseWalletManager:
|
||||
l.stop() for l in self.ledgers.values()
|
||||
))
|
||||
self.running = False
|
||||
|
||||
def get_wallet_or_default(self, wallet_id: Optional[str]) -> Wallet:
|
||||
if wallet_id is None:
|
||||
return self.default_wallet
|
||||
return self.get_wallet_or_error(wallet_id)
|
||||
|
||||
def get_wallet_or_error(self, wallet_id: str) -> Wallet:
|
||||
for wallet in self.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
return wallet
|
||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
||||
@@ -73,7 +73,7 @@ class ClientSession(BaseClientSession):
|
||||
return reply
|
||||
except (RPCError, ProtocolError) as e:
|
||||
if str(e).find('.*no such .*transaction.*'):
|
||||
# shouldn't the server return none instead?
|
||||
# shouldnt the server return none instead?
|
||||
return None
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
@@ -185,9 +185,6 @@ class BaseNetwork:
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
|
||||
@@ -256,14 +253,11 @@ class BaseNetwork:
|
||||
try:
|
||||
return await self.rpc('blockchain.address.subscribe', [address], True)
|
||||
except asyncio.TimeoutError:
|
||||
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
|
||||
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
|
||||
if self.client:
|
||||
self.client.abort()
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
def get_server_features(self):
|
||||
return self.rpc('server.features', (), restricted=True)
|
||||
|
||||
|
||||
class SessionPool:
|
||||
|
||||
@@ -287,7 +281,7 @@ class SessionPool:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
for session in self.available_sessions],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
import typing
|
||||
from typing import List, Iterable, Optional, Tuple
|
||||
from typing import List, Iterable, Optional
|
||||
from binascii import hexlify
|
||||
|
||||
from torba.client.basescript import BaseInputScript, BaseOutputScript
|
||||
@@ -12,7 +12,7 @@ from torba.client.util import ReadOnlyList
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from torba.client import baseledger, wallet as basewallet
|
||||
from torba.client import baseledger
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
@@ -431,32 +431,21 @@ class BaseTransaction:
|
||||
self.locktime = stream.read_uint32()
|
||||
|
||||
@classmethod
|
||||
def ensure_all_have_same_ledger_and_wallet(
|
||||
cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> Tuple['baseledger.BaseLedger', 'basewallet.Wallet']:
|
||||
ledger = wallet = None
|
||||
def ensure_all_have_same_ledger(cls, funding_accounts: Iterable[BaseAccount],
|
||||
change_account: BaseAccount = None) -> 'baseledger.BaseLedger':
|
||||
ledger = None
|
||||
for account in funding_accounts:
|
||||
if ledger is None:
|
||||
ledger = account.ledger
|
||||
wallet = account.wallet
|
||||
if ledger != account.ledger:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be on the same ledger.'
|
||||
)
|
||||
if wallet != account.wallet:
|
||||
raise ValueError(
|
||||
'All funding accounts used to create a transaction must be from the same wallet.'
|
||||
)
|
||||
if change_account is not None:
|
||||
if change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if change_account.wallet != wallet:
|
||||
raise ValueError('Change account must use same wallet as funding accounts.')
|
||||
if change_account is not None and change_account.ledger != ledger:
|
||||
raise ValueError('Change account must use same ledger as funding accounts.')
|
||||
if ledger is None:
|
||||
raise ValueError('No ledger found.')
|
||||
if wallet is None:
|
||||
raise ValueError('No wallet found.')
|
||||
return ledger, wallet
|
||||
return ledger
|
||||
|
||||
@classmethod
|
||||
async def create(cls, inputs: Iterable[BaseInput], outputs: Iterable[BaseOutput],
|
||||
@@ -469,7 +458,7 @@ class BaseTransaction:
|
||||
.add_inputs(inputs) \
|
||||
.add_outputs(outputs)
|
||||
|
||||
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
|
||||
# value of the outputs plus associated fees
|
||||
cost = (
|
||||
@@ -507,17 +496,18 @@ class BaseTransaction:
|
||||
|
||||
if tx._outputs:
|
||||
break
|
||||
# this condition and the outer range(5) loop cover an edge case
|
||||
# whereby a single input is just enough to cover the fee and
|
||||
# has some change left over, but the change left over is less
|
||||
# than the cost_of_change: thus the input is completely
|
||||
# consumed and no output is added, which is an invalid tx.
|
||||
# to be able to spend this input we must increase the cost
|
||||
# of the TX and run through the balance algorithm a second time
|
||||
# adding an extra input and change output, making tx valid.
|
||||
# we do this 5 times in case the other UTXOs added are also
|
||||
# less than the fee, after 5 attempts we give up and go home
|
||||
cost += cost_of_change + 1
|
||||
else:
|
||||
# this condition and the outer range(5) loop cover an edge case
|
||||
# whereby a single input is just enough to cover the fee and
|
||||
# has some change left over, but the change left over is less
|
||||
# than the cost_of_change: thus the input is completely
|
||||
# consumed and no output is added, which is an invalid tx.
|
||||
# to be able to spend this input we must increase the cost
|
||||
# of the TX and run through the balance algorithm a second time
|
||||
# adding an extra input and change output, making tx valid.
|
||||
# we do this 5 times in case the other UTXOs added are also
|
||||
# less than the fee, after 5 attempts we give up and go home
|
||||
cost += cost_of_change + 1
|
||||
|
||||
if sign:
|
||||
await tx.sign(funding_accounts)
|
||||
@@ -534,15 +524,14 @@ class BaseTransaction:
|
||||
return hash_type
|
||||
|
||||
async def sign(self, funding_accounts: Iterable[BaseAccount]):
|
||||
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
|
||||
ledger = self.ensure_all_have_same_ledger(funding_accounts)
|
||||
for i, txi in enumerate(self._inputs):
|
||||
assert txi.script is not None
|
||||
assert txi.txo_ref.txo is not None
|
||||
txo_script = txi.txo_ref.txo.script
|
||||
if txo_script.is_pay_pubkey_hash:
|
||||
address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
|
||||
private_key = await ledger.get_private_key_for_address(wallet, address)
|
||||
assert private_key is not None, 'Cannot find private key for signing output.'
|
||||
private_key = await ledger.get_private_key_for_address(address)
|
||||
tx = self._serialize_for_signature(i)
|
||||
txi.script.values['signature'] = \
|
||||
private_key.sign(tx) + bytes((self.signature_hash_type(1),))
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import os
|
||||
import time
|
||||
import stat
|
||||
import json
|
||||
import zlib
|
||||
import typing
|
||||
from typing import List, Sequence, MutableSequence, Optional
|
||||
from collections import UserDict
|
||||
from typing import Sequence, MutableSequence
|
||||
from hashlib import sha256
|
||||
from operator import attrgetter
|
||||
from torba.client.hash import better_aes_encrypt, better_aes_decrypt
|
||||
@@ -14,36 +12,6 @@ if typing.TYPE_CHECKING:
|
||||
from torba.client import basemanager, baseaccount, baseledger
|
||||
|
||||
|
||||
class TimestampedPreferences(UserDict):
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.data[key]['value']
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.data[key] = {
|
||||
'value': value,
|
||||
'ts': time.time()
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.to_dict_without_ts())
|
||||
|
||||
def to_dict_without_ts(self):
|
||||
return {
|
||||
key: value['value'] for key, value in self.data.items()
|
||||
}
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
return sha256(json.dumps(self.data).encode()).digest()
|
||||
|
||||
def merge(self, other: dict):
|
||||
for key, value in other.items():
|
||||
if key in self.data and value['ts'] < self.data[key]['ts']:
|
||||
continue
|
||||
self.data[key] = value
|
||||
|
||||
|
||||
class Wallet:
|
||||
""" The primary role of Wallet is to encapsulate a collection
|
||||
of accounts (seed/private keys) and the spending rules / settings
|
||||
@@ -51,66 +19,23 @@ class Wallet:
|
||||
by physical files on the filesystem.
|
||||
"""
|
||||
|
||||
preferences: TimestampedPreferences
|
||||
|
||||
def __init__(self, name: str = 'Wallet', accounts: MutableSequence['baseaccount.BaseAccount'] = None,
|
||||
storage: 'WalletStorage' = None, preferences: dict = None) -> None:
|
||||
storage: 'WalletStorage' = None) -> None:
|
||||
self.name = name
|
||||
self.accounts = accounts or []
|
||||
self.storage = storage or WalletStorage()
|
||||
self.preferences = TimestampedPreferences(preferences or {})
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
if self.storage.path:
|
||||
return os.path.basename(self.storage.path)
|
||||
return self.name
|
||||
|
||||
def add_account(self, account: 'baseaccount.BaseAccount'):
|
||||
def add_account(self, account):
|
||||
self.accounts.append(account)
|
||||
|
||||
def generate_account(self, ledger: 'baseledger.BaseLedger') -> 'baseaccount.BaseAccount':
|
||||
return ledger.account_class.generate(ledger, self)
|
||||
|
||||
@property
|
||||
def default_account(self) -> Optional['baseaccount.BaseAccount']:
|
||||
for account in self.accounts:
|
||||
return account
|
||||
return None
|
||||
|
||||
def get_account_or_default(self, account_id: str) -> Optional['baseaccount.BaseAccount']:
|
||||
if account_id is None:
|
||||
return self.default_account
|
||||
return self.get_account_or_error(account_id)
|
||||
|
||||
def get_account_or_error(self, account_id: str) -> 'baseaccount.BaseAccount':
|
||||
for account in self.accounts:
|
||||
if account.id == account_id:
|
||||
return account
|
||||
raise ValueError(f"Couldn't find account: {account_id}.")
|
||||
|
||||
def get_accounts_or_all(self, account_ids: List[str]) -> Sequence['baseaccount.BaseAccount']:
|
||||
return [
|
||||
self.get_account_or_error(account_id)
|
||||
for account_id in account_ids
|
||||
] if account_ids else self.accounts
|
||||
|
||||
async def get_detailed_accounts(self, **kwargs):
|
||||
ledgers = {}
|
||||
for i, account in enumerate(self.accounts):
|
||||
details = await account.get_details(**kwargs)
|
||||
details['is_default'] = i == 0
|
||||
ledger_id = account.ledger.get_id()
|
||||
ledgers.setdefault(ledger_id, [])
|
||||
ledgers[ledger_id].append(details)
|
||||
return ledgers
|
||||
|
||||
@classmethod
|
||||
def from_storage(cls, storage: 'WalletStorage', manager: 'basemanager.BaseWalletManager') -> 'Wallet':
|
||||
json_dict = storage.read()
|
||||
wallet = cls(
|
||||
name=json_dict.get('name', 'Wallet'),
|
||||
preferences=json_dict.get('preferences', {}),
|
||||
storage=storage
|
||||
)
|
||||
account_dicts: Sequence[dict] = json_dict.get('accounts', [])
|
||||
@@ -123,17 +48,20 @@ class Wallet:
|
||||
return {
|
||||
'version': WalletStorage.LATEST_VERSION,
|
||||
'name': self.name,
|
||||
'preferences': self.preferences.data,
|
||||
'accounts': [a.to_dict() for a in self.accounts]
|
||||
}
|
||||
|
||||
def save(self):
|
||||
self.storage.write(self.to_dict())
|
||||
|
||||
@property
|
||||
def default_account(self):
|
||||
for account in self.accounts:
|
||||
return account
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256()
|
||||
h.update(self.preferences.hash)
|
||||
for account in sorted(self.accounts, key=attrgetter('id')):
|
||||
h.update(account.hash)
|
||||
return h.digest()
|
||||
@@ -149,69 +77,6 @@ class Wallet:
|
||||
decompressed = zlib.decompress(decrypted)
|
||||
return json.loads(decompressed)
|
||||
|
||||
def merge(self, manager: 'basemanager.BaseWalletManager',
|
||||
password: str, data: str) -> List['baseaccount.BaseAccount']:
|
||||
added_accounts = []
|
||||
decrypted_data = self.unpack(password, data)
|
||||
self.preferences.merge(decrypted_data.get('preferences', {}))
|
||||
for account_dict in decrypted_data['accounts']:
|
||||
ledger = manager.get_or_create_ledger(account_dict['ledger'])
|
||||
_, _, pubkey = ledger.account_class.keys_from_dict(ledger, account_dict)
|
||||
account_id = pubkey.address
|
||||
local_match = None
|
||||
for local_account in self.accounts:
|
||||
if account_id == local_account.id:
|
||||
local_match = local_account
|
||||
break
|
||||
if local_match is not None:
|
||||
local_match.merge(account_dict)
|
||||
else:
|
||||
new_account = ledger.account_class.from_dict(ledger, self, account_dict)
|
||||
added_accounts.append(new_account)
|
||||
return added_accounts
|
||||
|
||||
@property
|
||||
def is_locked(self) -> bool:
|
||||
for account in self.accounts:
|
||||
if account.encrypted:
|
||||
return True
|
||||
return False
|
||||
|
||||
def unlock(self, password):
|
||||
for account in self.accounts:
|
||||
if account.encrypted:
|
||||
account.decrypt(password)
|
||||
return True
|
||||
|
||||
def lock(self):
|
||||
for account in self.accounts:
|
||||
if not account.encrypted:
|
||||
assert account.password is not None, "account was never encrypted"
|
||||
account.encrypt(account.password)
|
||||
return True
|
||||
|
||||
@property
|
||||
def is_encrypted(self) -> bool:
|
||||
for account in self.accounts:
|
||||
if account.serialize_encrypted:
|
||||
return True
|
||||
return False
|
||||
|
||||
def decrypt(self):
|
||||
for account in self.accounts:
|
||||
account.serialize_encrypted = False
|
||||
self.save()
|
||||
return True
|
||||
|
||||
def encrypt(self, password):
|
||||
for account in self.accounts:
|
||||
if not account.encrypted:
|
||||
account.encrypt(password)
|
||||
account.serialize_encrypted = True
|
||||
self.save()
|
||||
self.unlock(password)
|
||||
return True
|
||||
|
||||
|
||||
class WalletStorage:
|
||||
|
||||
@@ -222,7 +87,6 @@ class WalletStorage:
|
||||
self._default = default or {
|
||||
'version': self.LATEST_VERSION,
|
||||
'name': 'My Wallet',
|
||||
'preferences': {},
|
||||
'accounts': []
|
||||
}
|
||||
|
||||
|
||||
@@ -14,11 +14,11 @@ from torba.client.baseheader import BaseHeaders, ArithUint256
|
||||
|
||||
|
||||
class MainHeaders(BaseHeaders):
|
||||
header_size: int = 80
|
||||
chunk_size: int = 2016
|
||||
max_target: int = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
header_size = 80
|
||||
chunk_size = 2016
|
||||
max_target = 0x00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash: Optional[bytes] = b'000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
|
||||
target_timespan: int = 14 * 24 * 60 * 60
|
||||
target_timespan = 14 * 24 * 60 * 60
|
||||
|
||||
@staticmethod
|
||||
def serialize(header: dict) -> bytes:
|
||||
|
||||
@@ -68,7 +68,7 @@ def set_logging(ledger_module, level, handler=None):
|
||||
class Conductor:
|
||||
|
||||
def __init__(self, ledger_module=None, manager_module=None, verbosity=logging.WARNING,
|
||||
enable_segwit=False, seed=None):
|
||||
enable_segwit=False):
|
||||
self.ledger_module = ledger_module or get_ledger_from_environment()
|
||||
self.manager_module = manager_module or get_manager_from_environment()
|
||||
self.spv_module = get_spvserver_from_ledger(self.ledger_module)
|
||||
@@ -76,9 +76,7 @@ class Conductor:
|
||||
self.blockchain_node = get_blockchain_node_from_ledger(self.ledger_module)
|
||||
self.blockchain_node.segwit_enabled = enable_segwit
|
||||
self.spv_node = SPVNode(self.spv_module)
|
||||
self.wallet_node = WalletNode(
|
||||
self.manager_module, self.ledger_module.RegTestLedger, default_seed=seed
|
||||
)
|
||||
self.wallet_node = WalletNode(self.manager_module, self.ledger_module.RegTestLedger)
|
||||
|
||||
set_logging(self.ledger_module, verbosity)
|
||||
|
||||
@@ -140,7 +138,7 @@ class Conductor:
|
||||
class WalletNode:
|
||||
|
||||
def __init__(self, manager_class: Type[BaseWalletManager], ledger_class: Type[BaseLedger],
|
||||
verbose: bool = False, port: int = 5280, default_seed: str = None) -> None:
|
||||
verbose: bool = False, port: int = 5280) -> None:
|
||||
self.manager_class = manager_class
|
||||
self.ledger_class = ledger_class
|
||||
self.verbose = verbose
|
||||
@@ -150,13 +148,10 @@ class WalletNode:
|
||||
self.account: Optional[BaseAccount] = None
|
||||
self.data_path: Optional[str] = None
|
||||
self.port = port
|
||||
self.default_seed = default_seed
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
wallet_file_name = os.path.join(self.data_path, 'my_wallet.json')
|
||||
with open(wallet_file_name, 'w') as wallet_file:
|
||||
wallet_file.write('{"version": 1, "accounts": []}\n')
|
||||
self.manager = self.manager_class.from_config({
|
||||
@@ -171,14 +166,14 @@ class WalletNode:
|
||||
})
|
||||
self.ledger = self.manager.ledgers[self.ledger_class]
|
||||
self.wallet = self.manager.default_wallet
|
||||
if not self.wallet:
|
||||
raise ValueError('Wallet is required.')
|
||||
if seed or self.default_seed:
|
||||
if seed is None and self.wallet is not None:
|
||||
self.wallet.generate_account(self.ledger)
|
||||
elif self.wallet is not None:
|
||||
self.ledger.account_class.from_dict(
|
||||
self.ledger, self.wallet, {'seed': seed or self.default_seed}
|
||||
self.ledger, self.wallet, {'seed': seed}
|
||||
)
|
||||
else:
|
||||
self.wallet.generate_account(self.ledger)
|
||||
raise ValueError('Wallet is required.')
|
||||
self.account = self.wallet.default_account
|
||||
if connect:
|
||||
await self.manager.start()
|
||||
|
||||
@@ -37,7 +37,7 @@ class FramerBase(object):
|
||||
"""Abstract base class for a framer.
|
||||
|
||||
A framer breaks an incoming byte stream into protocol messages,
|
||||
buffering if necessary. It also frames outgoing messages into
|
||||
buffering if necesary. It also frames outgoing messages into
|
||||
a byte stream.
|
||||
"""
|
||||
|
||||
|
||||
@@ -513,7 +513,7 @@ class JSONRPCv2(JSONRPC):
|
||||
|
||||
|
||||
class JSONRPCLoose(JSONRPC):
|
||||
"""A relaxed version of JSON RPC."""
|
||||
"""A relaxed versin of JSON RPC."""
|
||||
|
||||
# Don't be so loose we accept any old message ID
|
||||
_message_id = JSONRPCv2._message_id
|
||||
|
||||
@@ -93,7 +93,7 @@ class SessionBase(asyncio.Protocol):
|
||||
# Set when a connection is made
|
||||
self._address = None
|
||||
self._proxy_address = None
|
||||
# For logger.debug messages
|
||||
# For logger.debug messsages
|
||||
self.verbosity = 0
|
||||
# Cleared when the send socket is full
|
||||
self._can_send = Event()
|
||||
|
||||
@@ -117,7 +117,7 @@ class Daemon:
|
||||
async def _send(self, payload, processor):
|
||||
"""Send a payload to be converted to JSON.
|
||||
|
||||
Handles temporary connection issues. Daemon response errors
|
||||
Handles temporary connection issues. Daemon reponse errors
|
||||
are raise through DaemonError.
|
||||
"""
|
||||
def log_error(error):
|
||||
|
||||
@@ -80,8 +80,6 @@ class Env:
|
||||
self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
|
||||
@@ -228,7 +228,7 @@ class History:
|
||||
|
||||
def _compact_hashX(self, hashX, hist_map, hist_list,
|
||||
write_items, keys_to_delete):
|
||||
"""Compress history for a hashX. hist_list is an ordered list of
|
||||
"""Compres history for a hashX. hist_list is an ordered list of
|
||||
the histories to be compressed."""
|
||||
# History entries (tx numbers) are 4 bytes each. Distribute
|
||||
# over rows of up to 50KB in size. A fixed row size means
|
||||
|
||||
@@ -385,7 +385,7 @@ class SessionManager:
|
||||
return "peer '{}' added".format(real_name)
|
||||
|
||||
async def rpc_disconnect(self, session_ids):
|
||||
"""Disconnect sessions.
|
||||
"""Disconnect sesssions.
|
||||
|
||||
session_ids: array of session IDs
|
||||
"""
|
||||
@@ -397,7 +397,7 @@ class SessionManager:
|
||||
return await self._for_each_session(session_ids, close)
|
||||
|
||||
async def rpc_log(self, session_ids):
|
||||
"""Toggle logging of sessions.
|
||||
"""Toggle logging of sesssions.
|
||||
|
||||
session_ids: array of session IDs
|
||||
"""
|
||||
@@ -414,7 +414,7 @@ class SessionManager:
|
||||
try:
|
||||
self.daemon.set_url(daemon_url)
|
||||
except Exception as e:
|
||||
raise RPCError(BAD_REQUEST, f'an error occurred: {e!r}')
|
||||
raise RPCError(BAD_REQUEST, f'an error occured: {e!r}')
|
||||
return f'now using daemon at {self.daemon.logged_url()}'
|
||||
|
||||
async def rpc_stop(self):
|
||||
@@ -767,9 +767,6 @@ class ElectrumX(SessionBase):
|
||||
'protocol_min': min_str,
|
||||
'protocol_max': max_str,
|
||||
'genesis_hash': env.coin.GENESIS_HASH,
|
||||
'description': env.description,
|
||||
'payment_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
}
|
||||
|
||||
|
||||
@@ -457,7 +457,7 @@ class TxInputTokenPay(TxInput):
|
||||
self.script[1] == self.OP_ANON_MARKER)
|
||||
|
||||
def is_generation(self):
|
||||
# Transactions coming in from stealth addresses are seen by
|
||||
# Transactions comming in from stealth addresses are seen by
|
||||
# the blockchain as newly minted coins. The reverse, where coins
|
||||
# are sent TO a stealth address, are seen by the blockchain as
|
||||
# a coin burn.
|
||||
|
||||
@@ -183,7 +183,6 @@ class AdvanceTimeTestCase(AsyncioTestCase):
|
||||
|
||||
class IntegrationTestCase(AsyncioTestCase):
|
||||
|
||||
SEED = None
|
||||
LEDGER = None
|
||||
MANAGER = None
|
||||
ENABLE_SEGWIT = False
|
||||
@@ -202,7 +201,7 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.conductor = Conductor(
|
||||
ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY,
|
||||
enable_segwit=self.ENABLE_SEGWIT, seed=self.SEED
|
||||
enable_segwit=self.ENABLE_SEGWIT
|
||||
)
|
||||
await self.conductor.start_blockchain()
|
||||
self.addCleanup(self.conductor.stop_blockchain)
|
||||
|
||||
+2
-2
@@ -13,6 +13,6 @@ changedir = {toxinidir}/tests
|
||||
setenv =
|
||||
integration: TORBA_LEDGER={envname}
|
||||
commands =
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.unit
|
||||
unit: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.unit
|
||||
integration: orchstr8 download
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -vv -t . client_tests.integration
|
||||
integration: coverage run -p --source={envsitepackagesdir}/torba -m unittest discover -t . client_tests.integration
|
||||
Reference in new issue
Block a user