Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8529b7a6a8 |
No files matched your search
@@ -18,7 +18,7 @@ jobs:
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[lint]
|
||||
- run: pip install -e .[torrent,lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
string: ${{ runner.os }}
|
||||
- run: python -m pip install --user --upgrade pip wheel
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: pip install -e .[test]
|
||||
run: pip install -e .[torrent,test]
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
env:
|
||||
HOME: /tmp
|
||||
@@ -81,6 +81,8 @@ jobs:
|
||||
- claims
|
||||
- takeovers
|
||||
- transactions
|
||||
- claims_legacy_search
|
||||
- takeovers_legacy_search
|
||||
- other
|
||||
steps:
|
||||
- name: Configure sysctl limits
|
||||
|
||||
@@ -2,7 +2,6 @@ FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG projects_dir=/home/$user
|
||||
ARG db_dir=/database
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
@@ -28,16 +27,12 @@ RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN python3 -m pip install -U setuptools pip
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
VOLUME $db_dir
|
||||
ENTRYPOINT ["python3", "scripts/dht_node.py"]
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.110.0"
|
||||
__version__ = "0.109.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -624,10 +624,6 @@ class Config(CLIConfig):
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 2
|
||||
)
|
||||
is_bootstrap_node = Toggle(
|
||||
"When running as a bootstrap node, disable all logic related to balancing the routing table, so we can "
|
||||
"add as many peers as possible and better help first-runs.", False
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
|
||||
+15
-6
@@ -30,12 +30,12 @@ class Node:
|
||||
)
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
|
||||
storage: typing.Optional['SQLiteStorage'] = None):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index, is_bootstrap_node)
|
||||
split_buckets_under_index)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -70,6 +70,13 @@ class Node:
|
||||
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
|
||||
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
|
||||
# populate/split the buckets further
|
||||
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
|
||||
if buckets_with_contacts <= 3:
|
||||
for i in range(buckets_with_contacts):
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
|
||||
if self.protocol.routing_table.get_peers():
|
||||
# if we have node ids to look up, perform the iterative search until we have k results
|
||||
@@ -196,13 +203,15 @@ class Node:
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_results: int = constants.K) -> IterativeNodeFinder:
|
||||
shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
|
||||
return IterativeNodeFinder(self.loop, self.protocol, key, max_results, shortlist)
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
|
||||
return IterativeValueFinder(self.loop, self.protocol, key, max_results, shortlist)
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
|
||||
+3
-7
@@ -100,9 +100,6 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys())
|
||||
|
||||
def get_node_id_for_endpoint(self, address, port):
|
||||
return self._node_id_mapping.get((address, port))
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
@@ -153,10 +150,9 @@ class PeerManager:
|
||||
def peer_is_good(self, peer: 'KademliaPeer'):
|
||||
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
|
||||
|
||||
|
||||
def decode_tcp_peer_from_compact_address(compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
|
||||
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
|
||||
@@ -8,10 +8,11 @@ from typing import TYPE_CHECKING
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.peer import make_kademlia_peer, decode_tcp_peer_from_compact_address
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.serialization.datagram import PAGE_KEY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
@@ -26,15 +27,6 @@ class FindResponse:
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_close_kademlia_peers(self, peer_info) -> typing.Generator[typing.Iterator['KademliaPeer'], None, None]:
|
||||
for contact_triple in self.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
try:
|
||||
yield make_kademlia_peer(node_id, address, udp_port)
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer_info.address,
|
||||
peer_info.udp_port, address, udp_port)
|
||||
|
||||
|
||||
class FindNodeResponse(FindResponse):
|
||||
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
|
||||
@@ -65,19 +57,37 @@ class FindValueResponse(FindResponse):
|
||||
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
|
||||
|
||||
|
||||
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
|
||||
|
||||
:param routing_table: a TreeRoutingTable
|
||||
:param key: a 48 byte hash
|
||||
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
|
||||
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
|
||||
"""
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
return shortlist or routing_table.find_close_peers(key)
|
||||
|
||||
|
||||
class IterativeFinder(AsyncIterator):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop,
|
||||
protocol: 'KademliaProtocol', key: bytes,
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.loop = loop
|
||||
self.peer_manager = protocol.peer_manager
|
||||
self.peer_manager = peer_manager
|
||||
self.routing_table = routing_table
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.max_results = max(constants.K, max_results)
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.active: typing.Dict['KademliaPeer', int] = OrderedDict() # peer: distance, sorted
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
@@ -89,7 +99,7 @@ class IterativeFinder(AsyncIterator):
|
||||
self.iteration_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
for peer in shortlist:
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer, force=True)
|
||||
else:
|
||||
@@ -134,8 +144,13 @@ class IterativeFinder(AsyncIterator):
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
self._add_active(peer)
|
||||
for new_peer in response.get_close_kademlia_peers(peer):
|
||||
self._add_active(new_peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
try:
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
self._log_state(reason="check result")
|
||||
|
||||
@@ -183,6 +198,8 @@ class IterativeFinder(AsyncIterator):
|
||||
if index > (constants.K + len(self.running_probes)):
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
@@ -260,11 +277,13 @@ class IterativeFinder(AsyncIterator):
|
||||
type(self).__name__, id(self), self.key.hex()[:8])
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop,
|
||||
protocol: 'KademliaProtocol', key: bytes,
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, protocol, key, max_results, shortlist)
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
@@ -300,11 +319,13 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop,
|
||||
protocol: 'KademliaProtocol', key: bytes,
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, protocol, key, max_results, shortlist)
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
|
||||
@@ -323,7 +344,7 @@ class IterativeValueFinder(IterativeFinder):
|
||||
decoded_peers = set()
|
||||
for compact_addr in parsed.found_compact_addresses:
|
||||
try:
|
||||
decoded_peers.add(decode_tcp_peer_from_compact_address(compact_addr))
|
||||
decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned invalid peer for blob",
|
||||
peer.address, peer.udp_port)
|
||||
@@ -345,7 +366,7 @@ class IterativeValueFinder(IterativeFinder):
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
blob_peers = [decode_tcp_peer_from_compact_address(compact_addr)
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
for blob_peer in blob_peers:
|
||||
|
||||
@@ -218,10 +218,6 @@ class PingQueue:
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
@property
|
||||
def busy(self):
|
||||
return self._running and (any(self._running_pings) or any(self._pending_contacts))
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = delay if delay is not None else self._default_delay
|
||||
now = self._loop.time()
|
||||
@@ -233,7 +229,7 @@ class PingQueue:
|
||||
async def ping_task():
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if not self._protocol.routing_table.get_peer(peer.node_id):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
@@ -298,7 +294,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_boostrap_node: bool = False):
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
@@ -313,8 +309,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(
|
||||
self.loop, self.peer_manager, self.node_id, split_buckets_under_index, is_bootstrap_node=is_boostrap_node)
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
@@ -361,10 +356,72 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
async def probe(some_peer: 'KademliaPeer'):
|
||||
rpc_peer = self.get_rpc_peer(some_peer)
|
||||
await rpc_peer.ping()
|
||||
return await self.routing_table.add_peer(peer, probe)
|
||||
if not peer.node_id:
|
||||
log.warning("Tried adding a peer with no node id!")
|
||||
return False
|
||||
for my_peer in self.routing_table.get_peers():
|
||||
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(my_peer)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
# Retry the insertion attempt
|
||||
result = await self._add_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
return result
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for my_peer in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
|
||||
if not last_replied or last_replied + 60 < self.loop.time():
|
||||
not_recently_replied.append(my_peer)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
to_replace = self.routing_table.buckets[bucket_index].peers[0]
|
||||
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
|
||||
if last_replied and last_replied + 60 > self.loop.time():
|
||||
return False
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
|
||||
try:
|
||||
to_replace_rpc = self.get_rpc_peer(to_replace)
|
||||
await to_replace_rpc.ping()
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
|
||||
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
|
||||
if to_replace in self.routing_table.buckets[bucket_index]:
|
||||
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self._add_peer(peer)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer'):
|
||||
if peer.node_id == self.node_id:
|
||||
@@ -382,6 +439,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
async with self._split_lock:
|
||||
peer = self._to_remove.pop()
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
while self._to_add:
|
||||
async with self._split_lock:
|
||||
await self._add_peer(self._to_add.pop())
|
||||
@@ -424,8 +482,9 @@ class KademliaProtocol(DatagramProtocol):
|
||||
# This is an RPC method request
|
||||
self.received_request_metric.labels(method=request_datagram.method).inc()
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
if not peer:
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
|
||||
@@ -28,8 +28,7 @@ class KBucket:
|
||||
namespace="dht_node", labelnames=("amount",)
|
||||
)
|
||||
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int,
|
||||
node_id: bytes, capacity: int = constants.K):
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
|
||||
"""
|
||||
@param range_min: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@@ -37,12 +36,12 @@ class KBucket:
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self._peer_manager = peer_manager
|
||||
self.last_accessed = 0
|
||||
self.range_min = range_min
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
self.capacity = capacity
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
@@ -69,7 +68,7 @@ class KBucket:
|
||||
self.peers.remove(local_peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < self.capacity:
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").inc()
|
||||
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
|
||||
@@ -77,11 +76,13 @@ class KBucket:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def get_peer(self, node_id: bytes) -> 'KademliaPeer':
|
||||
for peer in self.peers:
|
||||
if peer.node_id == node_id:
|
||||
return peer
|
||||
raise IndexError(node_id)
|
||||
|
||||
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
@@ -178,13 +179,6 @@ class TreeRoutingTable:
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
|
||||
BOOTSTRAP MODE: if set to True, we always add all peers. This is so a
|
||||
bootstrap node does not get a bias towards its own node id and replies are
|
||||
the best it can provide (joining peer knows its neighbors immediately).
|
||||
Over time, this will need to be optimized so we use the disk as holding
|
||||
everything in memory won't be feasible anymore.
|
||||
See: https://github.com/bittorrent/bootstrap-dht
|
||||
"""
|
||||
bucket_in_routing_table_metric = Gauge(
|
||||
"buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node",
|
||||
@@ -192,22 +186,21 @@ class TreeRoutingTable:
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False):
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id,
|
||||
capacity=1 << 32 if is_bootstrap_node else constants.K
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id
|
||||
)
|
||||
]
|
||||
|
||||
def get_peers(self) -> typing.List['KademliaPeer']:
|
||||
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
|
||||
|
||||
def _should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
return True
|
||||
@@ -232,32 +225,39 @@ class TreeRoutingTable:
|
||||
return []
|
||||
|
||||
def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
|
||||
return self.buckets[self._kbucket_index(contact_id)].get_peer(contact_id)
|
||||
"""
|
||||
@raise IndexError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
|
||||
|
||||
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
|
||||
bucket_index = start_index
|
||||
refresh_ids = []
|
||||
for offset, _ in enumerate(self.buckets[start_index:]):
|
||||
refresh_ids.append(self._midpoint_id_in_bucket_range(start_index + offset))
|
||||
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
|
||||
# populate/split the buckets further
|
||||
buckets_with_contacts = self.buckets_with_contacts()
|
||||
if buckets_with_contacts <= 3:
|
||||
for i in range(buckets_with_contacts):
|
||||
refresh_ids.append(self._random_id_in_bucket_range(i))
|
||||
refresh_ids.append(self._random_id_in_bucket_range(i))
|
||||
now = int(self._loop.time())
|
||||
for bucket in self.buckets[start_index:]:
|
||||
if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL:
|
||||
to_search = self.midpoint_id_in_bucket_range(bucket_index)
|
||||
refresh_ids.append(to_search)
|
||||
bucket_index += 1
|
||||
return refresh_ids
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
if not peer.node_id:
|
||||
return
|
||||
bucket_index = self._kbucket_index(peer.node_id)
|
||||
bucket_index = self.kbucket_index(peer.node_id)
|
||||
try:
|
||||
self.buckets[bucket_index].remove_peer(peer)
|
||||
self._join_buckets()
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
def _kbucket_index(self, key: bytes) -> int:
|
||||
def touch_kbucket(self, key: bytes) -> None:
|
||||
self.touch_kbucket_by_index(self.kbucket_index(key))
|
||||
|
||||
def touch_kbucket_by_index(self, bucket_index: int):
|
||||
self.buckets[bucket_index].last_accessed = int(self._loop.time())
|
||||
|
||||
def kbucket_index(self, key: bytes) -> int:
|
||||
i = 0
|
||||
for bucket in self.buckets:
|
||||
if bucket.key_in_range(key):
|
||||
@@ -266,19 +266,19 @@ class TreeRoutingTable:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def _random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def _midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def _split_bucket(self, old_bucket_index: int) -> None:
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
@@ -303,7 +303,7 @@ class TreeRoutingTable:
|
||||
old_bucket.remove_peer(contact)
|
||||
self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
|
||||
|
||||
def _join_buckets(self):
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
|
||||
@@ -326,7 +326,14 @@ class TreeRoutingTable:
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
|
||||
self.buckets.remove(bucket)
|
||||
self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
|
||||
return self._join_buckets()
|
||||
return self.join_buckets()
|
||||
|
||||
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
|
||||
for bucket in self.buckets:
|
||||
for contact in bucket.get_peers(sort_distance_to=False):
|
||||
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
|
||||
return True
|
||||
return False
|
||||
|
||||
def buckets_with_contacts(self) -> int:
|
||||
count = 0
|
||||
@@ -334,70 +341,3 @@ class TreeRoutingTable:
|
||||
if len(bucket) > 0:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
async def add_peer(self, peer: 'KademliaPeer', probe: typing.Callable[['KademliaPeer'], typing.Awaitable]):
|
||||
if not peer.node_id:
|
||||
log.warning("Tried adding a peer with no node id!")
|
||||
return False
|
||||
for my_peer in self.get_peers():
|
||||
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
|
||||
self.remove_peer(my_peer)
|
||||
self._join_buckets()
|
||||
bucket_index = self._kbucket_index(peer.node_id)
|
||||
if self.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self._should_split(bucket_index, peer.node_id):
|
||||
self._split_bucket(bucket_index)
|
||||
# Retry the insertion attempt
|
||||
result = await self.add_peer(peer, probe)
|
||||
self._join_buckets()
|
||||
return result
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
not_good_contacts = self.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for my_peer in not_good_contacts:
|
||||
last_replied = self._peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
|
||||
if not last_replied or last_replied + 60 < self._loop.time():
|
||||
not_recently_replied.append(my_peer)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
to_replace = self.buckets[bucket_index].peers[0]
|
||||
last_replied = self._peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
|
||||
if last_replied and last_replied + 60 > self._loop.time():
|
||||
return False
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
|
||||
try:
|
||||
await probe(to_replace)
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
|
||||
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
|
||||
if to_replace in self.buckets[bucket_index]:
|
||||
self.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self.add_peer(peer, probe)
|
||||
@@ -1,5 +1,5 @@
|
||||
from lbry.extras.cli import execute_command
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.cli import execute_command
|
||||
|
||||
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
|
||||
@@ -28,7 +28,11 @@ from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
from lbry.torrent.tracker import TrackerClient
|
||||
from lbry.torrent.session import TorrentSession
|
||||
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -293,7 +297,6 @@ class DHTComponent(Component):
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index,
|
||||
is_bootstrap_node=self.conf.is_bootstrap_node,
|
||||
storage=storage
|
||||
)
|
||||
self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
|
||||
@@ -357,6 +360,10 @@ class FileManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.file_manager = FileManager(
|
||||
@@ -365,8 +372,7 @@ class FileManagerComponent(Component):
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if self.component_manager.has_component(LIBTORRENT_COMPONENT):
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
@@ -495,8 +501,9 @@ class TorrentComponent(Component):
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
|
||||
+112
-98
@@ -9,7 +9,7 @@ import inspect
|
||||
import typing
|
||||
import random
|
||||
import tracemalloc
|
||||
import itertools
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlencode, quote
|
||||
from typing import Callable, Optional, List
|
||||
from binascii import hexlify, unhexlify
|
||||
@@ -53,7 +53,7 @@ from lbry.extras.daemon.undecorated import undecorated
|
||||
from lbry.extras.daemon.security import ensure_request_allowed
|
||||
from lbry.file_analysis import VideoFileAnalyzer
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.url import URL
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -195,6 +195,61 @@ def paginate_list(items: List, page: Optional[int], page_size: Optional[int]):
|
||||
}
|
||||
|
||||
|
||||
def fix_kwargs_for_hub(**kwargs):
|
||||
repeated_fields = {"media_type", "stream_type", "claim_type"}
|
||||
value_fields = {"tx_nout", "has_source", "is_signature_valid"}
|
||||
opcodes = {'=': 0, '<=': 1, '>=': 2, '<': 3, '>': 4}
|
||||
for key, value in list(kwargs.items()):
|
||||
if value in (None, [], False):
|
||||
kwargs.pop(key)
|
||||
continue
|
||||
if key in REPLACEMENTS:
|
||||
kwargs[REPLACEMENTS[key]] = kwargs.pop(key)
|
||||
key = REPLACEMENTS[key]
|
||||
|
||||
if key == "normalized_name":
|
||||
kwargs[key] = normalize_name(value)
|
||||
if key == "limit_claims_per_channel":
|
||||
value = kwargs.pop("limit_claims_per_channel") or 0
|
||||
if value > 0:
|
||||
kwargs["limit_claims_per_channel"] = value
|
||||
elif key == "invalid_channel_signature":
|
||||
kwargs["is_signature_valid"] = {"value": not kwargs.pop("invalid_channel_signature")}
|
||||
elif key == "has_no_source":
|
||||
kwargs["has_source"] = {"value": not kwargs.pop("has_no_source")}
|
||||
elif key in value_fields:
|
||||
kwargs[key] = {"value": value} if not isinstance(value, dict) else value
|
||||
elif key in repeated_fields and isinstance(value, str):
|
||||
kwargs[key] = [value]
|
||||
elif key in ("claim_id", "channel_id"):
|
||||
kwargs[key] = {"invert": False, "value": [kwargs[key]]}
|
||||
elif key in ("claim_ids", "channel_ids"):
|
||||
kwargs[key[:-1]] = {"invert": False, "value": kwargs.pop(key)}
|
||||
elif key == "not_channel_ids":
|
||||
kwargs["channel_id"] = {"invert": True, "value": kwargs.pop("not_channel_ids")}
|
||||
elif key in MY_RANGE_FIELDS:
|
||||
constraints = []
|
||||
for val in value if isinstance(value, list) else [value]:
|
||||
operator = '='
|
||||
if isinstance(val, str) and val[0] in opcodes:
|
||||
operator_length = 2 if val[:2] in opcodes else 1
|
||||
operator, val = val[:operator_length], val[operator_length:]
|
||||
val = [int(val if key != 'fee_amount' else Decimal(val)*1000)]
|
||||
constraints.append({"op": opcodes[operator], "value": val})
|
||||
kwargs[key] = constraints
|
||||
elif key == 'order_by': # TODO: remove this after removing support for old trending args from the api
|
||||
value = value if isinstance(value, list) else [value]
|
||||
new_value = []
|
||||
for new_v in value:
|
||||
migrated = new_v if new_v not in (
|
||||
'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
|
||||
) else 'trending_score'
|
||||
if migrated not in new_value:
|
||||
new_value.append(migrated)
|
||||
kwargs[key] = new_value
|
||||
return kwargs
|
||||
|
||||
|
||||
DHT_HAS_CONTACTS = "dht_has_contacts"
|
||||
|
||||
|
||||
@@ -1327,66 +1382,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
c.wallets += [wallet_id]
|
||||
return wallet
|
||||
|
||||
@requires("wallet")
|
||||
async def jsonrpc_wallet_export(self, password=None, wallet_id=None):
|
||||
"""
|
||||
Export wallet data
|
||||
|
||||
Wallet must be unlocked to perform this operation.
|
||||
|
||||
Usage:
|
||||
wallet_export [--password=<password>] [--wallet_id=<wallet_id>]
|
||||
|
||||
Options:
|
||||
--password=<password> : (str) password to encrypt outgoing data
|
||||
--wallet_id=<wallet_id> : (str) wallet being exported
|
||||
|
||||
Returns:
|
||||
(str) data
|
||||
|
||||
"""
|
||||
assert password is not None, "passwordless use is not implemented yet"
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
encrypted = wallet.pack(password)
|
||||
return encrypted.decode()
|
||||
|
||||
@requires("wallet")
|
||||
async def jsonrpc_wallet_import(self, data, password=None, wallet_id=None, blocking=False):
|
||||
"""
|
||||
Import wallet data and merge accounts, preferences.
|
||||
|
||||
Wallet must be unlocked to perform this operation.
|
||||
|
||||
Usage:
|
||||
wallet_import (<data> | --data=<data>) [<password> | --password=<password>]
|
||||
[--wallet_id=<wallet_id>] [--blocking]
|
||||
|
||||
Options:
|
||||
--data=<data> : (str) incoming wallet data
|
||||
--password=<password> : (str) password to decrypt incoming data
|
||||
--wallet_id=<wallet_id> : (str) wallet being merged into
|
||||
--blocking : (bool) wait until any new accounts have merged
|
||||
|
||||
Returns:
|
||||
(str) data
|
||||
"""
|
||||
assert not data.strip().startswith("{"), "unencrypted wallet import is not implemented yet"
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data)
|
||||
for new_account in itertools.chain(added_accounts, merged_accounts):
|
||||
await new_account.maybe_migrate_certificates()
|
||||
if added_accounts and self.ledger.network.is_connected:
|
||||
if blocking:
|
||||
await asyncio.wait([
|
||||
a.ledger.subscribe_account(a) for a in added_accounts
|
||||
])
|
||||
else:
|
||||
for new_account in added_accounts:
|
||||
asyncio.create_task(self.ledger.subscribe_account(new_account))
|
||||
wallet.save()
|
||||
encrypted = wallet.pack(password)
|
||||
return encrypted.decode()
|
||||
|
||||
@requires("wallet")
|
||||
async def jsonrpc_wallet_add(self, wallet_id):
|
||||
"""
|
||||
@@ -1965,23 +1960,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return hexlify(wallet.hash).decode()
|
||||
|
||||
@requires("wallet")
|
||||
async def jsonrpc_sync_apply(self, password, data=None, wallet_id=None, blocking=False):
|
||||
async def jsonrpc_sync_apply(self, password, data=None, wallet_id=None, no_password_change=False, blocking=False):
|
||||
"""
|
||||
Apply incoming synchronization data, if provided, and return a sync hash and update wallet data.
|
||||
|
||||
Wallet must be unlocked to perform this operation.
|
||||
|
||||
If "encrypt-on-disk" preference is True and supplied password is different from local password,
|
||||
or there is no local password (because local wallet was not encrypted), then the supplied password
|
||||
will be used for local encryption (overwriting previous local encryption password).
|
||||
If "encrypt-on-disk" preference is True, then in all cases the wallet will be updated to use the
|
||||
supplied password for local encryption unless --only_export is passed.
|
||||
|
||||
Usage:
|
||||
sync_apply <password> [--data=<data>] [--wallet_id=<wallet_id>] [--blocking]
|
||||
sync_apply <password> [--data=<data>] [--wallet_id=<wallet_id>] [--no_password_change] [--blocking]
|
||||
|
||||
Options:
|
||||
--password=<password> : (str) password to decrypt incoming and encrypt outgoing data
|
||||
--data=<data> : (str) incoming sync data, if any
|
||||
--wallet_id=<wallet_id> : (str) wallet being sync'ed
|
||||
--no_password_change : (bool) do not update existing wallet encryption password
|
||||
--blocking : (bool) wait until any new accounts have sync'ed
|
||||
|
||||
Returns:
|
||||
@@ -1991,8 +1986,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
wallet_changed = False
|
||||
if data is not None:
|
||||
added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data)
|
||||
for new_account in itertools.chain(added_accounts, merged_accounts):
|
||||
added_accounts = wallet.merge(self.wallet_manager, password, data)
|
||||
for new_account in added_accounts:
|
||||
await new_account.maybe_migrate_certificates()
|
||||
if added_accounts and self.ledger.network.is_connected:
|
||||
if blocking:
|
||||
@@ -2003,7 +1998,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
for new_account in added_accounts:
|
||||
asyncio.create_task(self.ledger.subscribe_account(new_account))
|
||||
wallet_changed = True
|
||||
if wallet.preferences.get(ENCRYPT_ON_DISK, False) and password != wallet.encryption_password:
|
||||
if (
|
||||
wallet.preferences.get(ENCRYPT_ON_DISK, False) and
|
||||
password != wallet.encryption_password and
|
||||
not no_password_change
|
||||
):
|
||||
wallet.encryption_password = password
|
||||
wallet_changed = True
|
||||
if wallet_changed:
|
||||
@@ -2617,27 +2616,42 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
|
||||
kwargs.pop("claim_ids")
|
||||
if {'claim_id', 'claim_ids'}.issubset(kwargs):
|
||||
raise ConflictingInputValueError('claim_id', 'claim_ids')
|
||||
if kwargs.pop('valid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 1
|
||||
if kwargs.pop('invalid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 0
|
||||
if 'has_no_source' in kwargs:
|
||||
kwargs['has_source'] = not kwargs.pop('has_no_source')
|
||||
if 'order_by' in kwargs: # TODO: remove this after removing support for old trending args from the api
|
||||
value = kwargs.pop('order_by')
|
||||
value = value if isinstance(value, list) else [value]
|
||||
new_value = []
|
||||
for new_v in value:
|
||||
migrated = new_v if new_v not in (
|
||||
'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
|
||||
) else 'trending_score'
|
||||
if migrated not in new_value:
|
||||
new_value.append(migrated)
|
||||
kwargs['order_by'] = new_value
|
||||
if self.ledger.config.get('use_go_hub'):
|
||||
host = self.ledger.network.client.server[0]
|
||||
port = "50051"
|
||||
kwargs['new_sdk_server'] = f"{host}:{port}"
|
||||
if kwargs.get("channel"):
|
||||
channel = kwargs.pop("channel")
|
||||
channel_obj = (await self.jsonrpc_resolve(channel))[channel]
|
||||
if isinstance(channel_obj, dict):
|
||||
# This happens when the channel doesn't exist
|
||||
kwargs["channel_id"] = ""
|
||||
else:
|
||||
kwargs["channel_id"] = channel_obj.claim_id
|
||||
kwargs = fix_kwargs_for_hub(**kwargs)
|
||||
else:
|
||||
# Don't do this if using the hub server, it screws everything up
|
||||
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
|
||||
kwargs.pop("claim_ids")
|
||||
if {'claim_id', 'claim_ids'}.issubset(kwargs):
|
||||
raise ConflictingInputValueError('claim_id', 'claim_ids')
|
||||
if kwargs.pop('valid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 1
|
||||
if kwargs.pop('invalid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 0
|
||||
if 'has_no_source' in kwargs:
|
||||
kwargs['has_source'] = not kwargs.pop('has_no_source')
|
||||
if 'order_by' in kwargs: # TODO: remove this after removing support for old trending args from the api
|
||||
value = kwargs.pop('order_by')
|
||||
value = value if isinstance(value, list) else [value]
|
||||
new_value = []
|
||||
for new_v in value:
|
||||
migrated = new_v if new_v not in (
|
||||
'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
|
||||
) else 'trending_score'
|
||||
if migrated not in new_value:
|
||||
new_value.append(migrated)
|
||||
kwargs['order_by'] = new_value
|
||||
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
|
||||
wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
|
||||
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
|
||||
@@ -2773,7 +2787,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, txo, claim_address, claim, name
|
||||
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)]))
|
||||
self.component_manager.loop.create_task(self.analytics_manager.send_new_channel())
|
||||
else:
|
||||
@@ -2932,7 +2946,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)]))
|
||||
self.component_manager.loop.create_task(self.analytics_manager.send_new_channel())
|
||||
else:
|
||||
@@ -3531,7 +3545,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
async def save_claims():
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, claim, name
|
||||
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
if file_path is not None:
|
||||
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
|
||||
@@ -3768,7 +3782,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
async def save_claims():
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
if stream_hash:
|
||||
await self.storage.save_content_claim(stream_hash, new_txo.id)
|
||||
@@ -4338,7 +4352,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'nout': tx.position,
|
||||
'address': claim_address,
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(new_txo.amount)
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]})
|
||||
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('new_support'))
|
||||
else:
|
||||
@@ -5468,11 +5482,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _old_get_temp_claim_info(tx, txo, address, claim_dict, name):
|
||||
def _old_get_temp_claim_info(tx, txo, address, claim_dict, name, bid):
|
||||
return {
|
||||
"claim_id": txo.claim_id,
|
||||
"name": name,
|
||||
"amount": dewies_to_lbc(txo.amount),
|
||||
"amount": bid,
|
||||
"address": address,
|
||||
"txid": tx.id,
|
||||
"nout": txo.position,
|
||||
|
||||
@@ -196,9 +196,9 @@ FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
BittrexUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
# HotbitBTCFeed,
|
||||
# HotbitUSDFeed,
|
||||
# UPbitBTCFeed,
|
||||
HotbitBTCFeed,
|
||||
HotbitUSDFeed,
|
||||
UPbitBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -177,6 +177,19 @@ class Outputs:
|
||||
outputs.blocked, outputs.blocked_total
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_grpc(cls, outputs: OutputsMessage) -> 'Outputs':
|
||||
txs = set()
|
||||
for txo_message in chain(outputs.txos, outputs.extra_txos):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
continue
|
||||
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
|
||||
return cls(
|
||||
outputs.txos, outputs.extra_txos, txs,
|
||||
outputs.offset, outputs.total,
|
||||
outputs.blocked, outputs.blocked_total
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
|
||||
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
|
||||
|
||||
@@ -0,0 +1,960 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: hub.proto
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from . import result_pb2 as result__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='hub.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
|
||||
create_key=_descriptor._internal_create_key,
|
||||
serialized_pb=b'\n\thub.proto\x12\x02pb\x1a\x0cresult.proto\"\x0e\n\x0c\x45mptyMessage\".\n\rServerMessage\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\t\"N\n\x0cHelloMessage\x12\x0c\n\x04port\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\"\n\x07servers\x18\x03 \x03(\x0b\x32\x11.pb.ServerMessage\"0\n\x0fInvertibleField\x12\x0e\n\x06invert\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x03(\t\"\x1c\n\x0bStringValue\x12\r\n\x05value\x18\x01 \x01(\t\"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08\"\x1c\n\x0bUInt32Value\x12\r\n\x05value\x18\x01 \x01(\r\"j\n\nRangeField\x12\x1d\n\x02op\x18\x01 \x01(\x0e\x32\x11.pb.RangeField.Op\x12\r\n\x05value\x18\x02 \x03(\x05\".\n\x02Op\x12\x06\n\x02\x45Q\x10\x00\x12\x07\n\x03LTE\x10\x01\x12\x07\n\x03GTE\x10\x02\x12\x06\n\x02LT\x10\x03\x12\x06\n\x02GT\x10\x04\"\x8e\x0c\n\rSearchRequest\x12%\n\x08\x63laim_id\x18\x01 \x01(\x0b\x32\x13.pb.InvertibleField\x12\'\n\nchannel_id\x18\x02 \x01(\x0b\x32\x13.pb.InvertibleField\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x10\n\x08order_by\x18\x05 \x03(\t\x12\x0e\n\x06offset\x18\x06 \x01(\r\x12\x16\n\x0eis_controlling\x18\x07 \x01(\x08\x12\x1d\n\x15last_take_over_height\x18\x08 \x01(\t\x12\x12\n\nclaim_name\x18\t \x01(\t\x12\x17\n\x0fnormalized_name\x18\n \x01(\t\x12#\n\x0btx_position\x18\x0b \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06\x61mount\x18\x0c \x03(\x0b\x32\x0e.pb.RangeField\x12!\n\ttimestamp\x18\r \x03(\x0b\x32\x0e.pb.RangeField\x12*\n\x12\x63reation_timestamp\x18\x0e \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06height\x18\x0f \x03(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0f\x63reation_height\x18\x10 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x61\x63tivation_height\x18\x11 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x65xpiration_height\x18\x12 \x03(\x0b\x32\x0e.pb.RangeField\x12$\n\x0crelease_time\x18\x13 \x03(\x0b\x32\x0e.pb.RangeField\x12\x11\n\tshort_url\x18\x14 \x01(\t\x12\x15\n\rcanonical_url\x18\x15 \x01(\t\x12\r\n\x05title\x18\x16 \x01(\t\x12\x0e\n\x06\x61uthor\x18\x17 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x18 \x01(\t\x12\x12\n\nclaim_type\x18\x19 \x03(\t\x12$\n\x0crepost_count\x18\x1a \x03(\x0b\x32\x0e.pb.RangeField\x12\x13\n\x0bstream_type\x18\x1b \x03(\t\x12\x12\n\nmedia_type\x18\x1c \x03(\t\x12\"\n\nfee_amount\x18\x1d \x03(\x0b\x32\x0e.pb.RangeField\x12\x14\n\x0c\x66\x65\x65_currency\x18\x1e \x01(\t\x12 \n\x08\x64uration\x18\x1f \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11reposted_claim_id\x18 \x01(\t\x12#\n\x0b\x63\x65nsor_type\x18! \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11\x63laims_in_channel\x18\" \x01(\t\x12)\n\x12is_signature_valid\x18$ \x01(\x0b\x32\r.pb.BoolValue\x12(\n\x10\x65\x66\x66\x65\x63tive_amount\x18% \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0esupport_amount\x18& \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_score\x18\' \x03(\x0b\x32\x0e.pb.RangeField\x12\r\n\x05tx_id\x18+ \x01(\t\x12 \n\x07tx_nout\x18, \x01(\x0b\x32\x0f.pb.UInt32Value\x12\x11\n\tsignature\x18- \x01(\t\x12\x18\n\x10signature_digest\x18. \x01(\t\x12\x18\n\x10public_key_bytes\x18/ \x01(\t\x12\x15\n\rpublic_key_id\x18\x30 \x01(\t\x12\x10\n\x08\x61ny_tags\x18\x31 \x03(\t\x12\x10\n\x08\x61ll_tags\x18\x32 \x03(\t\x12\x10\n\x08not_tags\x18\x33 \x03(\t\x12\x1d\n\x15has_channel_signature\x18\x34 \x01(\x08\x12!\n\nhas_source\x18\x35 \x01(\x0b\x32\r.pb.BoolValue\x12 \n\x18limit_claims_per_channel\x18\x36 \x01(\x05\x12\x15\n\rany_languages\x18\x37 \x03(\t\x12\x15\n\rall_languages\x18\x38 \x03(\t\x12\x19\n\x11remove_duplicates\x18\x39 \x01(\x08\x12\x11\n\tno_totals\x18: \x01(\x08\x12\x0f\n\x07sd_hash\x18; \x01(\t2\x88\x03\n\x03Hub\x12*\n\x06Search\x12\x11.pb.SearchRequest\x1a\x0b.pb.Outputs\"\x00\x12+\n\x04Ping\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12-\n\x05Hello\x12\x10.pb.HelloMessage\x1a\x10.pb.HelloMessage\"\x00\x12/\n\x07\x41\x64\x64Peer\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12\x35\n\rPeerSubscribe\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12.\n\x07Version\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12/\n\x08\x46\x65\x61tures\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12\x30\n\tBroadcast\x12\x10.pb.EmptyMessage\x1a\x0f.pb.UInt32Value\"\x00\x42&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
|
||||
,
|
||||
dependencies=[result__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_RANGEFIELD_OP = _descriptor.EnumDescriptor(
|
||||
name='Op',
|
||||
full_name='pb.RangeField.Op',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='EQ', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LTE', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='GTE', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LT', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='GT', index=4, number=4,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=373,
|
||||
serialized_end=419,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_RANGEFIELD_OP)
|
||||
|
||||
|
||||
_EMPTYMESSAGE = _descriptor.Descriptor(
|
||||
name='EmptyMessage',
|
||||
full_name='pb.EmptyMessage',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=31,
|
||||
serialized_end=45,
|
||||
)
|
||||
|
||||
|
||||
_SERVERMESSAGE = _descriptor.Descriptor(
|
||||
name='ServerMessage',
|
||||
full_name='pb.ServerMessage',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='pb.ServerMessage.address', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='port', full_name='pb.ServerMessage.port', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=47,
|
||||
serialized_end=93,
|
||||
)
|
||||
|
||||
|
||||
_HELLOMESSAGE = _descriptor.Descriptor(
|
||||
name='HelloMessage',
|
||||
full_name='pb.HelloMessage',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='port', full_name='pb.HelloMessage.port', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='host', full_name='pb.HelloMessage.host', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='servers', full_name='pb.HelloMessage.servers', index=2,
|
||||
number=3, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=95,
|
||||
serialized_end=173,
|
||||
)
|
||||
|
||||
|
||||
_INVERTIBLEFIELD = _descriptor.Descriptor(
|
||||
name='InvertibleField',
|
||||
full_name='pb.InvertibleField',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='invert', full_name='pb.InvertibleField.invert', index=0,
|
||||
number=1, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.InvertibleField.value', index=1,
|
||||
number=2, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=175,
|
||||
serialized_end=223,
|
||||
)
|
||||
|
||||
|
||||
_STRINGVALUE = _descriptor.Descriptor(
|
||||
name='StringValue',
|
||||
full_name='pb.StringValue',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.StringValue.value', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=225,
|
||||
serialized_end=253,
|
||||
)
|
||||
|
||||
|
||||
_BOOLVALUE = _descriptor.Descriptor(
|
||||
name='BoolValue',
|
||||
full_name='pb.BoolValue',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.BoolValue.value', index=0,
|
||||
number=1, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=255,
|
||||
serialized_end=281,
|
||||
)
|
||||
|
||||
|
||||
_UINT32VALUE = _descriptor.Descriptor(
|
||||
name='UInt32Value',
|
||||
full_name='pb.UInt32Value',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.UInt32Value.value', index=0,
|
||||
number=1, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=283,
|
||||
serialized_end=311,
|
||||
)
|
||||
|
||||
|
||||
_RANGEFIELD = _descriptor.Descriptor(
|
||||
name='RangeField',
|
||||
full_name='pb.RangeField',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='op', full_name='pb.RangeField.op', index=0,
|
||||
number=1, type=14, cpp_type=8, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.RangeField.value', index=1,
|
||||
number=2, type=5, cpp_type=1, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_RANGEFIELD_OP,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=313,
|
||||
serialized_end=419,
|
||||
)
|
||||
|
||||
|
||||
_SEARCHREQUEST = _descriptor.Descriptor(
|
||||
name='SearchRequest',
|
||||
full_name='pb.SearchRequest',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_id', full_name='pb.SearchRequest.claim_id', index=0,
|
||||
number=1, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel_id', full_name='pb.SearchRequest.channel_id', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='text', full_name='pb.SearchRequest.text', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='limit', full_name='pb.SearchRequest.limit', index=3,
|
||||
number=4, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='order_by', full_name='pb.SearchRequest.order_by', index=4,
|
||||
number=5, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='offset', full_name='pb.SearchRequest.offset', index=5,
|
||||
number=6, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_controlling', full_name='pb.SearchRequest.is_controlling', index=6,
|
||||
number=7, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='last_take_over_height', full_name='pb.SearchRequest.last_take_over_height', index=7,
|
||||
number=8, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_name', full_name='pb.SearchRequest.claim_name', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='normalized_name', full_name='pb.SearchRequest.normalized_name', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_position', full_name='pb.SearchRequest.tx_position', index=10,
|
||||
number=11, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='pb.SearchRequest.amount', index=11,
|
||||
number=12, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='timestamp', full_name='pb.SearchRequest.timestamp', index=12,
|
||||
number=13, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_timestamp', full_name='pb.SearchRequest.creation_timestamp', index=13,
|
||||
number=14, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='height', full_name='pb.SearchRequest.height', index=14,
|
||||
number=15, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_height', full_name='pb.SearchRequest.creation_height', index=15,
|
||||
number=16, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='activation_height', full_name='pb.SearchRequest.activation_height', index=16,
|
||||
number=17, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='expiration_height', full_name='pb.SearchRequest.expiration_height', index=17,
|
||||
number=18, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='release_time', full_name='pb.SearchRequest.release_time', index=18,
|
||||
number=19, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='short_url', full_name='pb.SearchRequest.short_url', index=19,
|
||||
number=20, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='canonical_url', full_name='pb.SearchRequest.canonical_url', index=20,
|
||||
number=21, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='pb.SearchRequest.title', index=21,
|
||||
number=22, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='pb.SearchRequest.author', index=22,
|
||||
number=23, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='pb.SearchRequest.description', index=23,
|
||||
number=24, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_type', full_name='pb.SearchRequest.claim_type', index=24,
|
||||
number=25, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='repost_count', full_name='pb.SearchRequest.repost_count', index=25,
|
||||
number=26, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream_type', full_name='pb.SearchRequest.stream_type', index=26,
|
||||
number=27, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='media_type', full_name='pb.SearchRequest.media_type', index=27,
|
||||
number=28, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee_amount', full_name='pb.SearchRequest.fee_amount', index=28,
|
||||
number=29, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee_currency', full_name='pb.SearchRequest.fee_currency', index=29,
|
||||
number=30, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='duration', full_name='pb.SearchRequest.duration', index=30,
|
||||
number=31, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='reposted_claim_id', full_name='pb.SearchRequest.reposted_claim_id', index=31,
|
||||
number=32, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='censor_type', full_name='pb.SearchRequest.censor_type', index=32,
|
||||
number=33, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claims_in_channel', full_name='pb.SearchRequest.claims_in_channel', index=33,
|
||||
number=34, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_signature_valid', full_name='pb.SearchRequest.is_signature_valid', index=34,
|
||||
number=36, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='effective_amount', full_name='pb.SearchRequest.effective_amount', index=35,
|
||||
number=37, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='support_amount', full_name='pb.SearchRequest.support_amount', index=36,
|
||||
number=38, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_score', full_name='pb.SearchRequest.trending_score', index=37,
|
||||
number=39, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_id', full_name='pb.SearchRequest.tx_id', index=38,
|
||||
number=43, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_nout', full_name='pb.SearchRequest.tx_nout', index=39,
|
||||
number=44, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='pb.SearchRequest.signature', index=40,
|
||||
number=45, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature_digest', full_name='pb.SearchRequest.signature_digest', index=41,
|
||||
number=46, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key_bytes', full_name='pb.SearchRequest.public_key_bytes', index=42,
|
||||
number=47, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key_id', full_name='pb.SearchRequest.public_key_id', index=43,
|
||||
number=48, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='any_tags', full_name='pb.SearchRequest.any_tags', index=44,
|
||||
number=49, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='all_tags', full_name='pb.SearchRequest.all_tags', index=45,
|
||||
number=50, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='not_tags', full_name='pb.SearchRequest.not_tags', index=46,
|
||||
number=51, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='has_channel_signature', full_name='pb.SearchRequest.has_channel_signature', index=47,
|
||||
number=52, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='has_source', full_name='pb.SearchRequest.has_source', index=48,
|
||||
number=53, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='limit_claims_per_channel', full_name='pb.SearchRequest.limit_claims_per_channel', index=49,
|
||||
number=54, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='any_languages', full_name='pb.SearchRequest.any_languages', index=50,
|
||||
number=55, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='all_languages', full_name='pb.SearchRequest.all_languages', index=51,
|
||||
number=56, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='remove_duplicates', full_name='pb.SearchRequest.remove_duplicates', index=52,
|
||||
number=57, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='no_totals', full_name='pb.SearchRequest.no_totals', index=53,
|
||||
number=58, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sd_hash', full_name='pb.SearchRequest.sd_hash', index=54,
|
||||
number=59, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=422,
|
||||
serialized_end=1972,
|
||||
)
|
||||
|
||||
_HELLOMESSAGE.fields_by_name['servers'].message_type = _SERVERMESSAGE
|
||||
_RANGEFIELD.fields_by_name['op'].enum_type = _RANGEFIELD_OP
|
||||
_RANGEFIELD_OP.containing_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['claim_id'].message_type = _INVERTIBLEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['channel_id'].message_type = _INVERTIBLEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['tx_position'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['timestamp'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['creation_timestamp'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['creation_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['activation_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['expiration_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['release_time'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['repost_count'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['fee_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['duration'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['censor_type'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['is_signature_valid'].message_type = _BOOLVALUE
|
||||
_SEARCHREQUEST.fields_by_name['effective_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['support_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['trending_score'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['tx_nout'].message_type = _UINT32VALUE
|
||||
_SEARCHREQUEST.fields_by_name['has_source'].message_type = _BOOLVALUE
|
||||
DESCRIPTOR.message_types_by_name['EmptyMessage'] = _EMPTYMESSAGE
|
||||
DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE
|
||||
DESCRIPTOR.message_types_by_name['HelloMessage'] = _HELLOMESSAGE
|
||||
DESCRIPTOR.message_types_by_name['InvertibleField'] = _INVERTIBLEFIELD
|
||||
DESCRIPTOR.message_types_by_name['StringValue'] = _STRINGVALUE
|
||||
DESCRIPTOR.message_types_by_name['BoolValue'] = _BOOLVALUE
|
||||
DESCRIPTOR.message_types_by_name['UInt32Value'] = _UINT32VALUE
|
||||
DESCRIPTOR.message_types_by_name['RangeField'] = _RANGEFIELD
|
||||
DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
EmptyMessage = _reflection.GeneratedProtocolMessageType('EmptyMessage', (_message.Message,), {
|
||||
'DESCRIPTOR' : _EMPTYMESSAGE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.EmptyMessage)
|
||||
})
|
||||
_sym_db.RegisterMessage(EmptyMessage)
|
||||
|
||||
ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), {
|
||||
'DESCRIPTOR' : _SERVERMESSAGE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.ServerMessage)
|
||||
})
|
||||
_sym_db.RegisterMessage(ServerMessage)
|
||||
|
||||
HelloMessage = _reflection.GeneratedProtocolMessageType('HelloMessage', (_message.Message,), {
|
||||
'DESCRIPTOR' : _HELLOMESSAGE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.HelloMessage)
|
||||
})
|
||||
_sym_db.RegisterMessage(HelloMessage)
|
||||
|
||||
InvertibleField = _reflection.GeneratedProtocolMessageType('InvertibleField', (_message.Message,), {
|
||||
'DESCRIPTOR' : _INVERTIBLEFIELD,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.InvertibleField)
|
||||
})
|
||||
_sym_db.RegisterMessage(InvertibleField)
|
||||
|
||||
StringValue = _reflection.GeneratedProtocolMessageType('StringValue', (_message.Message,), {
|
||||
'DESCRIPTOR' : _STRINGVALUE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.StringValue)
|
||||
})
|
||||
_sym_db.RegisterMessage(StringValue)
|
||||
|
||||
BoolValue = _reflection.GeneratedProtocolMessageType('BoolValue', (_message.Message,), {
|
||||
'DESCRIPTOR' : _BOOLVALUE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.BoolValue)
|
||||
})
|
||||
_sym_db.RegisterMessage(BoolValue)
|
||||
|
||||
UInt32Value = _reflection.GeneratedProtocolMessageType('UInt32Value', (_message.Message,), {
|
||||
'DESCRIPTOR' : _UINT32VALUE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.UInt32Value)
|
||||
})
|
||||
_sym_db.RegisterMessage(UInt32Value)
|
||||
|
||||
RangeField = _reflection.GeneratedProtocolMessageType('RangeField', (_message.Message,), {
|
||||
'DESCRIPTOR' : _RANGEFIELD,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.RangeField)
|
||||
})
|
||||
_sym_db.RegisterMessage(RangeField)
|
||||
|
||||
SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
|
||||
'DESCRIPTOR' : _SEARCHREQUEST,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.SearchRequest)
|
||||
})
|
||||
_sym_db.RegisterMessage(SearchRequest)
|
||||
|
||||
|
||||
DESCRIPTOR._options = None
|
||||
|
||||
_HUB = _descriptor.ServiceDescriptor(
|
||||
name='Hub',
|
||||
full_name='pb.Hub',
|
||||
file=DESCRIPTOR,
|
||||
index=0,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
serialized_start=1975,
|
||||
serialized_end=2367,
|
||||
methods=[
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Search',
|
||||
full_name='pb.Hub.Search',
|
||||
index=0,
|
||||
containing_service=None,
|
||||
input_type=_SEARCHREQUEST,
|
||||
output_type=result__pb2._OUTPUTS,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Ping',
|
||||
full_name='pb.Hub.Ping',
|
||||
index=1,
|
||||
containing_service=None,
|
||||
input_type=_EMPTYMESSAGE,
|
||||
output_type=_STRINGVALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Hello',
|
||||
full_name='pb.Hub.Hello',
|
||||
index=2,
|
||||
containing_service=None,
|
||||
input_type=_HELLOMESSAGE,
|
||||
output_type=_HELLOMESSAGE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='AddPeer',
|
||||
full_name='pb.Hub.AddPeer',
|
||||
index=3,
|
||||
containing_service=None,
|
||||
input_type=_SERVERMESSAGE,
|
||||
output_type=_STRINGVALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='PeerSubscribe',
|
||||
full_name='pb.Hub.PeerSubscribe',
|
||||
index=4,
|
||||
containing_service=None,
|
||||
input_type=_SERVERMESSAGE,
|
||||
output_type=_STRINGVALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Version',
|
||||
full_name='pb.Hub.Version',
|
||||
index=5,
|
||||
containing_service=None,
|
||||
input_type=_EMPTYMESSAGE,
|
||||
output_type=_STRINGVALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Features',
|
||||
full_name='pb.Hub.Features',
|
||||
index=6,
|
||||
containing_service=None,
|
||||
input_type=_EMPTYMESSAGE,
|
||||
output_type=_STRINGVALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Broadcast',
|
||||
full_name='pb.Hub.Broadcast',
|
||||
index=7,
|
||||
containing_service=None,
|
||||
input_type=_EMPTYMESSAGE,
|
||||
output_type=_UINT32VALUE,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
])
|
||||
_sym_db.RegisterServiceDescriptor(_HUB)
|
||||
|
||||
DESCRIPTOR.services_by_name['Hub'] = _HUB
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -0,0 +1,298 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
import grpc
|
||||
|
||||
from . import hub_pb2 as hub__pb2
|
||||
from . import result_pb2 as result__pb2
|
||||
|
||||
|
||||
class HubStub(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
|
||||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.Search = channel.unary_unary(
|
||||
'/pb.Hub/Search',
|
||||
request_serializer=hub__pb2.SearchRequest.SerializeToString,
|
||||
response_deserializer=result__pb2.Outputs.FromString,
|
||||
)
|
||||
self.Ping = channel.unary_unary(
|
||||
'/pb.Hub/Ping',
|
||||
request_serializer=hub__pb2.EmptyMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.StringValue.FromString,
|
||||
)
|
||||
self.Hello = channel.unary_unary(
|
||||
'/pb.Hub/Hello',
|
||||
request_serializer=hub__pb2.HelloMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.HelloMessage.FromString,
|
||||
)
|
||||
self.AddPeer = channel.unary_unary(
|
||||
'/pb.Hub/AddPeer',
|
||||
request_serializer=hub__pb2.ServerMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.StringValue.FromString,
|
||||
)
|
||||
self.PeerSubscribe = channel.unary_unary(
|
||||
'/pb.Hub/PeerSubscribe',
|
||||
request_serializer=hub__pb2.ServerMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.StringValue.FromString,
|
||||
)
|
||||
self.Version = channel.unary_unary(
|
||||
'/pb.Hub/Version',
|
||||
request_serializer=hub__pb2.EmptyMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.StringValue.FromString,
|
||||
)
|
||||
self.Features = channel.unary_unary(
|
||||
'/pb.Hub/Features',
|
||||
request_serializer=hub__pb2.EmptyMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.StringValue.FromString,
|
||||
)
|
||||
self.Broadcast = channel.unary_unary(
|
||||
'/pb.Hub/Broadcast',
|
||||
request_serializer=hub__pb2.EmptyMessage.SerializeToString,
|
||||
response_deserializer=hub__pb2.UInt32Value.FromString,
|
||||
)
|
||||
|
||||
|
||||
class HubServicer(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def Search(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Ping(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Hello(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def AddPeer(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def PeerSubscribe(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Version(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Features(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Broadcast(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
|
||||
def add_HubServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'Search': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Search,
|
||||
request_deserializer=hub__pb2.SearchRequest.FromString,
|
||||
response_serializer=result__pb2.Outputs.SerializeToString,
|
||||
),
|
||||
'Ping': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Ping,
|
||||
request_deserializer=hub__pb2.EmptyMessage.FromString,
|
||||
response_serializer=hub__pb2.StringValue.SerializeToString,
|
||||
),
|
||||
'Hello': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Hello,
|
||||
request_deserializer=hub__pb2.HelloMessage.FromString,
|
||||
response_serializer=hub__pb2.HelloMessage.SerializeToString,
|
||||
),
|
||||
'AddPeer': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.AddPeer,
|
||||
request_deserializer=hub__pb2.ServerMessage.FromString,
|
||||
response_serializer=hub__pb2.StringValue.SerializeToString,
|
||||
),
|
||||
'PeerSubscribe': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.PeerSubscribe,
|
||||
request_deserializer=hub__pb2.ServerMessage.FromString,
|
||||
response_serializer=hub__pb2.StringValue.SerializeToString,
|
||||
),
|
||||
'Version': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Version,
|
||||
request_deserializer=hub__pb2.EmptyMessage.FromString,
|
||||
response_serializer=hub__pb2.StringValue.SerializeToString,
|
||||
),
|
||||
'Features': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Features,
|
||||
request_deserializer=hub__pb2.EmptyMessage.FromString,
|
||||
response_serializer=hub__pb2.StringValue.SerializeToString,
|
||||
),
|
||||
'Broadcast': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Broadcast,
|
||||
request_deserializer=hub__pb2.EmptyMessage.FromString,
|
||||
response_serializer=hub__pb2.UInt32Value.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
'pb.Hub', rpc_method_handlers)
|
||||
server.add_generic_rpc_handlers((generic_handler,))
|
||||
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class Hub(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
@staticmethod
|
||||
def Search(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Search',
|
||||
hub__pb2.SearchRequest.SerializeToString,
|
||||
result__pb2.Outputs.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def Ping(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Ping',
|
||||
hub__pb2.EmptyMessage.SerializeToString,
|
||||
hub__pb2.StringValue.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def Hello(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Hello',
|
||||
hub__pb2.HelloMessage.SerializeToString,
|
||||
hub__pb2.HelloMessage.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def AddPeer(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/AddPeer',
|
||||
hub__pb2.ServerMessage.SerializeToString,
|
||||
hub__pb2.StringValue.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def PeerSubscribe(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/PeerSubscribe',
|
||||
hub__pb2.ServerMessage.SerializeToString,
|
||||
hub__pb2.StringValue.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def Version(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Version',
|
||||
hub__pb2.EmptyMessage.SerializeToString,
|
||||
hub__pb2.StringValue.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def Features(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Features',
|
||||
hub__pb2.EmptyMessage.SerializeToString,
|
||||
hub__pb2.StringValue.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def Broadcast(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Broadcast',
|
||||
hub__pb2.EmptyMessage.SerializeToString,
|
||||
hub__pb2.UInt32Value.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
@@ -0,0 +1,4 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
import grpc
|
||||
|
||||
+10
-12
@@ -19,7 +19,7 @@ from lbry.conf import Config
|
||||
from lbry.wallet.util import satoshis_to_coins
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
from lbry.wallet.orchstr8 import Conductor
|
||||
from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode
|
||||
from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
@@ -237,6 +237,7 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.conductor: Optional[Conductor] = None
|
||||
self.blockchain: Optional[LBCWalletNode] = None
|
||||
self.hub: Optional[HubNode] = None
|
||||
self.wallet_node: Optional[WalletNode] = None
|
||||
self.manager: Optional[WalletManager] = None
|
||||
self.ledger: Optional[Ledger] = None
|
||||
@@ -253,7 +254,10 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
self.addCleanup(self.conductor.stop_spv)
|
||||
await self.conductor.start_wallet()
|
||||
self.addCleanup(self.conductor.stop_wallet)
|
||||
await self.conductor.start_hub()
|
||||
self.addCleanup(self.conductor.stop_hub)
|
||||
self.blockchain = self.conductor.lbcwallet_node
|
||||
self.hub = self.conductor.hub_node
|
||||
self.wallet_node = self.conductor.wallet_node
|
||||
self.manager = self.wallet_node.manager
|
||||
self.ledger = self.wallet_node.ledger
|
||||
@@ -308,11 +312,8 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
while True:
|
||||
await self.conductor.spv_node.server.synchronized.wait()
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
if self.conductor.spv_node.server.db.db_height < height:
|
||||
continue
|
||||
if self.conductor.spv_node.server._es_height < height:
|
||||
continue
|
||||
break
|
||||
if self.conductor.spv_node.server.db.db_height >= height:
|
||||
break
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
@@ -327,18 +328,15 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
height = self.blockchain.block_expected
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
await self.blockchain.generate(blocks)
|
||||
height = self.blockchain.block_expected
|
||||
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
|
||||
while True:
|
||||
await self.conductor.spv_node.server.synchronized.wait()
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
if self.conductor.spv_node.server.db.db_height < height:
|
||||
continue
|
||||
if self.conductor.spv_node.server._es_height < height:
|
||||
continue
|
||||
break
|
||||
if self.conductor.spv_node.server.db.db_height >= height:
|
||||
break
|
||||
|
||||
|
||||
class FakeExchangeRateManager(ExchangeRateManager):
|
||||
|
||||
+38
-2
@@ -10,13 +10,47 @@ from typing import Optional
|
||||
import libtorrent
|
||||
|
||||
|
||||
NOTIFICATION_MASKS = [
|
||||
"error",
|
||||
"peer",
|
||||
"port_mapping",
|
||||
"storage",
|
||||
"tracker",
|
||||
"debug",
|
||||
"status",
|
||||
"progress",
|
||||
"ip_block",
|
||||
"dht",
|
||||
"stats",
|
||||
"session_log",
|
||||
"torrent_log",
|
||||
"peer_log",
|
||||
"incoming_request",
|
||||
"dht_log",
|
||||
"dht_operation",
|
||||
"port_mapping_log",
|
||||
"picker_log",
|
||||
"file_progress",
|
||||
"piece_progress",
|
||||
"upload",
|
||||
"block_progress"
|
||||
]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||
)
|
||||
|
||||
|
||||
def get_notification_type(notification) -> str:
|
||||
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||
if (1 << i) & notification:
|
||||
return notification_type
|
||||
raise ValueError("unrecognized notification type")
|
||||
|
||||
|
||||
class TorrentHandle:
|
||||
def __init__(self, loop, executor, handle):
|
||||
self._loop = loop
|
||||
@@ -122,8 +156,10 @@ class TorrentSession:
|
||||
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||
settings = {
|
||||
'listen_interfaces': f"{interface}:{port}",
|
||||
'enable_natpmp': False,
|
||||
'enable_upnp': False
|
||||
'enable_outgoing_utp': True,
|
||||
'enable_incoming_utp': True,
|
||||
'enable_outgoing_tcp': False,
|
||||
'enable_incoming_tcp': False
|
||||
}
|
||||
self._session = await self._loop.run_in_executor(
|
||||
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||
|
||||
+20
-10
@@ -780,9 +780,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
include_is_my_output=False,
|
||||
include_sent_supports=False,
|
||||
include_sent_tips=False,
|
||||
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
|
||||
include_received_tips=False,
|
||||
hub_server=False) -> Tuple[List[Output], dict, int, int]:
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or '') # TODO: why is the server returning None?
|
||||
if hub_server:
|
||||
outputs = Outputs.from_grpc(encoded_outputs)
|
||||
else:
|
||||
outputs = Outputs.from_base64(encoded_outputs or '') # TODO: why is the server returning None?
|
||||
txs: List[Transaction] = []
|
||||
if len(outputs.txs) > 0:
|
||||
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
|
||||
@@ -858,10 +862,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
txo.received_tips = tips
|
||||
return txos, blocked, outputs.offset, outputs.total
|
||||
|
||||
async def resolve(self, accounts, urls, **kwargs):
|
||||
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
|
||||
txos = []
|
||||
urls_copy = list(urls)
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
if new_sdk_server:
|
||||
resolve = partial(self.network.new_resolve, new_sdk_server)
|
||||
else:
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:100], urls_copy[100:]
|
||||
txos.extend(
|
||||
@@ -886,14 +893,17 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return await self.network.sum_supports(new_sdk_server, **kwargs)
|
||||
|
||||
async def claim_search(
|
||||
self, accounts,
|
||||
include_purchase_receipt=False,
|
||||
include_is_my_output=False,
|
||||
**kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
|
||||
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||
if new_sdk_server:
|
||||
claim_search = partial(self.network.new_claim_search, new_sdk_server)
|
||||
else:
|
||||
claim_search = self.network.claim_search
|
||||
return await self._inflate_outputs(
|
||||
self.network.claim_search(**kwargs), accounts,
|
||||
claim_search(**kwargs), accounts,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output
|
||||
include_is_my_output=include_is_my_output,
|
||||
hub_server=new_sdk_server is not None
|
||||
)
|
||||
|
||||
# async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
|
||||
|
||||
@@ -3,6 +3,7 @@ import json
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from distutils.util import strtobool
|
||||
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
@@ -182,6 +183,7 @@ class WalletManager:
|
||||
}[config.blockchain_name]
|
||||
|
||||
ledger_config = {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'auto_connect': True,
|
||||
'explicit_servers': [],
|
||||
'hub_timeout': config.hub_timeout,
|
||||
@@ -236,6 +238,7 @@ class WalletManager:
|
||||
|
||||
async def reset(self):
|
||||
self.ledger.config = {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'auto_connect': True,
|
||||
'explicit_servers': [],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
|
||||
@@ -7,6 +7,9 @@ from time import perf_counter
|
||||
from collections import defaultdict
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
import grpc
|
||||
from lbry.schema.types.v2 import hub_pb2_grpc
|
||||
from lbry.schema.types.v2.hub_pb2 import SearchRequest
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
@@ -390,6 +393,7 @@ class Network:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
@@ -472,6 +476,21 @@ class Network:
|
||||
def claim_search(self, session_override=None, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
|
||||
|
||||
async def new_resolve(self, server, urls):
|
||||
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
async def new_claim_search(self, server, **kwargs):
|
||||
async with grpc.aio.insecure_channel(server) as channel:
|
||||
stub = hub_pb2_grpc.HubStub(channel)
|
||||
try:
|
||||
response = await stub.Search(SearchRequest(**kwargs))
|
||||
except grpc.aio.AioRpcError as error:
|
||||
raise RPCError(error.code(), error.details())
|
||||
return response
|
||||
|
||||
async def sum_supports(self, server, **kwargs):
|
||||
message = {"method": "support_sum", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
|
||||
@@ -1,2 +1,5 @@
|
||||
__hub_url__ = (
|
||||
"https://github.com/lbryio/herald/releases/download/v0.2022.01.21.1/hub"
|
||||
)
|
||||
from lbry.wallet.orchstr8.node import Conductor
|
||||
from lbry.wallet.orchstr8.service import ConductorService
|
||||
@@ -1,5 +1,6 @@
|
||||
# pylint: disable=import-error
|
||||
import os
|
||||
import signal
|
||||
import json
|
||||
import shutil
|
||||
import asyncio
|
||||
@@ -9,6 +10,7 @@ import logging
|
||||
import tempfile
|
||||
import subprocess
|
||||
import platform
|
||||
from distutils.util import strtobool
|
||||
|
||||
from binascii import hexlify
|
||||
from typing import Type, Optional
|
||||
@@ -19,6 +21,7 @@ from uuid import uuid4
|
||||
import lbry
|
||||
from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
|
||||
from lbry.conf import KnownHubsList, Config
|
||||
from lbry.wallet.orchstr8 import __hub_url__
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,10 +62,13 @@ class Conductor:
|
||||
self.wallet_node = WalletNode(
|
||||
self.manager_module, RegTestLedger, default_seed=seed
|
||||
)
|
||||
self.hub_node = HubNode(__hub_url__, "hub", self.spv_node)
|
||||
|
||||
self.lbcd_started = False
|
||||
self.lbcwallet_started = False
|
||||
self.spv_started = False
|
||||
self.wallet_started = False
|
||||
self.hub_started = False
|
||||
|
||||
self.log = log.getChild('conductor')
|
||||
|
||||
@@ -76,6 +82,17 @@ class Conductor:
|
||||
await self.lbcd_node.stop(cleanup)
|
||||
self.lbcd_started = False
|
||||
|
||||
async def start_hub(self):
|
||||
if not self.hub_started:
|
||||
await self.hub_node.start()
|
||||
await self.lbcwallet_node.running.wait()
|
||||
self.hub_started = True
|
||||
|
||||
async def stop_hub(self, cleanup=True):
|
||||
if self.hub_started:
|
||||
await self.hub_node.stop(cleanup)
|
||||
self.hub_started = False
|
||||
|
||||
async def start_spv(self):
|
||||
if not self.spv_started:
|
||||
await self.spv_node.start(self.lbcwallet_node)
|
||||
@@ -116,11 +133,13 @@ class Conductor:
|
||||
await self.start_lbcd()
|
||||
await self.start_lbcwallet()
|
||||
await self.start_spv()
|
||||
await self.start_hub()
|
||||
await self.start_wallet()
|
||||
|
||||
async def stop(self):
|
||||
all_the_stops = [
|
||||
self.stop_wallet,
|
||||
self.stop_hub,
|
||||
self.stop_spv,
|
||||
self.stop_lbcwallet,
|
||||
self.stop_lbcd
|
||||
@@ -165,6 +184,7 @@ class WalletNode:
|
||||
self.manager = self.manager_class.from_config({
|
||||
'ledgers': {
|
||||
self.ledger_class.get_id(): {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'api_port': self.port,
|
||||
'explicit_servers': [(spv_node.hostname, spv_node.port)],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
@@ -673,3 +693,139 @@ class LBCWalletNode:
|
||||
|
||||
def get_raw_transaction(self, txid):
|
||||
return self._cli_cmnd('getrawtransaction', txid, '1')
|
||||
|
||||
|
||||
class HubProcess(asyncio.SubprocessProtocol):
|
||||
def __init__(self, ready, stopped):
|
||||
self.ready = ready
|
||||
self.stopped = stopped
|
||||
self.log = log.getChild('hub')
|
||||
self.transport = None
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
self.stopped.clear()
|
||||
self.ready.set()
|
||||
if self.log:
|
||||
self.log.info(data.decode())
|
||||
if b'error' in data.lower():
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'listening on' in data:
|
||||
self.ready.set()
|
||||
str_lines = str(data.decode()).split("\n")
|
||||
for line in str_lines:
|
||||
if 'releaseTime' in line:
|
||||
print(line)
|
||||
|
||||
def process_exited(self):
|
||||
self.ready.clear()
|
||||
self.stopped.set()
|
||||
|
||||
async def stop(self):
|
||||
t = asyncio.create_task(self.stopped.wait())
|
||||
try:
|
||||
self.transport.send_signal(signal.SIGINT)
|
||||
await asyncio.wait_for(t, 3)
|
||||
# log.warning("stopped go hub")
|
||||
except asyncio.TimeoutError:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
self.transport.terminate()
|
||||
await self.stopped.wait()
|
||||
log.warning("terminated go hub")
|
||||
|
||||
|
||||
class HubNode:
|
||||
def __init__(self, url, daemon, spv_node):
|
||||
self.spv_node = spv_node
|
||||
self.latest_release_url = url
|
||||
self.project_dir = os.path.dirname(os.path.dirname(__file__))
|
||||
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.cli_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.log = log.getChild('hub')
|
||||
self.transport = None
|
||||
self.protocol = None
|
||||
self.hostname = 'localhost'
|
||||
self.rpcport = 50051 # avoid conflict with default rpc port
|
||||
self._stopped = asyncio.Event()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def stopped(self):
|
||||
return not self.running.is_set()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.cli_bin) and
|
||||
os.path.exists(self.daemon_bin)
|
||||
)
|
||||
|
||||
def download(self):
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir,
|
||||
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
self.log.info('Downloading: %s', self.latest_release_url)
|
||||
with urllib.request.urlopen(self.latest_release_url) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
self.log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
if downloaded_file.endswith('.zip'):
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.cli_bin, 0o755)
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
elif downloaded_file.endswith('.tar.gz'):
|
||||
with tarfile.open(downloaded_file) as tar:
|
||||
tar.extractall(self.bin_dir)
|
||||
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
return self.exists
|
||||
|
||||
def ensure(self):
|
||||
return self.exists or self.download()
|
||||
|
||||
async def start(self):
|
||||
assert self.ensure()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
command = [
|
||||
self.daemon_bin, 'serve', '--esindex', self.spv_node.index_name + 'claims', '--debug'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
self.protocol = HubProcess(self.running, self._stopped)
|
||||
try:
|
||||
self.transport, _ = await loop.subprocess_exec(
|
||||
lambda: self.protocol, *command
|
||||
)
|
||||
self.protocol.transport = self.transport
|
||||
except Exception as e:
|
||||
log.exception('failed to start go hub', exc_info=e)
|
||||
raise e
|
||||
await self.protocol.ready.wait()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
if self.protocol:
|
||||
await self.protocol.stop()
|
||||
except Exception as e:
|
||||
log.exception('failed to stop go hub', exc_info=e)
|
||||
raise e
|
||||
finally:
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
@@ -175,9 +175,9 @@ class Wallet:
|
||||
return json.loads(decompressed)
|
||||
|
||||
def merge(self, manager: 'WalletManager',
|
||||
password: str, data: str) -> (List['Account'], List['Account']):
|
||||
password: str, data: str) -> List['Account']:
|
||||
assert not self.is_locked, "Cannot sync apply on a locked wallet."
|
||||
added_accounts, merged_accounts = [], []
|
||||
added_accounts = []
|
||||
decrypted_data = self.unpack(password, data)
|
||||
self.preferences.merge(decrypted_data.get('preferences', {}))
|
||||
for account_dict in decrypted_data['accounts']:
|
||||
@@ -191,11 +191,10 @@ class Wallet:
|
||||
break
|
||||
if local_match is not None:
|
||||
local_match.merge(account_dict)
|
||||
merged_accounts.append(local_match)
|
||||
else:
|
||||
new_account = Account.from_dict(ledger, self, account_dict)
|
||||
added_accounts.append(new_account)
|
||||
return added_accounts, merged_accounts
|
||||
return added_accounts
|
||||
|
||||
@property
|
||||
def is_locked(self) -> bool:
|
||||
|
||||
@@ -1,520 +0,0 @@
|
||||
import sys
|
||||
import datetime
|
||||
import logging
|
||||
import asyncio
|
||||
import os.path
|
||||
import random
|
||||
import time
|
||||
import typing
|
||||
from dataclasses import dataclass, astuple, replace
|
||||
|
||||
from aiohttp import web
|
||||
from prometheus_client import Gauge, generate_latest as prom_generate_latest, Counter, Histogram
|
||||
|
||||
import lbry.dht.error
|
||||
from lbry.dht.constants import generate_id
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import make_kademlia_peer, PeerManager, decode_tcp_peer_from_compact_address
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.protocol.iterative_find import FindValueResponse, FindNodeResponse, FindResponse
|
||||
from lbry.extras.daemon.storage import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.utils import resolve_host
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SDHashSamples:
|
||||
def __init__(self, samples_file_path):
|
||||
with open(samples_file_path, "rb") as sample_file:
|
||||
self._samples = sample_file.read()
|
||||
assert len(self._samples) % 48 == 0
|
||||
self.size = len(self._samples) // 48
|
||||
|
||||
def read_samples(self, count=1):
|
||||
for _ in range(count):
|
||||
offset = 48 * random.randrange(0, self.size)
|
||||
yield self._samples[offset:offset + 48]
|
||||
|
||||
|
||||
class PeerStorage(SQLiteMixin):
|
||||
CREATE_TABLES_QUERY = """
|
||||
PRAGMA JOURNAL_MODE=WAL;
|
||||
CREATE TABLE IF NOT EXISTS peer (
|
||||
peer_id INTEGER NOT NULL,
|
||||
node_id VARCHAR(96),
|
||||
address VARCHAR,
|
||||
udp_port INTEGER,
|
||||
tcp_port INTEGER,
|
||||
first_online DATETIME,
|
||||
errors INTEGER,
|
||||
last_churn INTEGER,
|
||||
added_on DATETIME NOT NULL,
|
||||
last_check DATETIME,
|
||||
last_seen DATETIME,
|
||||
latency INTEGER,
|
||||
PRIMARY KEY (peer_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS connection (
|
||||
from_peer_id INTEGER NOT NULL,
|
||||
to_peer_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (from_peer_id, to_peer_id),
|
||||
FOREIGN KEY(from_peer_id) REFERENCES peer (peer_id),
|
||||
FOREIGN KEY(to_peer_id) REFERENCES peer (peer_id)
|
||||
);
|
||||
"""
|
||||
|
||||
async def open(self):
|
||||
await super().open()
|
||||
self.db.writer_connection.row_factory = dict_row_factory
|
||||
|
||||
async def all_peers(self):
|
||||
return [
|
||||
DHTPeer(**peer) for peer in await self.db.execute_fetchall(
|
||||
"select * from peer where latency > 0 or last_seen > datetime('now', '-1 hour')")
|
||||
]
|
||||
|
||||
async def save_peers(self, *peers):
|
||||
log.info("Saving graph nodes (peers) to DB")
|
||||
await self.db.executemany(
|
||||
"INSERT OR REPLACE INTO peer("
|
||||
"node_id, address, udp_port, tcp_port, first_online, errors, last_churn,"
|
||||
"added_on, last_check, last_seen, latency, peer_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
[astuple(peer) for peer in peers]
|
||||
)
|
||||
log.info("Finished saving graph nodes (peers) to DB")
|
||||
|
||||
async def save_connections(self, connections_map):
|
||||
log.info("Saving graph edges (connections) to DB")
|
||||
await self.db.executemany(
|
||||
"DELETE FROM connection WHERE from_peer_id = ?", [(key,) for key in connections_map])
|
||||
for from_peer_id in connections_map:
|
||||
await self.db.executemany(
|
||||
"INSERT INTO connection(from_peer_id, to_peer_id) VALUES(?,?)",
|
||||
[(from_peer_id, to_peer_id) for to_peer_id in connections_map[from_peer_id]])
|
||||
log.info("Finished saving graph edges (connections) to DB")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DHTPeer:
|
||||
node_id: str
|
||||
address: str
|
||||
udp_port: int
|
||||
tcp_port: int = None
|
||||
first_online: datetime.datetime = None
|
||||
errors: int = None
|
||||
last_churn: int = None
|
||||
added_on: datetime.datetime = None
|
||||
last_check: datetime.datetime = None
|
||||
last_seen: datetime.datetime = None
|
||||
latency: int = None
|
||||
peer_id: int = None
|
||||
|
||||
@classmethod
|
||||
def from_kad_peer(cls, peer, peer_id):
|
||||
node_id = peer.node_id.hex() if peer.node_id else None
|
||||
return DHTPeer(
|
||||
node_id=node_id, address=peer.address, udp_port=peer.udp_port, tcp_port=peer.tcp_port,
|
||||
peer_id=peer_id, added_on=datetime.datetime.utcnow())
|
||||
|
||||
def to_kad_peer(self):
|
||||
node_id = bytes.fromhex(self.node_id) if self.node_id else None
|
||||
return make_kademlia_peer(node_id, self.address, self.udp_port, self.tcp_port)
|
||||
|
||||
|
||||
def new_node(address="0.0.0.0", udp_port=0, node_id=None):
|
||||
node_id = node_id or generate_id()
|
||||
loop = asyncio.get_event_loop()
|
||||
return Node(loop, PeerManager(loop), node_id, udp_port, udp_port, 3333, address)
|
||||
|
||||
|
||||
class Crawler:
|
||||
unique_total_hosts_metric = Gauge(
|
||||
"unique_total_hosts", "Number of unique hosts seen in the last interval", namespace="dht_crawler_node",
|
||||
)
|
||||
reachable_hosts_metric = Gauge(
|
||||
"reachable_hosts", "Number of hosts that replied in the last interval", namespace="dht_crawler_node",
|
||||
)
|
||||
total_historic_hosts_metric = Gauge(
|
||||
"history_total_hosts", "Number of hosts seen since first run.", namespace="dht_crawler_node",
|
||||
)
|
||||
pending_check_hosts_metric = Gauge(
|
||||
"pending_hosts", "Number of hosts on queue to be checked.", namespace="dht_crawler_node",
|
||||
)
|
||||
hosts_with_errors_metric = Gauge(
|
||||
"error_hosts", "Number of hosts that raised errors during contact.", namespace="dht_crawler_node",
|
||||
)
|
||||
ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS = tuple(map(float, range(100))) + (
|
||||
500., 1000., 2000., float('inf')
|
||||
)
|
||||
connections_found_metric = Histogram(
|
||||
"connections_found", "Number of hosts returned by the last successful contact.", namespace="dht_crawler_node",
|
||||
buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
|
||||
)
|
||||
known_connections_found_metric = Histogram(
|
||||
"known_connections_found", "Number of already known hosts returned by last contact.",
|
||||
namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
|
||||
)
|
||||
reachable_connections_found_metric = Histogram(
|
||||
"reachable_connections_found", "Number of reachable known hosts returned by last contact.",
|
||||
namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
|
||||
)
|
||||
LATENCY_HISTOGRAM_BUCKETS = (
|
||||
0., 5., 10., 15., 30., 60., 120., 180., 240., 300., 600., 1200., 1800., 4000., 6000., float('inf')
|
||||
)
|
||||
host_latency_metric = Histogram(
|
||||
"host_latency", "Time spent on the last request, in milliseconds.", namespace="dht_crawler_node",
|
||||
buckets=LATENCY_HISTOGRAM_BUCKETS
|
||||
)
|
||||
probed_streams_metric = Counter(
|
||||
"probed_streams", "Amount of streams probed.", namespace="dht_crawler_node",
|
||||
)
|
||||
announced_streams_metric = Counter(
|
||||
"announced_streams", "Amount of streams where announcements were found.", namespace="dht_crawler_node",
|
||||
)
|
||||
working_streams_metric = Counter(
|
||||
"working_streams", "Amount of streams with reachable hosts.", namespace="dht_crawler_node",
|
||||
)
|
||||
|
||||
def __init__(self, db_path: str, sd_hash_samples: SDHashSamples):
|
||||
self.node = new_node()
|
||||
self.db = PeerStorage(db_path)
|
||||
self.sd_hashes = sd_hash_samples
|
||||
self._memory_peers = {}
|
||||
self._reachable_by_node_id = {}
|
||||
self._connections = {}
|
||||
|
||||
async def open(self):
|
||||
await self.db.open()
|
||||
self._memory_peers = {
|
||||
(peer.address, peer.udp_port): peer for peer in await self.db.all_peers()
|
||||
}
|
||||
self.refresh_reachable_set()
|
||||
|
||||
def refresh_reachable_set(self):
|
||||
self._reachable_by_node_id = {
|
||||
bytes.fromhex(peer.node_id): peer for peer in self._memory_peers.values() if (peer.latency or 0) > 0
|
||||
}
|
||||
|
||||
async def probe_files(self):
|
||||
if not self.sd_hashes:
|
||||
return
|
||||
while True:
|
||||
for sd_hash in self.sd_hashes.read_samples(10_000):
|
||||
self.refresh_reachable_set()
|
||||
distance = Distance(sd_hash)
|
||||
node_ids = list(self._reachable_by_node_id.keys())
|
||||
node_ids.sort(key=lambda node_id: distance(node_id))
|
||||
k_closest = [self._reachable_by_node_id[node_id] for node_id in node_ids[:8]]
|
||||
found = False
|
||||
working = False
|
||||
for response in asyncio.as_completed(
|
||||
[self.request_peers(peer.address, peer.udp_port, peer.node_id, sd_hash) for peer in k_closest]):
|
||||
response = await response
|
||||
if response and response.found:
|
||||
found = True
|
||||
blob_peers = []
|
||||
for compact_addr in response.found_compact_addresses:
|
||||
try:
|
||||
blob_peers.append(decode_tcp_peer_from_compact_address(compact_addr))
|
||||
except ValueError as e:
|
||||
log.error("Error decoding compact peers: %s", e)
|
||||
for blob_peer in blob_peers:
|
||||
response = await self.request_peers(blob_peer.address, blob_peer.tcp_port, blob_peer.node_id, sd_hash)
|
||||
if response:
|
||||
working = True
|
||||
log.info("Found responsive peer for %s: %s:%d(%d)",
|
||||
sd_hash.hex()[:8], blob_peer.address,
|
||||
blob_peer.udp_port or -1, blob_peer.tcp_port or -1)
|
||||
else:
|
||||
log.info("Found dead peer for %s: %s:%d(%d)",
|
||||
sd_hash.hex()[:8], blob_peer.address,
|
||||
blob_peer.udp_port or -1, blob_peer.tcp_port or -1)
|
||||
self.probed_streams_metric.inc()
|
||||
if found:
|
||||
self.announced_streams_metric.inc()
|
||||
if working:
|
||||
self.working_streams_metric.inc()
|
||||
log.info("Done querying stream %s for peers. Found: %s, working: %s", sd_hash.hex()[:8], found, working)
|
||||
await asyncio.sleep(.5)
|
||||
|
||||
@property
|
||||
def refresh_limit(self):
|
||||
return datetime.datetime.utcnow() - datetime.timedelta(hours=1)
|
||||
|
||||
@property
|
||||
def all_peers(self):
|
||||
return [
|
||||
peer for peer in self._memory_peers.values()
|
||||
if (peer.last_seen and peer.last_seen > self.refresh_limit) or (peer.latency or 0) > 0
|
||||
]
|
||||
|
||||
@property
|
||||
def active_peers_count(self):
|
||||
return len(self.all_peers)
|
||||
|
||||
@property
|
||||
def checked_peers_count(self):
|
||||
return len([peer for peer in self.all_peers if peer.last_check and peer.last_check > self.refresh_limit])
|
||||
|
||||
@property
|
||||
def unreachable_peers_count(self):
|
||||
return len([peer for peer in self.all_peers
|
||||
if peer.last_check and peer.last_check > self.refresh_limit and not peer.latency])
|
||||
|
||||
@property
|
||||
def peers_with_errors_count(self):
|
||||
return len([peer for peer in self.all_peers if (peer.errors or 0) > 0])
|
||||
|
||||
def get_peers_needing_check(self):
|
||||
to_check = [peer for peer in self.all_peers if peer.last_check is None or peer.last_check < self.refresh_limit]
|
||||
return to_check
|
||||
|
||||
def remove_expired_peers(self):
|
||||
for key, peer in list(self._memory_peers.items()):
|
||||
if (peer.latency or 0) < 1 and peer.last_seen < self.refresh_limit:
|
||||
del self._memory_peers[key]
|
||||
|
||||
def add_peers(self, *peers):
|
||||
for peer in peers:
|
||||
db_peer = self.get_from_peer(peer)
|
||||
if db_peer and db_peer.node_id is None and peer.node_id is not None:
|
||||
db_peer = replace(db_peer, node_id=peer.node_id.hex())
|
||||
elif not db_peer:
|
||||
db_peer = DHTPeer.from_kad_peer(peer, len(self._memory_peers) + 1)
|
||||
db_peer = replace(db_peer, last_seen=datetime.datetime.utcnow())
|
||||
self._memory_peers[(peer.address, peer.udp_port)] = db_peer
|
||||
|
||||
async def flush_to_db(self):
|
||||
await self.db.save_peers(*self._memory_peers.values())
|
||||
connections_to_save = self._connections
|
||||
self._connections = {}
|
||||
# await self.db.save_connections(connections_to_save) heavy call
|
||||
self.remove_expired_peers()
|
||||
|
||||
def get_from_peer(self, peer):
|
||||
return self._memory_peers.get((peer.address, peer.udp_port), None)
|
||||
|
||||
def set_latency(self, peer, latency=None):
|
||||
if latency:
|
||||
self.host_latency_metric.observe(latency / 1_000_000.0)
|
||||
db_peer = self.get_from_peer(peer)
|
||||
if not db_peer:
|
||||
return
|
||||
db_peer = replace(db_peer, latency=latency)
|
||||
if not db_peer.node_id and peer.node_id:
|
||||
db_peer = replace(db_peer, node_id=peer.node_id.hex())
|
||||
if db_peer.first_online and latency is None:
|
||||
db_peer = replace(db_peer, last_churn=(datetime.datetime.utcnow() - db_peer.first_online).seconds)
|
||||
elif latency is not None and db_peer.first_online is None:
|
||||
db_peer = replace(db_peer, first_online=datetime.datetime.utcnow())
|
||||
db_peer = replace(db_peer, last_check=datetime.datetime.utcnow())
|
||||
self._memory_peers[(db_peer.address, db_peer.udp_port)] = db_peer
|
||||
|
||||
def inc_errors(self, peer):
|
||||
db_peer = self.get_from_peer(peer)
|
||||
self._memory_peers[(peer.address, peer.node_id)] = replace(db_peer, errors=(db_peer.errors or 0) + 1)
|
||||
|
||||
def associate_peers(self, peer, other_peers):
|
||||
self._connections[self.get_from_peer(peer).peer_id] = [
|
||||
self.get_from_peer(other_peer).peer_id for other_peer in other_peers]
|
||||
|
||||
async def request_peers(self, host, port, node_id, key=None) -> typing.Optional[FindResponse]:
|
||||
key = key or node_id
|
||||
peer = make_kademlia_peer(key, await resolve_host(host, port, 'udp'), port)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
req_start = time.perf_counter_ns()
|
||||
if key == node_id:
|
||||
response = await self.node.protocol.get_rpc_peer(peer).find_node(key)
|
||||
response = FindNodeResponse(key, response)
|
||||
latency = time.perf_counter_ns() - req_start
|
||||
self.set_latency(peer, latency)
|
||||
else:
|
||||
response = await self.node.protocol.get_rpc_peer(peer).find_value(key)
|
||||
response = FindValueResponse(key, response)
|
||||
await asyncio.sleep(0.05)
|
||||
return response
|
||||
except asyncio.TimeoutError:
|
||||
if key == node_id:
|
||||
self.set_latency(peer, None)
|
||||
continue
|
||||
except lbry.dht.error.TransportNotConnected:
|
||||
log.info("Transport unavailable, waiting 1s to retry")
|
||||
await asyncio.sleep(1)
|
||||
except lbry.dht.error.RemoteException as e:
|
||||
log.info('Peer errored: %s:%d attempt #%d - %s',
|
||||
host, port, (attempt + 1), str(e))
|
||||
if key == node_id:
|
||||
self.inc_errors(peer)
|
||||
self.set_latency(peer, None)
|
||||
continue
|
||||
|
||||
async def crawl_routing_table(self, host, port, node_id=None):
|
||||
start = time.time()
|
||||
log.debug("querying %s:%d", host, port)
|
||||
address = await resolve_host(host, port, 'udp')
|
||||
key = node_id or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port)
|
||||
peer = make_kademlia_peer(key, address, port)
|
||||
self.add_peers(peer)
|
||||
if not key:
|
||||
latency = None
|
||||
for _ in range(3):
|
||||
try:
|
||||
ping_start = time.perf_counter_ns()
|
||||
await self.node.protocol.get_rpc_peer(peer).ping()
|
||||
await asyncio.sleep(0.05)
|
||||
key = key or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port)
|
||||
peer = make_kademlia_peer(key, address, port)
|
||||
latency = time.perf_counter_ns() - ping_start
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
except lbry.dht.error.RemoteException:
|
||||
self.inc_errors(peer)
|
||||
pass
|
||||
self.set_latency(peer, latency if peer.node_id else None)
|
||||
if not latency or not peer.node_id:
|
||||
if latency and not peer.node_id:
|
||||
log.warning("No node id from %s:%d", host, port)
|
||||
return set()
|
||||
distance = Distance(key)
|
||||
max_distance = int.from_bytes(bytes([0xff] * 48), 'big')
|
||||
peers = set()
|
||||
factor = 2048
|
||||
for i in range(1000):
|
||||
response = await self.request_peers(address, port, key)
|
||||
new_peers = list(response.get_close_kademlia_peers(peer)) if response else None
|
||||
if not new_peers:
|
||||
break
|
||||
new_peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
peers.update(new_peers)
|
||||
far_key = new_peers[-1].node_id
|
||||
if distance(far_key) <= distance(key):
|
||||
current_distance = distance(key)
|
||||
next_jump = current_distance + int(max_distance // factor) # jump closer
|
||||
factor /= 2
|
||||
if factor > 8 and next_jump < max_distance:
|
||||
key = int.from_bytes(peer.node_id, 'big') ^ next_jump
|
||||
if key.bit_length() > 384:
|
||||
break
|
||||
key = key.to_bytes(48, 'big')
|
||||
else:
|
||||
break
|
||||
else:
|
||||
key = far_key
|
||||
factor = 2048
|
||||
if peers:
|
||||
log.info("Done querying %s:%d in %.2f seconds: %d peers found over %d requests.",
|
||||
host, port, (time.time() - start), len(peers), i)
|
||||
if peers:
|
||||
self.connections_found_metric.observe(len(peers))
|
||||
known_peers = 0
|
||||
reachable_connections = 0
|
||||
for peer in peers:
|
||||
known_peer = self.get_from_peer(peer)
|
||||
known_peers += 1 if known_peer else 0
|
||||
reachable_connections += 1 if known_peer and (known_peer.latency or 0) > 0 else 0
|
||||
self.known_connections_found_metric.observe(known_peers)
|
||||
self.reachable_connections_found_metric.observe(reachable_connections)
|
||||
self.add_peers(*peers)
|
||||
self.associate_peers(peer, peers)
|
||||
return peers
|
||||
|
||||
async def process(self):
|
||||
to_process = {}
|
||||
|
||||
def submit(_peer):
|
||||
f = asyncio.ensure_future(
|
||||
self.crawl_routing_table(_peer.address, _peer.udp_port, bytes.fromhex(_peer.node_id)))
|
||||
to_process[_peer.peer_id] = f
|
||||
f.add_done_callback(lambda _: to_process.pop(_peer.peer_id))
|
||||
|
||||
to_check = self.get_peers_needing_check()
|
||||
last_flush = datetime.datetime.utcnow()
|
||||
while True:
|
||||
for peer in to_check[:200]:
|
||||
if peer.peer_id not in to_process:
|
||||
submit(peer)
|
||||
await asyncio.sleep(.05)
|
||||
await asyncio.sleep(0)
|
||||
self.unique_total_hosts_metric.set(self.checked_peers_count)
|
||||
self.reachable_hosts_metric.set(self.checked_peers_count - self.unreachable_peers_count)
|
||||
self.total_historic_hosts_metric.set(len(self._memory_peers))
|
||||
self.pending_check_hosts_metric.set(len(to_check))
|
||||
self.hosts_with_errors_metric.set(self.peers_with_errors_count)
|
||||
log.info("%d known, %d contacted recently, %d unreachable, %d error, %d processing, %d on queue",
|
||||
self.active_peers_count, self.checked_peers_count, self.unreachable_peers_count,
|
||||
self.peers_with_errors_count, len(to_process), len(to_check))
|
||||
if to_process:
|
||||
await asyncio.wait(to_process.values(), return_when=asyncio.FIRST_COMPLETED)
|
||||
to_check = self.get_peers_needing_check()
|
||||
if (datetime.datetime.utcnow() - last_flush).seconds > 60:
|
||||
log.info("flushing to db")
|
||||
await self.flush_to_db()
|
||||
last_flush = datetime.datetime.utcnow()
|
||||
while not to_check and not to_process:
|
||||
port = self.node.listening_port.get_extra_info('socket').getsockname()[1]
|
||||
self.node.stop()
|
||||
await self.node.start_listening()
|
||||
log.info("Idle, sleeping a minute. Port changed to %d", port)
|
||||
await asyncio.sleep(60.0)
|
||||
to_check = self.get_peers_needing_check()
|
||||
|
||||
|
||||
class SimpleMetrics:
|
||||
def __init__(self, port):
|
||||
self.prometheus_port = port
|
||||
|
||||
async def handle_metrics_get_request(self, _):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
log.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def start(self):
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
metrics_runner = web.AppRunner(prom_app)
|
||||
await metrics_runner.setup()
|
||||
prom_site = web.TCPSite(metrics_runner, "0.0.0.0", self.prometheus_port)
|
||||
await prom_site.start()
|
||||
|
||||
|
||||
def dict_row_factory(cursor, row):
|
||||
d = {}
|
||||
for idx, col in enumerate(cursor.description):
|
||||
if col[0] in ('added_on', 'first_online', 'last_seen', 'last_check'):
|
||||
d[col[0]] = datetime.datetime.fromisoformat(row[idx]) if row[idx] else None
|
||||
else:
|
||||
d[col[0]] = row[idx]
|
||||
return d
|
||||
|
||||
|
||||
async def test():
|
||||
db_path = "/tmp/peers.db" if len(sys.argv) == 1 else sys.argv[-1]
|
||||
asyncio.get_event_loop().set_debug(True)
|
||||
metrics = SimpleMetrics('8080')
|
||||
await metrics.start()
|
||||
conf = Config()
|
||||
hosting_samples = SDHashSamples("test.sample") if os.path.isfile("test.sample") else None
|
||||
crawler = Crawler(db_path, hosting_samples)
|
||||
await crawler.open()
|
||||
await crawler.flush_to_db()
|
||||
await crawler.node.start_listening()
|
||||
if crawler.active_peers_count < 100:
|
||||
probes = []
|
||||
for (host, port) in conf.known_dht_nodes:
|
||||
probes.append(asyncio.create_task(crawler.crawl_routing_table(host, port)))
|
||||
await asyncio.gather(*probes)
|
||||
await crawler.flush_to_db()
|
||||
await asyncio.gather(crawler.process(), crawler.probe_files())
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(test())
|
||||
+1
-1
@@ -83,7 +83,7 @@ async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional
|
||||
await storage.open()
|
||||
node = Node(
|
||||
loop, PeerManager(loop), node_id, port, port, 3333, None,
|
||||
storage=storage, is_bootstrap_node=True
|
||||
storage=storage
|
||||
)
|
||||
if prometheus_port > 0:
|
||||
metrics = SimpleMetrics(prometheus_port, node if export else None)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
from pprint import pprint
|
||||
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch._async.helpers import async_scan, async_bulk
|
||||
|
||||
DB = {}
|
||||
INDEX = 'claims'
|
||||
|
||||
|
||||
async def generate_support_amounts(client: AsyncElasticsearch):
|
||||
async for doc in async_scan(client):
|
||||
DB[doc['_id']] = doc['_source']['support_amount']
|
||||
if len(DB) > 10:
|
||||
break
|
||||
pprint(DB)
|
||||
|
||||
|
||||
def generate_support_to_trending():
|
||||
for claim_id, amount in DB.items():
|
||||
yield {'doc': {"trending_mixed": amount}, '_id': claim_id, '_index': INDEX, '_op_type': 'update'}
|
||||
|
||||
|
||||
async def write_trending(client: AsyncElasticsearch):
|
||||
await async_bulk(client, generate_support_to_trending())
|
||||
|
||||
|
||||
def get_client(host='localhost', port=9201):
|
||||
hosts = [{'host': host, 'port': port}]
|
||||
return AsyncElasticsearch(hosts, timeout=port)
|
||||
|
||||
|
||||
async def run():
|
||||
client = get_client()
|
||||
await generate_support_amounts(client)
|
||||
await write_trending(client)
|
||||
for claim_id, value in DB.items():
|
||||
if value > 0:
|
||||
break
|
||||
doc = await client.get(INDEX, claim_id)
|
||||
pprint(doc)
|
||||
pprint(DB[claim_id])
|
||||
await client.close()
|
||||
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(run())
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Hook for libtorrent.
|
||||
"""
|
||||
|
||||
import os
|
||||
import glob
|
||||
import os.path
|
||||
from PyInstaller.utils.hooks import get_module_file_attribute
|
||||
from PyInstaller import compat
|
||||
|
||||
|
||||
def get_binaries():
|
||||
if compat.is_win:
|
||||
files = ('c:/Windows/System32/libssl-1_1-x64.dll', 'c:/Windows/System32/libcrypto-1_1-x64.dll')
|
||||
for file in files:
|
||||
if not os.path.isfile(file):
|
||||
print(f"MISSING {file}")
|
||||
return [(file, '.') for file in files]
|
||||
return []
|
||||
|
||||
|
||||
binaries = get_binaries()
|
||||
for file in glob.glob(os.path.join(get_module_file_attribute('libtorrent'), 'libtorrent*pyd*')):
|
||||
binaries.append((file, 'libtorrent'))
|
||||
@@ -1,44 +0,0 @@
|
||||
import asyncio
|
||||
from typing import Iterable
|
||||
|
||||
from lbry.extras.daemon.client import daemon_rpc
|
||||
from lbry.conf import Config
|
||||
conf = Config()
|
||||
|
||||
|
||||
async def sample_prefix(prefix: bytes):
|
||||
result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page_size=50)
|
||||
total_pages = result['total_pages']
|
||||
print(total_pages)
|
||||
sd_hashes = set()
|
||||
for page in range(1, total_pages + 1):
|
||||
if page > 1:
|
||||
result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page=page, page_size=50)
|
||||
for item in result['items']:
|
||||
sd_hash = item.get('value', {}).get('source', {}).get('sd_hash')
|
||||
if not sd_hash:
|
||||
print('err', item)
|
||||
continue
|
||||
sd_hashes.add(sd_hash)
|
||||
print('page', page, len(sd_hashes))
|
||||
return sd_hashes
|
||||
|
||||
|
||||
def save_sample(name: str, samples: Iterable[str]):
|
||||
with open(name, 'wb') as outfile:
|
||||
for sample in samples:
|
||||
outfile.write(bytes.fromhex(sample))
|
||||
outfile.flush()
|
||||
print(outfile.tell())
|
||||
|
||||
|
||||
async def main():
|
||||
samples = set()
|
||||
futs = [asyncio.ensure_future(sample_prefix(bytes([i]))) for i in range(256)]
|
||||
for i, completed in enumerate(asyncio.as_completed(futs)):
|
||||
samples.update(await completed)
|
||||
print(i, len(samples))
|
||||
print(save_sample("test.sample", samples))
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -9,7 +9,7 @@ omit =
|
||||
lbry/wallet/orchstr8/
|
||||
.tox/*/lib/python*/site-packages/lbry/wallet/orchstr8/node.py
|
||||
|
||||
[cryptography.*,coincurve.*,pbkdf2,libtorrent]
|
||||
[cryptography.*,coincurve.*,pbkdf2, libtorrent]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[pylint]
|
||||
|
||||
@@ -7,6 +7,12 @@ BASE = os.path.dirname(__file__)
|
||||
with open(os.path.join(BASE, 'README.md'), encoding='utf-8') as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
|
||||
ROCKSDB = []
|
||||
if sys.platform.startswith('linux') or sys.platform.startswith('darwin'):
|
||||
ROCKSDB.append('lbry-rocksdb==0.8.2')
|
||||
|
||||
|
||||
setup(
|
||||
name=__name__,
|
||||
version=__version__,
|
||||
@@ -38,17 +44,23 @@ setup(
|
||||
'cffi==1.13.2',
|
||||
'cryptography==2.5',
|
||||
'protobuf==3.17.2',
|
||||
'msgpack==0.6.1',
|
||||
'prometheus_client==0.7.1',
|
||||
'ecdsa==0.13.3',
|
||||
'pyyaml==5.3.1',
|
||||
'docopt==0.6.2',
|
||||
'hachoir==3.1.2',
|
||||
'multidict==4.6.1',
|
||||
'coincurve==15.0.0',
|
||||
'pbkdf2==1.3',
|
||||
'attrs==18.2.0',
|
||||
'pylru==1.1.0',
|
||||
'elasticsearch==7.10.1',
|
||||
'grpcio==1.38.0',
|
||||
'filetype==1.0.9',
|
||||
'libtorrent==2.0.6',
|
||||
],
|
||||
] + ROCKSDB,
|
||||
extras_require={
|
||||
'torrent': ['lbry-libtorrent'],
|
||||
'lint': [
|
||||
'pylint==2.10.0'
|
||||
],
|
||||
@@ -57,7 +69,7 @@ setup(
|
||||
'jsonschema==4.4.0',
|
||||
],
|
||||
'hub': [
|
||||
'hub@git+https://github.com/lbryio/hub.git@9e3963ba23abf2f9ace4654089a7df1fd553f121'
|
||||
'hub@git+https://github.com/lbryio/hub.git@76dd9c392b776a2823015762814f375794120076'
|
||||
]
|
||||
},
|
||||
classifiers=[
|
||||
|
||||
@@ -102,7 +102,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.stop()
|
||||
initial_height = self.ledger.local_height_including_downloaded_height
|
||||
await self.blockchain.generate(100)
|
||||
while self.conductor.spv_node.server.session_manager.notified_height < initial_height + 100:
|
||||
while self.conductor.spv_node.server.session_manager.notified_height < initial_height + 99: # off by 1
|
||||
await asyncio.sleep(0.1)
|
||||
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
|
||||
await self.ledger.headers.open()
|
||||
|
||||
@@ -387,6 +387,11 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
# need to use new password2 in sync_apply
|
||||
with self.assertRaises(InvalidPasswordError):
|
||||
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
|
||||
# first test --no_password_change
|
||||
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True, no_password_change=True)
|
||||
# sync_apply should leave original password
|
||||
self.assertEqual(wallet.encryption_password, 'password')
|
||||
# now test with password change to password2
|
||||
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True)
|
||||
# sync_apply with new password2 also sets it as new local password
|
||||
self.assertEqual(wallet.encryption_password, 'password2')
|
||||
@@ -415,69 +420,3 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
|
||||
daemon2.jsonrpc_wallet_lock()
|
||||
self.assertTrue(await daemon2.jsonrpc_wallet_unlock('password3'))
|
||||
|
||||
async def test_wallet_import_and_export(self):
|
||||
daemon, daemon2 = self.daemon, self.daemon2
|
||||
|
||||
# Preferences
|
||||
self.assertFalse(daemon.jsonrpc_preference_get())
|
||||
self.assertFalse(daemon2.jsonrpc_preference_get())
|
||||
|
||||
daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]')
|
||||
daemon.jsonrpc_preference_set("one", "1")
|
||||
daemon.jsonrpc_preference_set("conflict", "1")
|
||||
daemon2.jsonrpc_preference_set("another", "A")
|
||||
await asyncio.sleep(1)
|
||||
# these preferences will win after merge since they are "newer"
|
||||
daemon2.jsonrpc_preference_set("two", "2")
|
||||
daemon2.jsonrpc_preference_set("conflict", "2")
|
||||
daemon.jsonrpc_preference_set("another", "B")
|
||||
|
||||
self.assertDictEqual(daemon.jsonrpc_preference_get(), {
|
||||
"one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"]
|
||||
})
|
||||
self.assertDictEqual(daemon2.jsonrpc_preference_get(), {
|
||||
"two": "2", "conflict": "2", "another": "A"
|
||||
})
|
||||
|
||||
self.assertItemCount(await daemon.jsonrpc_account_list(), 1)
|
||||
|
||||
data = await daemon2.jsonrpc_wallet_export(password='password')
|
||||
await daemon.jsonrpc_wallet_import(data=data, password='password', blocking=True)
|
||||
|
||||
self.assertItemCount(await daemon.jsonrpc_account_list(), 2)
|
||||
self.assertDictEqual(
|
||||
# "two" key added and "conflict" value changed to "2"
|
||||
daemon.jsonrpc_preference_get(),
|
||||
{"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]}
|
||||
)
|
||||
|
||||
# Channel Certificate
|
||||
# non-deterministic channel
|
||||
self.daemon2.wallet_manager.default_account.channel_keys['mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8'] = (
|
||||
'-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95'
|
||||
'466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh'
|
||||
'0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS'
|
||||
'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n'
|
||||
)
|
||||
channel = await self.create_nondeterministic_channel('@foo', '0.1', unhexlify(
|
||||
'3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1'
|
||||
'66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16'
|
||||
'a97a6d6a4a8effd29d748901bb9789352519cd00b13d'
|
||||
), self.daemon2, blocking=True)
|
||||
await self.confirm_tx(channel['txid'], self.daemon2.ledger)
|
||||
|
||||
# both daemons will have the channel but only one has the cert so far
|
||||
self.assertItemCount(await daemon.jsonrpc_channel_list(), 1)
|
||||
self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
|
||||
self.assertItemCount(await daemon2.jsonrpc_channel_list(), 1)
|
||||
self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
|
||||
|
||||
data = await daemon2.jsonrpc_wallet_export(password='password')
|
||||
await daemon.jsonrpc_wallet_import(data=data, password='password', blocking=True)
|
||||
|
||||
# both daemons have the cert after sync'ing
|
||||
self.assertEqual(
|
||||
daemon2.wallet_manager.default_account.channel_keys,
|
||||
daemon.wallet_manager.default_wallet.accounts[1].channel_keys
|
||||
)
|
||||
@@ -126,28 +126,12 @@ class TestESSync(CommandTestCase):
|
||||
# stop the es writer and advance the chain by 1, adding a new claim. upon resuming the es writer, it should
|
||||
# add the new claim
|
||||
await es_writer.stop()
|
||||
|
||||
stream11 = self.get_claim_id(await self.stream_create(f"stream11", bid='0.001', confirm=False))
|
||||
current_height = self.conductor.spv_node.writer.height
|
||||
await self.stream_create(f"stream11", bid='0.001', confirm=False)
|
||||
generate_block_task = asyncio.create_task(self.generate(1))
|
||||
await self.conductor.spv_node.writer.wait_until_block(current_height + 1)
|
||||
|
||||
await es_writer.start()
|
||||
await generate_block_task
|
||||
self.assertEqual(11, len(await self.claim_search(order_by=['height'])))
|
||||
|
||||
# stop/delete es and advance the chain by 1, removing stream11
|
||||
await es_writer.delete_index()
|
||||
await es_writer.stop()
|
||||
server_search_client.clear_caches()
|
||||
await self.stream_abandon(stream11, confirm=False)
|
||||
current_height = self.conductor.spv_node.writer.height
|
||||
generate_block_task = asyncio.create_task(self.generate(1))
|
||||
await self.conductor.spv_node.writer.wait_until_block(current_height + 1)
|
||||
await es_writer.start(reindex=True)
|
||||
await generate_block_task
|
||||
self.assertEqual(10, len(await self.claim_search(order_by=['height'])))
|
||||
|
||||
|
||||
# # this time we will test a migration from unversioned to v1
|
||||
# await db.search_index.sync_client.indices.delete_template(db.search_index.index)
|
||||
|
||||
@@ -1527,7 +1527,7 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(searched_repost['signing_channel']['claim_id'], spam_claim_id)
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
some_channel_id = self.get_claim_id(await self.channel_create('@some_channel', '0.1'))
|
||||
await self.channel_create('@some_channel', '0.1')
|
||||
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
|
||||
bad_content_id = self.get_claim_id(
|
||||
await self.stream_create('bad_content', '0.1', channel_name='@some_channel', tags=['bad'])
|
||||
@@ -1591,22 +1591,15 @@ class StreamCommands(ClaimTestCase):
|
||||
await self.daemon.wallet_manager.reset()
|
||||
|
||||
self.assertEqual(0, len(self.conductor.spv_node.es_writer.db.blocked_streams))
|
||||
block1_claim_id = self.get_claim_id(
|
||||
await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking',
|
||||
description='a descriptive good reason')
|
||||
)
|
||||
await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking')
|
||||
self.assertEqual(1, len(self.conductor.spv_node.es_writer.db.blocked_streams))
|
||||
|
||||
# blocked content is not resolveable
|
||||
error = (await self.resolve('lbry://@some_channel/bad_content'))['error']
|
||||
print((await self.resolve('lbry://@some_channel/bad_content'))['error']['text'])
|
||||
self.assertEqual(error['name'], 'BLOCKED')
|
||||
self.assertTrue(error['text'].startswith(f"Resolve of 'lbry://@some_channel#{some_channel_id[:1]}/bad_content#{bad_content_id[:1]}' was blocked by lbry://@blocking"))
|
||||
self.assertTrue(error['text'].endswith(f"Reason given: a descriptive good reason"))
|
||||
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
|
||||
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
|
||||
|
||||
self.assertEqual((await self.resolve('lbry://@blocking/block1'))['error']['name'], 'BLOCKED')
|
||||
|
||||
# a filtered/blocked channel impacts all content inside it
|
||||
bad_channel_id = self.get_claim_id(
|
||||
await self.channel_create('@bad_channel', '0.1', tags=['bad-stuff'])
|
||||
@@ -1663,10 +1656,6 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED')
|
||||
self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED')
|
||||
|
||||
self.assertTrue('error' in await self.resolve('lbry://@some_channel/bad_content'))
|
||||
await self.stream_abandon(block1_claim_id)
|
||||
self.assertTrue('error' not in await self.resolve('lbry://@some_channel/bad_content'))
|
||||
|
||||
async def test_publish_updates_file_list(self):
|
||||
tx = await self.stream_create(title='created')
|
||||
txo = tx['outputs'][0]
|
||||
@@ -1692,6 +1681,7 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(tx['txid'], files[0]['txid'])
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
@skip
|
||||
async def test_setting_stream_fields(self):
|
||||
values = {
|
||||
'title': "Cool Content",
|
||||
|
||||
@@ -18,17 +18,11 @@ class FileCommands(CommandTestCase):
|
||||
super().__init__(*a, **kw)
|
||||
self.skip_libtorrent = False
|
||||
|
||||
async def add_forever(self):
|
||||
while True:
|
||||
for handle in self.client_session._handles.values():
|
||||
handle._handle.connect_peer(('127.0.0.1', 4040))
|
||||
await asyncio.sleep(.1)
|
||||
|
||||
async def initialize_torrent(self, tx_to_update=None):
|
||||
if not hasattr(self, 'seeder_session'):
|
||||
self.seeder_session = TorrentSession(self.loop, None)
|
||||
self.addCleanup(self.seeder_session.stop)
|
||||
await self.seeder_session.bind('127.0.0.1', port=4040)
|
||||
await self.seeder_session.bind(port=4040)
|
||||
btih = await self.seeder_session.add_fake_torrent()
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
if not tx_to_update:
|
||||
@@ -46,9 +40,8 @@ class FileCommands(CommandTestCase):
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
|
||||
self.client_session._session.add_dht_node(('localhost', 4040))
|
||||
self.client_session.wait_start = False # fixme: this is super slow on tests
|
||||
task = asyncio.create_task(self.add_forever())
|
||||
self.addCleanup(task.cancel)
|
||||
return tx, btih
|
||||
|
||||
@skipIf(TorrentSession is None, "libtorrent not installed")
|
||||
|
||||
@@ -131,7 +131,7 @@ class RangeRequests(CommandTestCase):
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
|
||||
# test that repeated range requests do not create duplicate files
|
||||
for _ in range(3):
|
||||
@@ -140,7 +140,7 @@ class RangeRequests(CommandTestCase):
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
@@ -148,7 +148,7 @@ class RangeRequests(CommandTestCase):
|
||||
# test that a range request after restart does not create a duplicate file
|
||||
await self._restart_stream_manager()
|
||||
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
@@ -162,7 +162,7 @@ class RangeRequests(CommandTestCase):
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
@@ -174,7 +174,7 @@ class RangeRequests(CommandTestCase):
|
||||
stream = (await self.daemon.jsonrpc_file_list())['items'][0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
|
||||
# test that repeated range requests do not create duplicate files
|
||||
for _ in range(3):
|
||||
@@ -182,7 +182,7 @@ class RangeRequests(CommandTestCase):
|
||||
stream = (await self.daemon.jsonrpc_file_list())['items'][0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
@@ -190,7 +190,7 @@ class RangeRequests(CommandTestCase):
|
||||
# test that a range request after restart does not create a duplicate file
|
||||
await self._restart_stream_manager()
|
||||
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
@@ -202,7 +202,7 @@ class RangeRequests(CommandTestCase):
|
||||
stream = (await self.daemon.jsonrpc_file_list())['items'][0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(self.daemon.conf.data_dir))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
ExchangeRate, ExchangeRateManager, FEEDS, MarketFeed
|
||||
)
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateManager, FEEDS, MarketFeed
|
||||
|
||||
|
||||
class TestExchangeRateManager(AsyncioTestCase):
|
||||
|
||||
@@ -215,21 +215,6 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1][:17]}', colliding_claim_ids[1])
|
||||
await self.assertResolvesToClaimId(f'@abc#{colliding_claim_ids[1]}', colliding_claim_ids[1])
|
||||
|
||||
# test resolving different streams for a channel using short urls
|
||||
self.get_claim_id(
|
||||
await self.stream_create('foo1', '0.01', channel_id=colliding_claim_ids[0])
|
||||
)
|
||||
self.get_claim_id(
|
||||
await self.stream_create('foo2', '0.01', channel_id=colliding_claim_ids[0])
|
||||
)
|
||||
duplicated_resolved = list((
|
||||
await self.ledger.resolve([], [
|
||||
f'@abc#{colliding_claim_ids[0][:2]}/foo1', f'@abc#{colliding_claim_ids[0][:2]}/foo2'
|
||||
])
|
||||
).values())
|
||||
self.assertEqual('foo1', duplicated_resolved[0].normalized_name)
|
||||
self.assertEqual('foo2', duplicated_resolved[1].normalized_name)
|
||||
|
||||
async def test_abandon_channel_and_claims_in_same_tx(self):
|
||||
channel_id = self.get_claim_id(
|
||||
await self.channel_create('@abc', '0.01')
|
||||
@@ -1550,10 +1535,18 @@ class ResolveAfterReorg(BaseResolveTestCase):
|
||||
blocks = self.ledger.headers.height - start
|
||||
self.blockchain.block_expected = start - 1
|
||||
|
||||
|
||||
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
|
||||
# go back to start
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(start)).decode())
|
||||
# go to previous + 1
|
||||
await self.generate(blocks + 2)
|
||||
await self.blockchain.generate(blocks + 2)
|
||||
|
||||
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
|
||||
await self.conductor.spv_node.server.synchronized.wait()
|
||||
# await asyncio.wait_for(self.on_header(self.blockchain.block_expected), 30.0)
|
||||
|
||||
async def assertBlockHash(self, height):
|
||||
reader_db = self.conductor.spv_node.server.db
|
||||
|
||||
@@ -11,35 +11,6 @@ from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
|
||||
class TestBootstrapNode(AsyncioTestCase):
|
||||
TIMEOUT = 10.0 # do not increase. Hitting a timeout is a real failure
|
||||
|
||||
async def test_bootstrap_node_adds_all_peers(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.set_debug(False)
|
||||
|
||||
with dht_mocks.mock_network_loop(loop):
|
||||
advance = dht_mocks.get_time_accelerator(loop)
|
||||
self.bootstrap_node = Node(self.loop, PeerManager(loop), constants.generate_id(),
|
||||
4444, 4444, 3333, '1.2.3.4', is_bootstrap_node=True)
|
||||
self.bootstrap_node.start('1.2.3.4', [])
|
||||
self.bootstrap_node.protocol.ping_queue._default_delay = 0
|
||||
self.addCleanup(self.bootstrap_node.stop)
|
||||
|
||||
# start the nodes
|
||||
nodes = {}
|
||||
futs = []
|
||||
for i in range(100):
|
||||
nodes[i] = Node(loop, PeerManager(loop), constants.generate_id(i), 4444, 4444, 3333, f'1.3.3.{i}')
|
||||
nodes[i].start(f'1.3.3.{i}', [('1.2.3.4', 4444)])
|
||||
self.addCleanup(nodes[i].stop)
|
||||
futs.append(nodes[i].joined.wait())
|
||||
await asyncio.gather(*futs)
|
||||
while self.bootstrap_node.protocol.ping_queue.busy:
|
||||
await advance(1)
|
||||
self.assertEqual(100, len(self.bootstrap_node.protocol.routing_table.get_peers()))
|
||||
|
||||
|
||||
class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
async def test_ping_queue_discover(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
@@ -209,7 +209,7 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
self.assertEqual(len(wallet1.accounts), 1)
|
||||
self.assertEqual(wallet1.preferences, {'one': 1, 'conflict': 1})
|
||||
|
||||
added, _ = wallet1.merge(self.manager, 'password', wallet2.pack('password'))
|
||||
added = wallet1.merge(self.manager, 'password', wallet2.pack('password'))
|
||||
self.assertEqual(added[0].id, wallet2.default_account.id)
|
||||
self.assertEqual(len(wallet1.accounts), 2)
|
||||
self.assertEqual(wallet1.accounts[1].id, wallet2.default_account.id)
|
||||
|
||||
@@ -5,10 +5,12 @@ deps =
|
||||
extras =
|
||||
test
|
||||
hub
|
||||
torrent
|
||||
changedir = {toxinidir}/tests
|
||||
setenv =
|
||||
HOME=/tmp
|
||||
ELASTIC_HOST={env:ELASTIC_HOST:localhost}
|
||||
ENABLE_LEGACY_SEARCH=0
|
||||
commands =
|
||||
orchstr8 download
|
||||
blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs}
|
||||
@@ -17,3 +19,13 @@ commands =
|
||||
transactions: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.transactions {posargs}
|
||||
datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs}
|
||||
other: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.other {posargs}
|
||||
[testenv:claims_legacy_search]
|
||||
setenv =
|
||||
ENABLE_LEGACY_SEARCH=1
|
||||
commands =
|
||||
coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.claims {posargs}
|
||||
[testenv:takeovers_legacy_search]
|
||||
setenv =
|
||||
ENABLE_LEGACY_SEARCH=1
|
||||
commands =
|
||||
coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.takeovers {posargs}
|
||||
Reference in new issue
Block a user