Compare commits
63
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1134a5a76c | ||
|
|
ad489ed606 | ||
|
|
bb541901d9 | ||
|
|
ca4ba19a5e | ||
|
|
f05943ff79 | ||
|
|
7ded8a1333 | ||
|
|
c2478d4add | ||
|
|
f69747bc89 | ||
|
|
441cc950aa | ||
|
|
a76a0ac8c4 | ||
|
|
8b1009161a | ||
|
|
868a620e91 | ||
|
|
a0e34b0bc8 | ||
|
|
612dbcb2f3 | ||
|
|
b3614d965d | ||
|
|
5d7137255e | ||
|
|
6ff867ef55 | ||
|
|
c14915df29 | ||
|
|
7d4966e2ae | ||
|
|
3876e0317d | ||
|
|
0b2b10f759 | ||
|
|
9a79b33664 | ||
|
|
af1a6edd15 | ||
|
|
b78929f4d5 | ||
|
|
fb6e342043 | ||
|
|
0faa2d35da | ||
|
|
511e57c231 | ||
|
|
d762d675c4 | ||
|
|
3fdadee87c | ||
|
|
1aa4d9d585 | ||
|
|
8019f4bdb3 | ||
|
|
ca65c1ebc5 | ||
|
|
f0e47aae86 | ||
|
|
dc7cd545ba | ||
|
|
76bd59d82e | ||
|
|
461687ffb4 | ||
|
|
dd5b9ca81b | ||
|
|
89ed04f8a7 | ||
|
|
ec0d9f06c5 | ||
|
|
03b59ac6fc | ||
|
|
43ac3336d7 | ||
|
|
d12c78db74 | ||
|
|
bfaf1b0957 | ||
|
|
bb60c385d5 | ||
|
|
c96d1d9c32 | ||
|
|
7c7a0d4bdf | ||
|
|
cc829a7bf4 | ||
|
|
e0ea6383e2 | ||
|
|
bcec5dc2ae | ||
|
|
cba9c16a06 | ||
|
|
dd68fb077b | ||
|
|
c2294e97db | ||
|
|
c0f512ace7 | ||
|
|
3305eb67c6 | ||
|
|
c9d637b4da | ||
|
|
ae3e8fadf5 | ||
|
|
a1abd94387 | ||
|
|
9b463a8cab | ||
|
|
babc54a240 | ||
|
|
5836a93b21 | ||
|
|
557348e345 | ||
|
|
9adfec6b00 | ||
|
|
3a496902f8 |
No files matched your search
@@ -36,31 +36,29 @@ class DiskSpaceManager:
|
||||
await self._clean(True)
|
||||
|
||||
async def _clean(self, is_network_blob=False):
|
||||
space_used_bytes = await self.get_space_used_bytes()
|
||||
space_used_mb = await self.get_space_used_mb(cached=False)
|
||||
if is_network_blob:
|
||||
space_used_bytes = space_used_bytes['network_storage']
|
||||
space_used_mb = space_used_mb['network_storage']
|
||||
else:
|
||||
space_used_bytes = space_used_bytes['content_storage'] + space_used_bytes['private_storage']
|
||||
space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage']
|
||||
storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
|
||||
storage_limit = storage_limit_mb*1024*1024 if storage_limit_mb else None
|
||||
if self.analytics:
|
||||
asyncio.create_task(
|
||||
self.analytics.send_disk_space_used(space_used_bytes, storage_limit, is_network_blob)
|
||||
self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob)
|
||||
)
|
||||
if not storage_limit:
|
||||
return 0
|
||||
delete = []
|
||||
available = storage_limit - space_used_bytes
|
||||
if available > 0:
|
||||
available = storage_limit_mb - space_used_mb
|
||||
if storage_limit_mb == 0 if not is_network_blob else available >= 0:
|
||||
return 0
|
||||
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob):
|
||||
delete.append(blob_hash)
|
||||
available += file_size
|
||||
if available > 0:
|
||||
available += int(file_size/1024.0/1024.0)
|
||||
if available >= 0:
|
||||
break
|
||||
if delete:
|
||||
await self.db.stop_all_files()
|
||||
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
|
||||
self._used_space_bytes = None
|
||||
return len(delete)
|
||||
|
||||
async def cleaning_loop(self):
|
||||
|
||||
@@ -97,21 +97,15 @@ class BlobDownloader:
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
if peer in self.active_connections or not self.should_race_continue(blob):
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
|
||||
+1
-1
@@ -622,7 +622,7 @@ class Config(CLIConfig):
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
"use.", 2
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
|
||||
+38
-16
@@ -1,6 +1,9 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,24 +12,39 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
announcements_sent_metric = Counter(
|
||||
"announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
|
||||
labelnames=("peers", "error"),
|
||||
)
|
||||
announcement_queue_size_metric = Gauge(
|
||||
"announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.announce_queue: typing.List[str] = []
|
||||
self._done = asyncio.Event()
|
||||
self.announced = set()
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
async def _run_consumer(self):
|
||||
while self.announce_queue:
|
||||
try:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
self.announcements_sent_metric.labels(peers=peers, error=False).inc()
|
||||
if peers > 4:
|
||||
self.announced.add(blob_hash)
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
self.announcements_sent_metric.labels(peers=0, error=True).inc()
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
@@ -37,17 +55,18 @@ class BlobAnnouncer:
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)], loop=self.loop)
|
||||
announced = list(filter(None, self.announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
self.announced.clear()
|
||||
self._done.set()
|
||||
self._done.clear()
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
@@ -56,3 +75,6 @@ class BlobAnnouncer:
|
||||
def stop(self):
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
|
||||
def wait(self):
|
||||
return self._done.wait()
|
||||
@@ -20,7 +20,6 @@ MAYBE_PING_DELAY = 300 # 5 minutes
|
||||
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
|
||||
RPC_ID_LENGTH = 20
|
||||
PROTOCOL_VERSION = 1
|
||||
BOTTOM_OUT_LIMIT = 3
|
||||
MSG_SIZE_LIMIT = 1400
|
||||
|
||||
|
||||
|
||||
+4
-6
@@ -202,25 +202,23 @@ class Node:
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
|
||||
max_results: int = constants.K) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
async for iteration_peers in self.get_iterative_node_finder(
|
||||
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
|
||||
node_id, shortlist=shortlist, max_results=max_results):
|
||||
peers.extend(iteration_peers)
|
||||
distance = Distance(node_id)
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
|
||||
+4
-1
@@ -11,7 +11,7 @@ from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
CACHE_SIZE = 2048
|
||||
CACHE_SIZE = 16384
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -190,3 +190,6 @@ class KademliaPeer:
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})"
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
from itertools import chain
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, OrderedDict
|
||||
import typing
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -74,7 +74,7 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
@@ -85,28 +85,22 @@ class IterativeFinder:
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.bottom_out_limit = bottom_out_limit
|
||||
self.max_results = max_results
|
||||
self.max_results = max(constants.K, max_results)
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.active: typing.Dict['KademliaPeer', int] = OrderedDict() # peer: distance, sorted
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {}
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer)
|
||||
self._add_active(peer, force=True)
|
||||
else:
|
||||
# seed nodes
|
||||
self._schedule_probe(peer)
|
||||
@@ -138,15 +132,14 @@ class IterativeFinder:
|
||||
"""
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _add_active(self, peer):
|
||||
def _add_active(self, peer, force=False):
|
||||
if not force and self.peer_manager.peer_is_good(peer) is False:
|
||||
return
|
||||
if peer in self.contacted:
|
||||
return
|
||||
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
|
||||
self.active.add(peer)
|
||||
if self._is_closer(peer):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = peer
|
||||
self.active[peer] = self.distance(peer.node_id)
|
||||
self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1]))
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
self._add_active(peer)
|
||||
@@ -158,33 +151,43 @@ class IterativeFinder:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
self._log_state()
|
||||
|
||||
def _reset_closest(self, peer):
|
||||
if peer in self.active:
|
||||
del self.active[peer]
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to closest active peers
|
||||
"""
|
||||
|
||||
added = 0
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.ALPHA:
|
||||
for index, peer in enumerate(self.active.keys()):
|
||||
if index == 0:
|
||||
log.debug("closest to probe: %s", peer.node_id.hex()[:8])
|
||||
if peer in self.contacted:
|
||||
continue
|
||||
if len(self.running_probes) >= constants.ALPHA:
|
||||
break
|
||||
if index > (constants.K + len(self.running_probes)):
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
@@ -206,33 +209,22 @@ class IterativeFinder:
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
self.running_probes.pop(peer, None)
|
||||
if self.running:
|
||||
self._search_round()
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
self.running_probes[peer] = t
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
def _log_state(self):
|
||||
log.debug("[%s] check result: %i active nodes %i contacted",
|
||||
self.key.hex()[:8], len(self.active), len(self.contacted))
|
||||
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
self.loop.call_soon(self._search_round)
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
@@ -252,20 +244,19 @@ class IterativeFinder:
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
for task in chain(self.tasks, self.running_probes.values()):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
@@ -276,14 +267,14 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
self.put_result(self.active.keys(), finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
and self.peer_manager.peer_is_good(peer) is True # return only peers who answered
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
|
||||
@@ -298,26 +289,16 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.active, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
return self.put_result(self.active.keys(), finish=True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
@@ -362,23 +343,12 @@ class IterativeValueFinder(IterativeFinder):
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
self.bottom_out_count = 0
|
||||
for blob_peer in blob_peers:
|
||||
if blob_peer not in self.blob_peers:
|
||||
self.blob_peers.add(blob_peer)
|
||||
to_yield.append(blob_peer)
|
||||
if to_yield:
|
||||
# log.info("found %i new peers for blob", len(to_yield))
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
# if self.max_results and len(self.blob_peers) >= self.max_results:
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
@@ -6,6 +6,7 @@ import itertools
|
||||
|
||||
from prometheus_client import Gauge
|
||||
|
||||
from lbry import utils
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -70,9 +71,8 @@ class KBucket:
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").inc()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).inc()
|
||||
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc()
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@@ -140,9 +140,8 @@ class KBucket:
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").dec()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).dec()
|
||||
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec()
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
|
||||
@@ -381,6 +381,7 @@ class FileManagerComponent(Component):
|
||||
|
||||
|
||||
class BackgroundDownloaderComponent(Component):
|
||||
MIN_PREFIX_COLLIDING_BITS = 8
|
||||
component_name = BACKGROUND_DOWNLOADER_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
|
||||
|
||||
@@ -412,12 +413,18 @@ class BackgroundDownloaderComponent(Component):
|
||||
while True:
|
||||
self.space_available = await self.space_manager.get_free_space_mb(True)
|
||||
if not self.is_busy and self.space_available > 10:
|
||||
blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if
|
||||
key.hex() not in self.blob_manager.completed_blob_hashes), None)
|
||||
if blob_hash:
|
||||
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash))
|
||||
self._download_next_close_blob_hash()
|
||||
await asyncio.sleep(self.download_loop_delay_seconds)
|
||||
|
||||
def _download_next_close_blob_hash(self):
|
||||
node_id = self.dht_node.protocol.node_id
|
||||
for blob_hash in self.dht_node.stored_blob_hashes:
|
||||
if blob_hash.hex() in self.blob_manager.completed_blob_hashes:
|
||||
continue
|
||||
if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS:
|
||||
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex()))
|
||||
return
|
||||
|
||||
async def start(self):
|
||||
self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
|
||||
if not self.component_manager.has_component(DHT_COMPONENT):
|
||||
|
||||
@@ -37,7 +37,8 @@ from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import (
|
||||
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
|
||||
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError,
|
||||
InputValueError
|
||||
)
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
@@ -3614,15 +3615,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_claims) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
raise InputValueError(
|
||||
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
|
||||
old_txo = existing_claims[0]
|
||||
if not old_txo.claim.is_stream:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream claim."
|
||||
if not old_txo.claim.is_stream and not old_txo.claim.is_repost:
|
||||
# in principle it should work with any type of claim, but its safer to
|
||||
# limit it to ones we know won't be broken. in the future we can expand
|
||||
# this if we have a test case for e.g. channel or support claims
|
||||
raise InputValueError(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream or repost claim."
|
||||
)
|
||||
|
||||
if bid is not None:
|
||||
@@ -3653,28 +3656,32 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if replace:
|
||||
claim = Claim()
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
if old_txo.claim.is_stream:
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
else:
|
||||
claim = Claim.from_bytes(old_txo.claim.to_bytes())
|
||||
|
||||
if old_txo.claim.is_stream:
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
|
||||
if clear_channel:
|
||||
claim.clear_signature()
|
||||
tx = await Transaction.claim_update(
|
||||
old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0],
|
||||
channel if not clear_channel else None
|
||||
)
|
||||
|
||||
new_txo = tx.outputs[0]
|
||||
stream_hash = None
|
||||
if not preview:
|
||||
if not preview and old_txo.claim.is_stream:
|
||||
old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)
|
||||
old_stream = old_stream[0] if old_stream else None
|
||||
if file_path is not None:
|
||||
@@ -4878,20 +4885,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None, page=None, page_size=None):
|
||||
async def jsonrpc_peer_list(self, blob_hash, page=None, page_size=None):
|
||||
"""
|
||||
Get peers for blob hash
|
||||
|
||||
Usage:
|
||||
peer_list (<blob_hash> | --blob_hash=<blob_hash>)
|
||||
[<search_bottom_out_limit> | --search_bottom_out_limit=<search_bottom_out_limit>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--blob_hash=<blob_hash> : (str) find available peers for this blob hash
|
||||
--search_bottom_out_limit=<search_bottom_out_limit> : (int) the number of search probes in a row
|
||||
that don't find any new peers
|
||||
before giving up and returning
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
|
||||
@@ -4903,13 +4906,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid blob hash")
|
||||
if search_bottom_out_limit is not None:
|
||||
search_bottom_out_limit = int(search_bottom_out_limit)
|
||||
if search_bottom_out_limit <= 0:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid bottom out limit")
|
||||
else:
|
||||
search_bottom_out_limit = 4
|
||||
peers = []
|
||||
peer_q = asyncio.Queue(loop=self.component_manager.loop)
|
||||
await self.dht_node._peers_for_value_producer(blob_hash, peer_q)
|
||||
|
||||
@@ -37,6 +37,8 @@ def migrate_db(conf, start, end):
|
||||
from .migrate13to14 import do_migration
|
||||
elif current == 14:
|
||||
from .migrate14to15 import do_migration
|
||||
elif current == 15:
|
||||
from .migrate15to16 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
update blob set should_announce=0
|
||||
where should_announce=1 and
|
||||
blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -187,8 +187,8 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
"update blob set should_announce=1 where blob_hash in (?)",
|
||||
(sd_blob.blob_hash,)
|
||||
).fetchall()
|
||||
|
||||
|
||||
@@ -462,7 +462,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
content_blobs = await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
|
||||
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc",
|
||||
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc, blob.blob_length asc",
|
||||
(is_mine,)
|
||||
)
|
||||
return content_blobs + sd_blobs
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import filetype
|
||||
import logging
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
@@ -166,10 +168,38 @@ types_map = {
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
# maps detected extensions to the possible analogs
|
||||
# i.e. .cbz file is actually a .zip
|
||||
synonyms_map = {
|
||||
'.zip': ['.cbz'],
|
||||
'.rar': ['.cbr'],
|
||||
'.ar': ['.a']
|
||||
}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
|
||||
try:
|
||||
kind = filetype.guess(path)
|
||||
if kind:
|
||||
real_extension = f".{kind.extension}"
|
||||
|
||||
if extension != real_extension:
|
||||
if extension:
|
||||
log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
|
||||
else:
|
||||
log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
|
||||
|
||||
if extension not in synonyms_map.get(real_extension, []):
|
||||
extension = real_extension
|
||||
|
||||
except OSError as error:
|
||||
pass
|
||||
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
|
||||
@@ -98,10 +98,6 @@ class StreamDownloader:
|
||||
if not self.descriptor:
|
||||
await self.load_descriptor(connection_id)
|
||||
|
||||
# add the head blob to the peer search
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to peer search for stream %s", self.sd_hash)
|
||||
|
||||
if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream:
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
|
||||
|
||||
+7
-1
@@ -204,7 +204,13 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
|
||||
def add_timeout(self):
|
||||
if self.TIMEOUT:
|
||||
self.loop.call_later(self.TIMEOUT, self.cancel)
|
||||
self.loop.call_later(self.TIMEOUT, self.check_timeout, time())
|
||||
|
||||
def check_timeout(self, started):
|
||||
if time() - started >= self.TIMEOUT:
|
||||
self.cancel()
|
||||
else:
|
||||
self.loop.call_later(self.TIMEOUT, self.check_timeout, started)
|
||||
|
||||
|
||||
class AdvanceTimeTestCase(AsyncioTestCase):
|
||||
|
||||
@@ -474,3 +474,18 @@ class LockWithMetrics(asyncio.Lock):
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
|
||||
|
||||
def get_colliding_prefix_bits(first_value: bytes, second_value: bytes):
|
||||
"""
|
||||
Calculates the amount of colliding prefix bits between <first_value> and <second_value>.
|
||||
This is given by the amount of bits that are the same until the first different one (via XOR),
|
||||
starting from the most significant bit to the least significant bit.
|
||||
:param first_value: first value to compare, bigger than size.
|
||||
:param second_value: second value to compare, bigger than size.
|
||||
:return: amount of prefix colliding bits.
|
||||
"""
|
||||
assert len(first_value) == len(second_value), "length should be the same"
|
||||
size = len(first_value) * 8
|
||||
first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big")
|
||||
return size - (first_value ^ second_value).bit_length()
|
||||
@@ -39,9 +39,14 @@ class DeterministicChannelKeyManager:
|
||||
self.account = account
|
||||
self.last_known = 0
|
||||
self.cache = {}
|
||||
self.private_key: Optional[PrivateKey] = None
|
||||
if account.private_key is not None:
|
||||
self.private_key = account.private_key.child(KeyPath.CHANNEL)
|
||||
self._private_key: Optional[PrivateKey] = None
|
||||
|
||||
@property
|
||||
def private_key(self):
|
||||
if self._private_key is None:
|
||||
if self.account.private_key is not None:
|
||||
self._private_key = self.account.private_key.child(KeyPath.CHANNEL)
|
||||
return self._private_key
|
||||
|
||||
def maybe_generate_deterministic_key_for_channel(self, txo):
|
||||
if self.private_key is None:
|
||||
|
||||
@@ -2,6 +2,7 @@ NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
DUST = 1000
|
||||
|
||||
TIMEOUT = 30.0
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
__hub_url__ = (
|
||||
"https://github.com/lbryio/hub/releases/download/v0.2021.12.18.1/hub"
|
||||
"https://github.com/lbryio/hub/releases/download/v0.2022.01.21.1/hub"
|
||||
)
|
||||
from .node import Conductor
|
||||
from .service import ConductorService
|
||||
@@ -14,7 +14,7 @@ from lbry.schema.purchase import Purchase
|
||||
from lbry.schema.support import Support
|
||||
|
||||
from .script import InputScript, OutputScript
|
||||
from .constants import COIN, NULL_HASH32
|
||||
from .constants import COIN, DUST, NULL_HASH32
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .hash import TXRef, TXRefImmutable
|
||||
from .util import ReadOnlyList
|
||||
@@ -818,10 +818,10 @@ class Transaction:
|
||||
)
|
||||
if payment > cost:
|
||||
change = payment - cost
|
||||
if change > cost_of_change:
|
||||
change_amount = change - cost_of_change
|
||||
if change_amount > DUST:
|
||||
change_address = await change_account.change.get_or_create_usable_address()
|
||||
change_hash160 = change_account.ledger.address_to_hash160(change_address)
|
||||
change_amount = change - cost_of_change
|
||||
change_output = Output.pay_pubkey_hash(change_amount, change_hash160)
|
||||
change_output.is_internal_transfer = True
|
||||
tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)])
|
||||
|
||||
@@ -203,11 +203,12 @@ class Wallet:
|
||||
return True
|
||||
return False
|
||||
|
||||
def unlock(self, password):
|
||||
async def unlock(self, password):
|
||||
for account in self.accounts:
|
||||
if account.encrypted:
|
||||
if not account.decrypt(password):
|
||||
return False
|
||||
await account.deterministic_channel_keys.ensure_cache_primed()
|
||||
self.encryption_password = password
|
||||
return True
|
||||
|
||||
|
||||
@@ -56,7 +56,8 @@ setup(
|
||||
'attrs==18.2.0',
|
||||
'pylru==1.1.0',
|
||||
'elasticsearch==7.10.1',
|
||||
'grpcio==1.38.0'
|
||||
'grpcio==1.38.0',
|
||||
'filetype==1.0.9'
|
||||
] + PLYVEL,
|
||||
extras_require={
|
||||
'torrent': ['lbry-libtorrent'],
|
||||
|
||||
+8
-18
@@ -9,7 +9,7 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
|
||||
def get_time_accelerator(loop: asyncio.AbstractEventLoop,
|
||||
now: typing.Optional[float] = None) -> typing.Callable[[float], typing.Awaitable[None]]:
|
||||
instant_step: bool = False) -> typing.Callable[[float], typing.Awaitable[None]]:
|
||||
"""
|
||||
Returns an async advance() function
|
||||
|
||||
@@ -17,32 +17,22 @@ def get_time_accelerator(loop: asyncio.AbstractEventLoop,
|
||||
made by call_later, call_at, and call_soon.
|
||||
"""
|
||||
|
||||
_time = now or loop.time()
|
||||
loop.time = functools.wraps(loop.time)(lambda: _time)
|
||||
original = loop.time
|
||||
_drift = 0
|
||||
loop.time = functools.wraps(loop.time)(lambda: original() + _drift)
|
||||
|
||||
async def accelerate_time(seconds: float) -> None:
|
||||
nonlocal _time
|
||||
nonlocal _drift
|
||||
if seconds < 0:
|
||||
raise ValueError(f'Cannot go back in time ({seconds} seconds)')
|
||||
_time += seconds
|
||||
await past_events()
|
||||
_drift += seconds
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def past_events() -> None:
|
||||
while loop._scheduled:
|
||||
timer: asyncio.TimerHandle = loop._scheduled[0]
|
||||
if timer not in loop._ready and timer._when <= _time:
|
||||
loop._scheduled.remove(timer)
|
||||
loop._ready.append(timer)
|
||||
if timer._when > _time:
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def accelerator(seconds: float):
|
||||
steps = seconds * 10.0
|
||||
steps = seconds * 10.0 if not instant_step else 1
|
||||
|
||||
for _ in range(max(int(steps), 1)):
|
||||
await accelerate_time(0.1)
|
||||
await accelerate_time(seconds/steps)
|
||||
|
||||
return accelerator
|
||||
|
||||
|
||||
@@ -327,7 +327,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
with self.assertRaisesRegex(AssertionError, "Cannot lock an unencrypted wallet, encrypt first."):
|
||||
daemon.jsonrpc_wallet_lock()
|
||||
# safe to call unlock and decrypt, they are no-ops at this point
|
||||
daemon.jsonrpc_wallet_unlock('password') # already unlocked
|
||||
await daemon.jsonrpc_wallet_unlock('password') # already unlocked
|
||||
daemon.jsonrpc_wallet_decrypt() # already not encrypted
|
||||
|
||||
daemon.jsonrpc_wallet_encrypt('password')
|
||||
@@ -343,7 +343,7 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
# can't sign transactions with locked wallet
|
||||
with self.assertRaises(AssertionError):
|
||||
await daemon.jsonrpc_channel_create('@foo', '1.0')
|
||||
daemon.jsonrpc_wallet_unlock('password')
|
||||
await daemon.jsonrpc_wallet_unlock('password')
|
||||
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
|
||||
'is_syncing': False})
|
||||
await daemon.jsonrpc_channel_create('@foo', '1.0')
|
||||
@@ -361,10 +361,17 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
await daemon2.jsonrpc_channel_import(exported)
|
||||
self.assertTrue(daemon2.jsonrpc_wallet_encrypt('password'))
|
||||
self.assertTrue(daemon2.jsonrpc_wallet_lock())
|
||||
self.assertTrue(daemon2.jsonrpc_wallet_unlock("password"))
|
||||
self.assertTrue(await daemon2.jsonrpc_wallet_unlock("password"))
|
||||
self.assertEqual(daemon2.jsonrpc_wallet_status(),
|
||||
{'is_locked': False, 'is_encrypted': True, 'is_syncing': False})
|
||||
|
||||
async def test_locking_unlocking_does_not_break_deterministic_channels(self):
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_encrypt("password"))
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_lock())
|
||||
self.account.deterministic_channel_keys._private_key = None
|
||||
self.assertTrue(await self.daemon.jsonrpc_wallet_unlock("password"))
|
||||
await self.channel_create()
|
||||
|
||||
async def test_sync_with_encryption_and_password_change(self):
|
||||
daemon, daemon2 = self.daemon, self.daemon2
|
||||
wallet, wallet2 = daemon.wallet_manager.default_wallet, daemon2.wallet_manager.default_wallet
|
||||
@@ -391,8 +398,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
|
||||
# check new password is active
|
||||
daemon.jsonrpc_wallet_lock()
|
||||
self.assertFalse(daemon.jsonrpc_wallet_unlock('password'))
|
||||
self.assertTrue(daemon.jsonrpc_wallet_unlock('password2'))
|
||||
self.assertFalse(await daemon.jsonrpc_wallet_unlock('password'))
|
||||
self.assertTrue(await daemon.jsonrpc_wallet_unlock('password2'))
|
||||
|
||||
# propagate disk encryption to daemon2
|
||||
data = await daemon.jsonrpc_sync_apply('password3')
|
||||
@@ -408,4 +415,4 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
self.assertWalletEncrypted(wallet2.storage.path, True)
|
||||
|
||||
daemon2.jsonrpc_wallet_lock()
|
||||
self.assertTrue(daemon2.jsonrpc_wallet_unlock('password3'))
|
||||
self.assertTrue(await daemon2.jsonrpc_wallet_unlock('password3'))
|
||||
@@ -405,6 +405,17 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
|
||||
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
|
||||
|
||||
async def test_no_source_and_valid_channel_signature_and_media_type(self):
|
||||
await self.channel_create('@spam2', '1.0')
|
||||
await self.stream_create('barrrrrr', '1.0', channel_name='@spam2', file_path=self.video_file_name)
|
||||
paradox_no_source_claims = await self.claim_search(has_no_source=True, valid_channel_signature=True,
|
||||
media_type="video/mp4")
|
||||
mp4_claims = await self.claim_search(media_type="video/mp4")
|
||||
no_source_claims = await self.claim_search(has_no_source=True, valid_channel_signature=True)
|
||||
self.assertEqual(0, len(paradox_no_source_claims))
|
||||
self.assertEqual(1, len(no_source_claims))
|
||||
self.assertEqual(1, len(mp4_claims))
|
||||
|
||||
async def test_limit_claims_per_channel(self):
|
||||
match = self.assertFindsClaims
|
||||
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
|
||||
@@ -1499,6 +1510,11 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
|
||||
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
|
||||
|
||||
await self.stream_update(repost_id, bid='0.42')
|
||||
searched_repost = (await self.claim_search(claim_id=repost_id))[0]
|
||||
self.assertEqual(searched_repost['amount'], '0.42')
|
||||
self.assertEqual(searched_repost['signing_channel']['claim_id'], spam_claim_id)
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
await self.channel_create('@some_channel', '0.1')
|
||||
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
|
||||
|
||||
@@ -107,18 +107,12 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
await self.stream_create('foo', '0.01')
|
||||
stream = (await self.daemon.jsonrpc_file_list())["items"][0]
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash})
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True))
|
||||
# announces on download
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
stream = await self.daemon.jsonrpc_get('foo')
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash})
|
||||
|
||||
async def _purge_file(self, claim_name, full_path):
|
||||
self.assertTrue(
|
||||
@@ -617,6 +611,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
|
||||
|
||||
background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager)
|
||||
self.daemon.conf.network_storage_limit = 32
|
||||
await self.clear()
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['total_used_mb'])
|
||||
@@ -625,6 +620,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
|
||||
self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
|
||||
self.daemon.conf.network_storage_limit = 48
|
||||
await background_downloader.download_blobs(content2)
|
||||
await self.assertBlobs(content1, content2)
|
||||
await self.blob_clean()
|
||||
@@ -654,3 +650,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
self.daemon.conf.blob_storage_limit = 1
|
||||
await self.blob_clean()
|
||||
await self.assertBlobs(content1, no_files=False)
|
||||
|
||||
self.daemon.conf.network_storage_limit = 0
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
|
||||
@@ -3,6 +3,17 @@ from lbry.testcase import CommandTestCase
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_txo_dust_prevention(self):
|
||||
address = await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
tx = await self.account_send('9.9997758', address)
|
||||
# dust prevention threshold not reached, small txo created
|
||||
self.assertEqual(2, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][1]['amount'], '0.0001002')
|
||||
tx = await self.account_send('9.999706', address)
|
||||
# dust prevention prevented dust
|
||||
self.assertEqual(1, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][0]['amount'], '9.999706')
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
|
||||
@@ -1,60 +1,68 @@
|
||||
import contextlib
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
import asyncio
|
||||
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.conf import Config
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from unittest import skip
|
||||
|
||||
|
||||
class TestBlobAnnouncer(AsyncioTestCase):
|
||||
async def setup_node(self, peer_addresses, address, node_id):
|
||||
self.nodes: typing.Dict[int, Node] = {}
|
||||
self.advance = dht_mocks.get_time_accelerator(self.loop, self.loop.time())
|
||||
self.advance = dht_mocks.get_time_accelerator(self.loop)
|
||||
self.instant_advance = dht_mocks.get_time_accelerator(self.loop)
|
||||
self.conf = Config()
|
||||
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
|
||||
await self.storage.open()
|
||||
self.peer_manager = PeerManager(self.loop)
|
||||
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
|
||||
await self.node.start_listening(address)
|
||||
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
|
||||
for node_id, address in peer_addresses:
|
||||
await self.add_peer(node_id, address)
|
||||
await asyncio.gather(*[self.add_peer(node_id, address) for node_id, address in peer_addresses])
|
||||
for first_peer in self.nodes.values():
|
||||
for second_peer in self.nodes.values():
|
||||
if first_peer == second_peer:
|
||||
continue
|
||||
self.add_peer_to_routing_table(first_peer, second_peer)
|
||||
self.add_peer_to_routing_table(second_peer, first_peer)
|
||||
await self.advance(0.1) # just to make pings go through
|
||||
self.node.joined.set()
|
||||
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
|
||||
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
|
||||
await self.storage.open()
|
||||
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
|
||||
|
||||
async def add_peer(self, node_id, address, add_to_routing_table=True):
|
||||
#print('add', node_id.hex()[:8], address)
|
||||
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
|
||||
await n.start_listening(address)
|
||||
self.nodes.update({len(self.nodes): n})
|
||||
if add_to_routing_table:
|
||||
self.node.protocol.add_peer(
|
||||
make_kademlia_peer(
|
||||
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
|
||||
)
|
||||
self.add_peer_to_routing_table(self.node, n)
|
||||
|
||||
def add_peer_to_routing_table(self, adder, being_added):
|
||||
adder.protocol.add_peer(
|
||||
make_kademlia_peer(
|
||||
being_added.protocol.node_id, being_added.protocol.external_ip, being_added.protocol.udp_port
|
||||
)
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _test_network_context(self, peer_addresses=None):
|
||||
self.peer_addresses = peer_addresses or [
|
||||
(constants.generate_id(2), '1.2.3.2'),
|
||||
(constants.generate_id(3), '1.2.3.3'),
|
||||
(constants.generate_id(4), '1.2.3.4'),
|
||||
(constants.generate_id(5), '1.2.3.5'),
|
||||
(constants.generate_id(6), '1.2.3.6'),
|
||||
(constants.generate_id(7), '1.2.3.7'),
|
||||
(constants.generate_id(8), '1.2.3.8'),
|
||||
(constants.generate_id(9), '1.2.3.9'),
|
||||
async def _test_network_context(self, peer_count=200):
|
||||
self.peer_addresses = [
|
||||
(constants.generate_id(i), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(1, peer_count + 1)
|
||||
]
|
||||
try:
|
||||
with dht_mocks.mock_network_loop(self.loop):
|
||||
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1))
|
||||
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1000))
|
||||
yield
|
||||
finally:
|
||||
self.blob_announcer.stop()
|
||||
@@ -73,43 +81,58 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
)
|
||||
)
|
||||
await peer.ping()
|
||||
return peer
|
||||
return last_node
|
||||
|
||||
@skip("Something from a previous test is leaking into this test and causing it to fail intermittently")
|
||||
async def test_announce_blobs(self):
|
||||
blob1 = binascii.hexlify(b'1' * 48).decode()
|
||||
blob2 = binascii.hexlify(b'2' * 48).decode()
|
||||
|
||||
async with self._test_network_context():
|
||||
await self.storage.add_blobs((blob1, 1024), (blob2, 1024), finished=True)
|
||||
await self.storage.db.execute(
|
||||
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
|
||||
(blob1, blob2)
|
||||
)
|
||||
async with self._test_network_context(peer_count=100):
|
||||
await self.storage.add_blobs((blob1, 1024, 0, True), (blob2, 1024, 0, True), finished=True)
|
||||
await self.storage.add_blobs(
|
||||
*((constants.generate_id(value).hex(), 1024, 0, True) for value in range(1000, 1090)),
|
||||
finished=True)
|
||||
await self.storage.db.execute("update blob set next_announce_time=0, should_announce=1")
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(2, len(to_announce))
|
||||
self.blob_announcer.start(batch_size=1) # so it covers batching logic
|
||||
self.assertEqual(92, len(to_announce))
|
||||
self.blob_announcer.start(batch_size=10) # so it covers batching logic
|
||||
# takes 60 seconds to start, but we advance 120 to ensure it processed all batches
|
||||
await self.advance(60.0 * 2)
|
||||
ongoing_announcements = asyncio.ensure_future(self.blob_announcer.wait())
|
||||
await self.instant_advance(60.0)
|
||||
await ongoing_announcements
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(0, len(to_announce))
|
||||
self.blob_announcer.stop()
|
||||
|
||||
# as routing table pollution will cause some peers to be hard to reach, we add a tolerance for CI
|
||||
tolerance = 0.8 # at least 80% of the announcements are within the top K
|
||||
for blob in await self.storage.get_all_blob_hashes():
|
||||
distance = Distance(bytes.fromhex(blob))
|
||||
candidates = list(self.nodes.values())
|
||||
candidates.sort(key=lambda sorting_node: distance(sorting_node.protocol.node_id))
|
||||
has_it = 0
|
||||
for index, node in enumerate(candidates[:constants.K], start=1):
|
||||
if node.protocol.data_store.get_peers_for_blob(bytes.fromhex(blob)):
|
||||
has_it += 1
|
||||
else:
|
||||
logging.warning("blob %s wasnt found between the best K (%s)", blob[:8], node.protocol.node_id.hex()[:8])
|
||||
self.assertGreaterEqual(has_it, int(tolerance * constants.K))
|
||||
|
||||
|
||||
# test that we can route from a poorly connected peer all the way to the announced blob
|
||||
|
||||
await self.chain_peer(constants.generate_id(10), '1.2.3.10')
|
||||
await self.chain_peer(constants.generate_id(11), '1.2.3.11')
|
||||
await self.chain_peer(constants.generate_id(12), '1.2.3.12')
|
||||
await self.chain_peer(constants.generate_id(13), '1.2.3.13')
|
||||
await self.chain_peer(constants.generate_id(14), '1.2.3.14')
|
||||
await self.advance(61.0)
|
||||
current = len(self.nodes)
|
||||
await self.chain_peer(constants.generate_id(current + 1), '1.2.3.10')
|
||||
await self.chain_peer(constants.generate_id(current + 2), '1.2.3.11')
|
||||
await self.chain_peer(constants.generate_id(current + 3), '1.2.3.12')
|
||||
await self.chain_peer(constants.generate_id(current + 4), '1.2.3.13')
|
||||
last = await self.chain_peer(constants.generate_id(current + 5), '1.2.3.14')
|
||||
|
||||
last = self.nodes[len(self.nodes) - 1]
|
||||
search_q, peer_q = asyncio.Queue(loop=self.loop), asyncio.Queue(loop=self.loop)
|
||||
search_q.put_nowait(blob1)
|
||||
|
||||
_, task = last.accumulate_peers(search_q, peer_q)
|
||||
found_peers = await peer_q.get()
|
||||
found_peers = await asyncio.wait_for(peer_q.get(), 1.0)
|
||||
task.cancel()
|
||||
|
||||
self.assertEqual(1, len(found_peers))
|
||||
@@ -119,21 +142,13 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
|
||||
async def test_popular_blob(self):
|
||||
peer_count = 150
|
||||
addresses = [
|
||||
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(peer_count)
|
||||
]
|
||||
blob_hash = b'1' * 48
|
||||
blob_hash = constants.generate_id(99999)
|
||||
|
||||
async with self._test_network_context(peer_addresses=addresses):
|
||||
async with self._test_network_context(peer_count=peer_count):
|
||||
total_seen = set()
|
||||
announced_to = self.nodes[0]
|
||||
for i in range(1, peer_count):
|
||||
node = self.nodes[i]
|
||||
kad_peer = make_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
|
||||
)
|
||||
await announced_to.protocol._add_peer(kad_peer)
|
||||
announced_to = self.nodes.pop(0)
|
||||
for i, node in enumerate(self.nodes.values()):
|
||||
self.add_peer_to_routing_table(announced_to, node)
|
||||
peer = node.protocol.get_rpc_peer(
|
||||
make_kademlia_peer(
|
||||
announced_to.protocol.node_id,
|
||||
@@ -144,15 +159,15 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
response = await peer.store(blob_hash)
|
||||
self.assertEqual(response, b'OK')
|
||||
peers_for_blob = await peer.find_value(blob_hash, 0)
|
||||
if i == 1:
|
||||
if i == 0:
|
||||
self.assertNotIn(blob_hash, peers_for_blob)
|
||||
self.assertEqual(peers_for_blob[b'p'], 0)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), min(i - 1, constants.K))
|
||||
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i)
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), min(i, constants.K))
|
||||
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i + 1)
|
||||
if i - 1 > constants.K:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), constants.K)
|
||||
self.assertEqual(peers_for_blob[b'p'], ((i - 1) // (constants.K + 1)) + 1)
|
||||
self.assertEqual(peers_for_blob[b'p'], (i // (constants.K + 1)) + 1)
|
||||
seen = set(peers_for_blob[blob_hash])
|
||||
self.assertEqual(len(seen), constants.K)
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
|
||||
@@ -167,5 +182,5 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
seen.intersection_update(page_x_set)
|
||||
total_seen.update(page_x_set)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), i - 1)
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), 8) # we always add 8 on first page
|
||||
self.assertEqual(len(total_seen), peer_count - 2)
|
||||
@@ -12,7 +12,6 @@ from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
|
||||
class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
TIMEOUT = None # not supported as it advances time
|
||||
async def test_ping_queue_discover(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.set_debug(False)
|
||||
@@ -29,7 +28,7 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
(constants.generate_id(9), '1.2.3.9'),
|
||||
]
|
||||
with dht_mocks.mock_network_loop(loop):
|
||||
advance = dht_mocks.get_time_accelerator(loop, loop.time())
|
||||
advance = dht_mocks.get_time_accelerator(loop)
|
||||
# start the nodes
|
||||
nodes: typing.Dict[int, Node] = {
|
||||
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address)
|
||||
@@ -93,7 +92,6 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
|
||||
|
||||
class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
TIMEOUT = None # not supported as it advances time
|
||||
@unittest.SkipTest
|
||||
async def test_losing_connection(self):
|
||||
async def wait_for(check_ok, insist, timeout=20):
|
||||
@@ -131,7 +129,7 @@ class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
await asyncio.gather(*[n.joined.wait() for n in nodes])
|
||||
|
||||
node = nodes[-1]
|
||||
advance = dht_mocks.get_time_accelerator(loop, loop.time())
|
||||
advance = dht_mocks.get_time_accelerator(loop)
|
||||
await advance(500)
|
||||
|
||||
# Join the network, assert that at least the known peers are in RT
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import unittest
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
|
||||
class MediaTypeTests(unittest.TestCase):
|
||||
def test_guess_media_type_from_path_only(self):
|
||||
kind = guess_media_type('/tmp/test.mkv')
|
||||
self.assertEqual(kind, ('video/x-matroska', 'video'))
|
||||
|
||||
def test_defaults_for_no_extension(self):
|
||||
kind = guess_media_type('/tmp/test')
|
||||
self.assertEqual(kind, ('application/octet-stream', 'binary'))
|
||||
|
||||
def test_defaults_for_unknown_extension(self):
|
||||
kind = guess_media_type('/tmp/test.unk')
|
||||
self.assertEqual(kind, ('application/x-ext-unk', 'binary'))
|
||||
|
||||
def test_spoofed_unknown(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_unknown.txt')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_lz4 = bytearray([0x04,0x22,0x4d,0x18])
|
||||
fd.write(bytes_lz4)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/x-ext-lz4', 'binary'))
|
||||
|
||||
def test_spoofed_known(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.avi')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/zip', 'binary'))
|
||||
|
||||
def test_spoofed_synonym(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.cbz')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/vnd.comicbook+zip', 'document'))
|
||||
@@ -97,7 +97,7 @@ class TestReflector(AsyncioTestCase):
|
||||
async def test_announces(self):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
|
||||
self.assertNotIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob set to announce")
|
||||
|
||||
async def test_result_from_disconnect_mid_sd_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import unittest
|
||||
from lbry import utils
|
||||
|
||||
|
||||
class UtilsTestCase(unittest.TestCase):
|
||||
|
||||
def test_get_colliding_prefix_bits(self):
|
||||
self.assertEqual(
|
||||
0, utils.get_colliding_prefix_bits(0xffffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
Reference in new issue
Block a user