Compare commits

...
13 Commits
Author SHA1 Message Date
Victor Shyba 92258a2764 add concurrent_hub_requests conf 2021-05-04 22:38:59 -03:00
Victor Shyba ac88ada50e move request semaphore to session and apply to all requests 2021-05-04 22:38:46 -03:00
Victor Shyba e7fdf2aeb3 add hub_timeout and propagate it to network code 2021-05-04 22:38:39 -03:00
Lex Berezhny 99f56f5d22 v0.96.0 2021-04-28 15:26:58 -04:00
Jack Robison ad6281090d Merge pull request #3275 from lbryio/search_caching_issues
add caching to "search ahead" code and invalidate short_url cache on every block
2021-04-28 14:14:29 -04:00
Victor Shyba f0d334d3e2 refactor from review 2021-04-28 13:28:38 -03:00
Victor Shyba 5f829b048f use separator to avoid cache key conflicts 2021-04-27 22:57:04 -03:00
Victor Shyba 1a961e66ff invalidate short_id cache on new block 2021-04-27 22:57:04 -03:00
Victor Shyba fdb0e22656 cache search_ahead 2021-04-27 22:57:04 -03:00
Jack Robison 132ee1915f Merge pull request #3283 from lbryio/fix_multiprocessing_db
fix multiprocessing support on client db
2021-04-27 16:46:28 -04:00
Victor Shyba 44bf4f3c8f fix if statement from always evaluating a string 2021-04-27 17:10:04 -03:00
Alex Grintsvayg 6237767d5a Merge branch 'make-test'
* make-test:
  run tests using make
2021-04-23 15:26:26 -04:00
Alex Grintsvayg dec9d96417 run tests using make 2021-04-23 15:25:40 -04:00
10 changed files with 103 additions and 54 deletions

No files matched your search

+2 -3
View File
@@ -22,10 +22,9 @@ jobs:
with:
python-version: '3.7'
- run: make install tools
- working-directory: lbry
env:
- env:
HOME: /tmp
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
run: make test-unit-coverage
tests-integration:
name: "tests / integration"
+10 -2
View File
@@ -1,4 +1,4 @@
.PHONY: install tools lint test idea
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
install:
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
@@ -16,7 +16,15 @@ lint:
pylint --rcfile=setup.cfg lbry
#mypy --ignore-missing-imports lbry
test:
test: test-unit test-integration
test-unit:
python -m unittest discover tests.unit
test-unit-coverage:
coverage run -p --source=lbry -m unittest discover -vv tests.unit
test-integration:
tox
idea:
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.95.0"
__version__ = "0.96.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+2
View File
@@ -544,6 +544,7 @@ class Config(CLIConfig):
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
hub_timeout = Float("Timeout when making a hub request", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
@@ -565,6 +566,7 @@ class Config(CLIConfig):
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
+1 -1
View File
@@ -18,7 +18,7 @@ from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
if platform.system() == 'Windows' or ({'ANDROID_ARGUMENT', 'KIVY_BUILD'} & os.environ.keys()):
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
else:
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
+4
View File
@@ -182,7 +182,9 @@ class WalletManager:
ledger_config = {
'auto_connect': True,
'hub_timeout': config.hub_timeout,
'default_servers': config.lbryum_servers,
'concurrent_hub_requests': config.concurrent_hub_requests,
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
@@ -226,6 +228,8 @@ class WalletManager:
self.ledger.config = {
'auto_connect': True,
'default_servers': self.config.lbryum_servers,
'hub_timeout': self.config.hub_timeout,
'concurrent_hub_requests': self.config.concurrent_hub_requests,
'data_path': self.config.wallet_dir,
}
await self.ledger.stop()
+21 -18
View File
@@ -19,7 +19,7 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
@@ -29,7 +29,11 @@ class ClientSession(BaseClientSession):
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._concurrency = asyncio.Semaphore(concurrency)
@property
def concurrency(self):
return self._concurrency._value
@property
def available(self):
@@ -55,9 +59,9 @@ class ClientSession(BaseClientSession):
return result
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
try:
await self._concurrency.acquire()
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
request = asyncio.ensure_future(super().send_request(method, args))
@@ -91,7 +95,7 @@ class ClientSession(BaseClientSession):
# self.synchronous_close()
raise
finally:
self.pending_amount -= 1
self._concurrency.release()
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
@@ -154,7 +158,6 @@ class Network:
# self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
self._concurrency = asyncio.Semaphore(16)
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
@@ -263,7 +266,8 @@ class Network:
async def connect_to_fastest(self) -> Optional[ClientSession]:
fastest_spvs = await self.get_n_fastest_spvs()
for (host, port) in fastest_spvs:
client = ClientSession(network=self, server=(host, port))
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
concurrency=self.config['concurrent_hub_requests'])
try:
await client.create_connection()
log.warning("Connected to spv server %s:%i", host, port)
@@ -344,18 +348,17 @@ class Network:
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def retriable_call(self, function, *args, **kwargs):
async with self._concurrency:
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("connection error")
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("connection error")
raise asyncio.CancelledError() # if we got here, we are shutting down
+3 -1
View File
@@ -131,7 +131,9 @@ class WalletNode:
self.ledger_class.get_id(): {
'api_port': self.port,
'default_servers': [(spv_node.hostname, spv_node.port)],
'data_path': self.data_path
'data_path': self.data_path,
'hub_timeout': 30,
'concurrent_hub_requests': 32,
}
},
'wallets': [wallet_file_name]
+45 -28
View File
@@ -1,7 +1,7 @@
import asyncio
import struct
from binascii import unhexlify
from collections import Counter
from collections import Counter, deque
from decimal import Decimal
from operator import itemgetter
from typing import Optional, List, Iterable, Union
@@ -42,7 +42,7 @@ class SearchIndex:
self.index = index_prefix + 'claims'
self.logger = class_logger(__name__, self.__class__.__name__)
self.claim_cache = LRUCache(2 ** 15)
self.short_id_cache = LRUCache(2 ** 17) # never invalidated, since short ids are forever
self.short_id_cache = LRUCache(2 ** 17)
self.search_cache = LRUCache(2 ** 17)
self.resolution_cache = LRUCache(2 ** 17)
self._elastic_host = elastic_host
@@ -134,6 +134,7 @@ class SearchIndex:
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
await self.sync_client.indices.refresh(self.index)
self.search_cache.clear()
self.short_id_cache.clear()
self.claim_cache.clear()
self.resolution_cache.clear()
@@ -198,7 +199,7 @@ class SearchIndex:
self.claim_cache.set(result['claim_id'], result)
async def full_id_from_short_id(self, name, short_id, channel_id=None):
key = (channel_id or '') + name + short_id
key = '#'.join((channel_id or '', name, short_id))
if key not in self.short_id_cache:
query = {'name': name, 'claim_id': short_id}
if channel_id:
@@ -233,40 +234,56 @@ class SearchIndex:
async def search_ahead(self, **kwargs):
# 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return
per_channel_per_page = kwargs.pop('limit_claims_per_channel')
limit = kwargs.pop('limit', 10)
page_size = kwargs.pop('limit', 10)
offset = kwargs.pop('offset', 0)
kwargs['limit'] = 1000
query = expand_query(**kwargs)
result = (await self.search_client.search(
cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache)
if cache_item.result is not None:
reordered_hits = cache_item.result
else:
async with cache_item.lock:
if cache_item.result:
reordered_hits = cache_item.result
else:
query = expand_query(**kwargs)
reordered_hits = await self.__search_ahead(query, page_size, per_channel_per_page)
cache_item.result = reordered_hits
return list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)])))
async def __search_ahead(self, query: dict, page_size: int, per_channel_per_page: int):
search_hits = deque((await self.search_client.search(
query, index=self.index, track_total_hits=False, _source_includes=['_id', 'channel_id']
))['hits']
to_inflate = []
channel_counter = Counter()
delayed = []
while result['hits'] or delayed:
if len(to_inflate) % limit == 0:
channel_counter.clear()
))['hits']['hits'])
reordered_hits = []
channel_counters = Counter()
next_page_hits_maybe_check_later = deque()
while search_hits or next_page_hits_maybe_check_later:
if reordered_hits and len(reordered_hits) % page_size == 0:
channel_counters.clear()
elif not reordered_hits:
pass
else:
break # means last page was incomplete and we are left with bad replacements
new_delayed = []
for id, chann in delayed:
if channel_counter[chann] < per_channel_per_page:
to_inflate.append((id, chann))
channel_counter[hit_channel_id] += 1
for _ in range(len(next_page_hits_maybe_check_later)):
claim_id, channel_id = next_page_hits_maybe_check_later.popleft()
if channel_counters[channel_id] < per_channel_per_page:
reordered_hits.append((claim_id, channel_id))
channel_counters[channel_id] += 1
else:
new_delayed.append((id, chann))
delayed = new_delayed
while result['hits']:
hit = result['hits'].pop(0)
next_page_hits_maybe_check_later.append((claim_id, channel_id))
while search_hits:
hit = search_hits.popleft()
hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id']
if channel_counter[hit_channel_id] < per_channel_per_page:
to_inflate.append((hit_id, hit_channel_id))
channel_counter[hit_channel_id] += 1
if len(to_inflate) % limit == 0:
if hit_channel_id is None:
reordered_hits.append((hit_id, hit_channel_id))
elif channel_counters[hit_channel_id] < per_channel_per_page:
reordered_hits.append((hit_id, hit_channel_id))
channel_counters[hit_channel_id] += 1
if len(reordered_hits) % page_size == 0:
break
else:
delayed.append((hit_id, hit_channel_id))
return list(await self.get_many(*(claim_id for claim_id, _ in to_inflate[offset:(offset + limit)])))
next_page_hits_maybe_check_later.append((hit_id, hit_channel_id))
return reordered_hits
async def resolve_url(self, raw_url):
if raw_url not in self.resolution_cache:
@@ -9,6 +9,7 @@ from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.rpc import RPCSession
from lbry.wallet.server.udp import StatusServer
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
from lbry.conf import Config
class NetworkTests(IntegrationTestCase):
@@ -138,6 +139,19 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_timeout_and_concurrency_propagated_from_config(self):
conf = Config()
self.assertEqual(self.ledger.network.client.timeout, 30)
self.assertEqual(self.ledger.network.client.concurrency, 32)
conf.hub_timeout = 123.0
conf.concurrent_hub_requests = 42
conf.lbryum_servers = self.ledger.config['default_servers']
conf.wallet_dir = self.ledger.config['data_path']
self.manager.config = conf
await self.manager.reset()
self.assertEqual(self.ledger.network.client.timeout, 123)
self.assertEqual(self.ledger.network.client.concurrency, 42)
# async def test_online_but_still_unavailable(self):
# # Edge case. See issue #2445 for context
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)