Compare commits

..
Author SHA1 Message Date
Lex Berezhny c1e54a3b0a StreamCommands 2021-04-28 15:28:16 -04:00
Lex Berezhny e8572f6f69 FindShortestID updated 2021-04-28 15:28:16 -04:00
Lex Berezhny b6fecad6ea finished switch from using hash # in URLs to colon : 2021-04-28 15:28:16 -04:00
11 changed files with 60 additions and 85 deletions

No files matched your search

-2
View File
@@ -544,7 +544,6 @@ class Config(CLIConfig):
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
hub_timeout = Float("Timeout when making a hub request", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
@@ -566,7 +565,6 @@ class Config(CLIConfig):
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
-4
View File
@@ -182,9 +182,7 @@ class WalletManager:
ledger_config = {
'auto_connect': True,
'hub_timeout': config.hub_timeout,
'default_servers': config.lbryum_servers,
'concurrent_hub_requests': config.concurrent_hub_requests,
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
@@ -228,8 +226,6 @@ class WalletManager:
self.ledger.config = {
'auto_connect': True,
'default_servers': self.config.lbryum_servers,
'hub_timeout': self.config.hub_timeout,
'concurrent_hub_requests': self.config.concurrent_hub_requests,
'data_path': self.config.wallet_dir,
}
await self.ledger.stop()
+18 -21
View File
@@ -19,7 +19,7 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
@@ -29,11 +29,7 @@ class ClientSession(BaseClientSession):
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self._concurrency = asyncio.Semaphore(concurrency)
@property
def concurrency(self):
return self._concurrency._value
self.pending_amount = 0
@property
def available(self):
@@ -59,9 +55,9 @@ class ClientSession(BaseClientSession):
return result
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
try:
await self._concurrency.acquire()
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
request = asyncio.ensure_future(super().send_request(method, args))
@@ -95,7 +91,7 @@ class ClientSession(BaseClientSession):
# self.synchronous_close()
raise
finally:
self._concurrency.release()
self.pending_amount -= 1
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
@@ -158,6 +154,7 @@ class Network:
# self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
self._concurrency = asyncio.Semaphore(16)
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
@@ -266,8 +263,7 @@ class Network:
async def connect_to_fastest(self) -> Optional[ClientSession]:
fastest_spvs = await self.get_n_fastest_spvs()
for (host, port) in fastest_spvs:
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
concurrency=self.config['concurrent_hub_requests'])
client = ClientSession(network=self, server=(host, port))
try:
await client.create_connection()
log.warning("Connected to spv server %s:%i", host, port)
@@ -348,17 +344,18 @@ class Network:
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def retriable_call(self, function, *args, **kwargs):
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("connection error")
async with self._concurrency:
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("connection error")
raise asyncio.CancelledError() # if we got here, we are shutting down
+1 -3
View File
@@ -131,9 +131,7 @@ class WalletNode:
self.ledger_class.get_id(): {
'api_port': self.port,
'default_servers': [(spv_node.hostname, spv_node.port)],
'data_path': self.data_path,
'hub_timeout': 30,
'concurrent_hub_requests': 32,
'data_path': self.data_path
}
},
'wallets': [wallet_file_name]
+1 -1
View File
@@ -15,7 +15,7 @@ class FindShortestID:
def finalize(self):
if self.short_id:
return '#'+self.short_id
return ':'+self.short_id
@classmethod
def factory(cls):
+4 -4
View File
@@ -47,8 +47,8 @@ class SQLDB:
expiration_height integer not null,
release_time integer not null,
short_url text not null, -- normalized#shortest-unique-claim_id
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
short_url text not null, -- normalized:shortest-unique-claim_id
canonical_url text, -- channel's-short_url/normalized:shortest-unique-claim_id-within-channel
title text,
author text,
@@ -452,7 +452,7 @@ class SQLDB:
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
:claim_name||COALESCE(
(SELECT shortest_id(claim_id, :claim_id) FROM claim WHERE normalized = :normalized),
'#'||substr(:claim_id, 1, 1)
':'||substr(:claim_id, 1, 1)
)
)""", claims)
@@ -665,7 +665,7 @@ class SQLDB:
WHERE other_claim.signature_valid = 1 AND
other_claim.channel_hash = :channel_hash AND
other_claim.normalized = claim.normalized),
'#'||substr(claim_id, 1, 1)
':'||substr(claim_id, 1, 1)
)
END
WHERE claim_hash=:claim_hash;
+1 -1
View File
@@ -384,7 +384,7 @@ class Output(InputOutput):
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
return f"lbry://{self.claim_name}#{self.claim_id}"
return f"lbry://{self.claim_name}:{self.claim_id}"
raise ValueError('No claim associated.')
@property
@@ -160,11 +160,11 @@ class ClaimSearchCommand(ClaimTestCase):
# three streams in channel, zero streams in abandoned channel
claims = [three, two, signed]
await self.assertFindsClaims(claims, channel_ids=[self.channel_id])
await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}")
await self.assertFindsClaims(claims, channel=f"@abc:{self.channel_id}")
await self.assertFindsClaims([], channel=f"@inexistent")
await self.assertFindsClaims([three, two, signed2, signed], channel_ids=[channel_id2, self.channel_id])
await self.channel_abandon(claim_id=self.channel_id)
await self.assertFindsClaims([], channel=f"@abc#{self.channel_id}", valid_channel_signature=True)
await self.assertFindsClaims([], channel=f"@abc:{self.channel_id}", valid_channel_signature=True)
await self.assertFindsClaims([], channel_ids=[self.channel_id], valid_channel_signature=True)
await self.assertFindsClaims([signed2], channel_ids=[channel_id2], valid_channel_signature=True)
# pass `invalid_channel_signature=False` to catch a bug in argument processing
@@ -1430,7 +1430,7 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering:'))
# same search, but details omitted by 'no_totals'
last_result = result
@@ -1444,7 +1444,7 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(1, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering:'))
# same search, but details omitted by 'no_totals'
last_result = result
@@ -1474,7 +1474,7 @@ class StreamCommands(ClaimTestCase):
error = (await self.resolve('lbry://@some_channel/bad_content'))['error']
self.assertEqual(error['name'], 'BLOCKED')
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking:'))
# a filtered/blocked channel impacts all content inside it
bad_channel_id = self.get_claim_id(
@@ -1502,7 +1502,7 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(3, filtered['total'])
self.assertEqual(1, len(filtered['channels']))
self.assertEqual(3, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering:'))
# same search, but details omitted by 'no_totals'
last_result = result
@@ -9,7 +9,6 @@ from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.rpc import RPCSession
from lbry.wallet.server.udp import StatusServer
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
from lbry.conf import Config
class NetworkTests(IntegrationTestCase):
@@ -139,19 +138,6 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_timeout_and_concurrency_propagated_from_config(self):
conf = Config()
self.assertEqual(self.ledger.network.client.timeout, 30)
self.assertEqual(self.ledger.network.client.concurrency, 32)
conf.hub_timeout = 123.0
conf.concurrent_hub_requests = 42
conf.lbryum_servers = self.ledger.config['default_servers']
conf.wallet_dir = self.ledger.config['data_path']
self.manager.config = conf
await self.manager.reset()
self.assertEqual(self.ledger.network.client.timeout, 123)
self.assertEqual(self.ledger.network.client.concurrency, 42)
# async def test_online_but_still_unavailable(self):
# # Edge case. See issue #2445 for context
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
@@ -132,8 +132,8 @@ class ResolveCommand(BaseResolveTestCase):
channel_id = self.get_claim_id(
await self.channel_create('@abc', '1.1', allow_duplicate_name=True))
await self.assertResolvesToClaimId(f'@abc', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id)
await self.assertResolvesToClaimId(f'@abc:{channel_id[:10]}', channel_id)
await self.assertResolvesToClaimId(f'@abc:{channel_id}', channel_id)
channel = (await self.claim_search(claim_id=channel_id))[0]
await self.assertResolvesToClaimId(channel['short_url'], channel_id)
await self.assertResolvesToClaimId(channel['canonical_url'], channel_id)
@@ -182,7 +182,7 @@ class ResolveCommand(BaseResolveTestCase):
response = await self.resolve('lbry://on-channel-claim')
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
direct_uri = 'lbry://on-channel-claim:' + orphan_claim_id
response = await self.resolve(direct_uri)
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
+26 -26
View File
@@ -415,8 +415,8 @@ class TestClaimtrie(TestSQLDB):
advance(1, [tx_chan_a])
advance(2, [tx_chan_ab])
(r_ab, r_a) = search(order_by=['creation_height'], limit=2)
self.assertEqual("@foo#a", r_a['short_url'])
self.assertEqual("@foo#ab", r_ab['short_url'])
self.assertEqual("@foo:a", r_a['short_url'])
self.assertEqual("@foo:ab", r_ab['short_url'])
self.assertIsNone(r_a['canonical_url'])
self.assertIsNone(r_ab['canonical_url'])
self.assertEqual(0, r_a['claims_in_channel'])
@@ -428,9 +428,9 @@ class TestClaimtrie(TestSQLDB):
advance(3, [tx_a])
advance(4, [tx_ab, tx_abc])
(r_abc, r_ab, r_a) = search(order_by=['creation_height', 'tx_position'], limit=3)
self.assertEqual("foo#a", r_a['short_url'])
self.assertEqual("foo#ab", r_ab['short_url'])
self.assertEqual("foo#abc", r_abc['short_url'])
self.assertEqual("foo:a", r_a['short_url'])
self.assertEqual("foo:ab", r_ab['short_url'])
self.assertEqual("foo:abc", r_abc['short_url'])
self.assertIsNone(r_a['canonical_url'])
self.assertIsNone(r_ab['canonical_url'])
self.assertIsNone(r_abc['canonical_url'])
@@ -442,17 +442,17 @@ class TestClaimtrie(TestSQLDB):
advance(6, [tx_a2])
advance(7, [tx_ab2])
(r_ab2, r_a2) = search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo#a/foo#a", r_a2['canonical_url'])
self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url'])
self.assertEqual(f"foo:{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo:{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo:a/foo:a", r_a2['canonical_url'])
self.assertEqual("@foo:a/foo:ab", r_ab2['canonical_url'])
self.assertEqual(2, search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# change channel public key, invaliding stream claim signatures
advance(8, [self.get_channel_update(txo_chan_a, COIN, key=b'a')])
(r_ab2, r_a2) = search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertEqual(f"foo:{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo:{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertIsNone(r_a2['canonical_url'])
self.assertIsNone(r_ab2['canonical_url'])
self.assertEqual(0, search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
@@ -461,18 +461,18 @@ class TestClaimtrie(TestSQLDB):
channel_update = self.get_channel_update(txo_chan_a, COIN, key=b'c')
advance(9, [channel_update])
(r_ab2, r_a2) = search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo#a/foo#a", r_a2['canonical_url'])
self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url'])
self.assertEqual(f"foo:{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo:{ab2_claim.claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo:a/foo:a", r_a2['canonical_url'])
self.assertEqual("@foo:a/foo:ab", r_ab2['canonical_url'])
self.assertEqual(2, search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
self.assertEqual(0, search(claim_id=txo_chan_ab.claim_id, limit=1)[0]['claims_in_channel'])
# change channel of stream
self.assertEqual("@foo#a/foo#ab", search(claim_id=ab2_claim.claim_id, limit=1)[0]['canonical_url'])
self.assertEqual("@foo:a/foo:ab", search(claim_id=ab2_claim.claim_id, limit=1)[0]['canonical_url'])
tx_ab2 = self.get_stream_update(tx_ab2, COIN, txo_chan_ab)
advance(10, [tx_ab2])
self.assertEqual("@foo#ab/foo#a", search(claim_id=ab2_claim.claim_id, limit=1)[0]['canonical_url'])
self.assertEqual("@foo:ab/foo:a", search(claim_id=ab2_claim.claim_id, limit=1)[0]['canonical_url'])
# TODO: currently there is a bug where stream leaving a channel does not update that channels claims count
self.assertEqual(2, search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# TODO: after bug is fixed remove test above and add test below
@@ -486,7 +486,7 @@ class TestClaimtrie(TestSQLDB):
# delete channel, invaliding stream claim signatures
advance(12, [self.get_abandon(channel_update)])
(r_a2,) = search(order_by=['creation_height'], limit=1)
self.assertEqual(f"foo#{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo:{a2_claim.claim_id[:2]}", r_a2['short_url'])
self.assertIsNone(r_a2['canonical_url'])
def test_resolve_issue_2448(self):
@@ -499,15 +499,15 @@ class TestClaimtrie(TestSQLDB):
advance(1, [tx_chan_a])
advance(2, [tx_chan_ab])
self.assertEqual(reader.resolve_url("@foo#a")['claim_hash'], txo_chan_a.claim_hash)
self.assertEqual(reader.resolve_url("@foo#ab")['claim_hash'], txo_chan_ab.claim_hash)
self.assertEqual(reader.resolve_url("@foo:a")['claim_hash'], txo_chan_a.claim_hash)
self.assertEqual(reader.resolve_url("@foo:ab")['claim_hash'], txo_chan_ab.claim_hash)
# update increase last height change of channel
advance(9, [self.get_channel_update(txo_chan_a, COIN, key=b'c')])
# make sure that activation_height is used instead of height (issue #2448)
self.assertEqual(reader.resolve_url("@foo#a")['claim_hash'], txo_chan_a.claim_hash)
self.assertEqual(reader.resolve_url("@foo#ab")['claim_hash'], txo_chan_ab.claim_hash)
self.assertEqual(reader.resolve_url("@foo:a")['claim_hash'], txo_chan_a.claim_hash)
self.assertEqual(reader.resolve_url("@foo:ab")['claim_hash'], txo_chan_ab.claim_hash)
def test_canonical_find_shortest_id(self):
new_hash = 'abcdef0123456789beef'
@@ -517,13 +517,13 @@ class TestClaimtrie(TestSQLDB):
other3 = 'abcdef0123456789bee1'
f = FindShortestID()
f.step(other0, new_hash)
self.assertEqual('#a', f.finalize())
self.assertEqual(':a', f.finalize())
f.step(other1, new_hash)
self.assertEqual('#abc', f.finalize())
self.assertEqual(':abc', f.finalize())
f.step(other2, new_hash)
self.assertEqual('#abcd', f.finalize())
self.assertEqual(':abcd', f.finalize())
f.step(other3, new_hash)
self.assertEqual('#abcdef0123456789beef', f.finalize())
self.assertEqual(':abcdef0123456789beef', f.finalize())
class TestTrending(TestSQLDB):