Compare commits

...
12 Commits
Author SHA1 Message Date
Jack Robison 5a39e53ed2 add outbound and incoming connection counts to connection status 2020-04-22 13:38:38 -04:00
Jack Robison 5e8b7e03af add debug_in_flight api 2020-04-22 13:38:10 -04:00
Lex Berezhny 51f573f1ea v0.70.0 2020-04-20 13:37:40 -04:00
Jack Robison c8b7cd8862 Merge pull request #2920 from lbryio/uploading-to-reflector-status
add `uploading_to_reflector` to `file_list` results
2020-04-20 13:35:02 -04:00
Jack Robison 21c112d059 lbrycrd url 2020-04-20 12:16:31 -04:00
Jack Robison 9432e1b5b2 add uploading_to_reflector to file_list results 2020-04-20 11:57:09 -04:00
Lex Berezhny 084f0ebdab v0.69.1 2020-04-17 12:55:07 -04:00
Lex Berezhny 58640c1521 Merge pull request #2917 from lbryio/faster-repost-search-query
improve how claim search query handles reposts
2020-04-17 12:53:50 -04:00
Jack Robison 7ffdfd12f8 faster not tags 2020-04-17 10:47:01 -04:00
Jack Robison cb9a30f285 faster query 2020-04-17 10:47:01 -04:00
Lex Berezhny e48bef809f Merge pull request #2919 from lbryio/exclude_internal_transfers_fix
fix issue with `--exclude_internal_transfers` where it was filtering out sent payments
2020-04-16 20:26:37 -04:00
Lex Berezhny f5d7570102 fix issue with --exclude_internal_transfers where it was filtering out sent payments 2020-04-16 17:55:49 -04:00
9 changed files with 96 additions and 31 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.69.0"
__version__ = "0.70.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+6 -3
View File
@@ -42,8 +42,7 @@ class ConnectionManager:
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
self.incoming_connected.add(host_and_port)
def outgoing_connection_lost(self, host_and_port: str):
if self._running and host_and_port in self.outgoing_connected:
@@ -62,7 +61,9 @@ class ConnectionManager:
'total_sent': 0,
'total_received': 0,
'max_incoming_mbs': 0.0,
'max_outgoing_mbs': 0.0
'max_outgoing_mbs': 0.0,
'outbound_connections': 0,
'incoming_connections': 0
}
while True:
@@ -87,6 +88,8 @@ class ConnectionManager:
self._max_outgoing_mbs = max(self._max_outgoing_mbs, self._status['total_outgoing_mbs'])
self._status['max_incoming_mbs'] = self._max_incoming_mbs
self._status['max_outgoing_mbs'] = self._max_outgoing_mbs
self._status['outbound_connections'] = len(self.outgoing_connected)
self._status['incoming_connections'] = len(self.incoming_connected)
def stop(self):
if self._task:
+28 -2
View File
@@ -331,6 +331,8 @@ class Daemon(metaclass=JSONRPCServerType):
self.need_connection_status_refresh = asyncio.Event()
self._connection_status_task: Optional[asyncio.Task] = None
self._received_requests_counter = 0
self._in_flight_requests: typing.Dict[int, typing.Tuple[str, float, typing.Tuple, typing.Dict]] = {}
@property
def dht_node(self) -> typing.Optional['Node']:
@@ -363,7 +365,7 @@ class Daemon(metaclass=JSONRPCServerType):
@classmethod
def get_api_definitions(cls):
prefix = 'jsonrpc_'
not_grouped = ['routing_table_get', 'ffmpeg_find']
not_grouped = ['routing_table_get', 'ffmpeg_find', 'debug_in_flight']
api = {
'groups': {
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
@@ -615,6 +617,8 @@ class Daemon(metaclass=JSONRPCServerType):
return await self.stream_manager.stream_partial_content(request, sd_hash)
async def _process_rpc_call(self, data):
request_id = int(self._received_requests_counter)
self._received_requests_counter += 1
args = data.get('params', {})
try:
@@ -663,7 +667,7 @@ class Daemon(metaclass=JSONRPCServerType):
JSONRPCError.CODE_INVALID_PARAMS,
params_error_message,
)
self._in_flight_requests[request_id] = (function_name, time.perf_counter(), _args, _kwargs)
try:
result = method(self, *_args, **_kwargs)
if asyncio.iscoroutine(result):
@@ -677,6 +681,8 @@ class Daemon(metaclass=JSONRPCServerType):
return JSONRPCError.create_command_exception(
command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc()
)
finally:
del self._in_flight_requests[request_id]
def _verify_method_is_callable(self, function_path):
if function_path not in self.callable_methods:
@@ -895,6 +901,7 @@ class Daemon(metaclass=JSONRPCServerType):
ffmpeg_status = await self._video_file_analyzer.status()
running_components = self.component_manager.get_components_status()
response = {
'requests_in_flight': len(self._in_flight_requests),
'installation_id': self.installation_id,
'is_running': all(running_components.values()),
'skipped_components': self.component_manager.skip_components,
@@ -911,6 +918,25 @@ class Daemon(metaclass=JSONRPCServerType):
response[component.component_name] = status
return response
def jsonrpc_debug_in_flight(self):
"""
Debug currently in flight api requests
Usage:
debug_in_flight
Options:
None
Returns:
(dict) Dictionary of in flight requests
"""
now = time.perf_counter()
return {
f"{method}-{req_id}": now - started_ts
for req_id, (method, started_ts, _, _) in self._in_flight_requests.items()
}
def jsonrpc_version(self): # pylint: disable=no-self-use
"""
Get lbrynet API server version information
+4 -2
View File
@@ -109,7 +109,8 @@ def encode_file_doc():
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100'
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
}
@@ -309,7 +310,8 @@ class JSONResponseEncoder(JSONEncoder):
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
}
def encode_claim(self, claim):
+5 -1
View File
@@ -74,7 +74,8 @@ class ManagedStream:
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt'
'finished_write_attempt',
'uploading_to_reflector'
]
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
@@ -103,6 +104,7 @@ class ManagedStream:
self.fully_reflected = asyncio.Event(loop=self.loop)
self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
@@ -432,6 +434,7 @@ class ManagedStream:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
@@ -458,6 +461,7 @@ class ManagedStream:
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+2 -1
View File
@@ -733,8 +733,9 @@ class Database(SQLiteMixin):
include_is_my_input = True
constraints['exclude_internal_payments__or'] = {
'txo.txo_type__not': TXO_TYPES['other'],
'txo.address__not_in': my_addresses,
'txi.address__is_null': True,
'txi.address__not_in': my_addresses
'txi.address__not_in': my_addresses,
}
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
if is_spent:
+26 -10
View File
@@ -545,11 +545,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
@@ -596,11 +604,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
"""
else:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
@@ -503,22 +503,35 @@ class TransactionOutputCommands(ClaimTestCase):
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create('@kept-channel')
await self.channel_create('@sent-channel', claim_address=address2)
await self.wallet_send('2.9', address2)
# all txos on second wallet
received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True)
received_payment, received_channel = await self.txo_list(
wallet_id=wallet2.id, is_my_input_or_output=True)
self.assertEqual('1.0', received_channel['amount'])
self.assertFalse(received_channel['is_my_input'])
self.assertTrue(received_channel['is_my_output'])
self.assertFalse(received_channel['is_internal_transfer'])
self.assertEqual('2.9', received_payment['amount'])
self.assertFalse(received_payment['is_my_input'])
self.assertTrue(received_payment['is_my_output'])
self.assertFalse(received_payment['is_internal_transfer'])
# all txos on default wallet
r = await self.txo_list(is_my_input_or_output=True)
self.assertEqual(
['1.0', '7.947786', '1.0', '8.973893', '10.0'],
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
[t['amount'] for t in r]
)
sent_channel, change2, kept_channel, change1, initial_funds = r
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_payment['is_my_input'])
self.assertFalse(sent_payment['is_my_output'])
self.assertFalse(sent_payment['is_internal_transfer'])
self.assertTrue(change3['is_my_input'])
self.assertTrue(change3['is_my_output'])
self.assertTrue(change3['is_internal_transfer'])
self.assertTrue(sent_channel['is_my_input'])
self.assertFalse(sent_channel['is_my_output'])
@@ -540,31 +553,31 @@ class TransactionOutputCommands(ClaimTestCase):
# my stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
self.assertEqual([sent_channel, kept_channel, initial_funds], r)
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
# my unspent stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
self.assertEqual([sent_channel, kept_channel], r)
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
# only "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
self.assertEqual([change2, change1], r)
self.assertEqual([change3, change2, change1], r)
# only unspent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
self.assertEqual([change2], r)
self.assertEqual([change3], r)
# only spent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
self.assertEqual([change1], r)
self.assertEqual([change2, change1], r)
# all my unspent stuff
r = await self.txo_list(is_my_output=True, is_not_spent=True)
self.assertEqual([change2, kept_channel], r)
self.assertEqual([change3, kept_channel], r)
# stuff i sent
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_channel], r)
self.assertEqual([sent_payment, sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)