Compare commits

...
Author SHA1 Message Date
Victor Shyba 400c4e429b better iterative find, first part
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (claims) (push) Waiting to run
ci / tests / integration (claims_legacy_search) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
ci / tests / integration (takeovers) (push) Waiting to run
ci / tests / integration (takeovers_legacy_search) (push) Waiting to run
ci / tests / integration (transactions) (push) Waiting to run
2021-12-03 14:23:24 -03:00
Jack Robison 7531401623 keep touched_or_deleted records 2021-11-21 13:52:03 -05:00
Victor Shyba d19e07d661 add blob endpoint for listing announced blobs 2021-11-17 13:27:19 -03:00
Victor Shyba 751ff6e21f add /peers.csv to monitoring endpoint 2021-11-17 13:27:19 -03:00
Brendon J. Brewer 3f6fe995b8 Rename trending 2021-11-16 10:59:10 -05:00
7 changed files with 47 additions and 41 deletions

No files matched your search

+2 -2
View File
@@ -357,7 +357,7 @@
"curl": "curl -d'{\"method\": \"status\", \"params\": {}}' http://localhost:5279/",
"lbrynet": "lbrynet status",
"python": "requests.post(\"http://localhost:5279\", json={\"method\": \"status\", \"params\": {}}).json()",
"output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"blob_manager\": {\n \"connections\": {\n \"incoming_bps\": {},\n \"max_incoming_mbs\": 0.0,\n \"max_outgoing_mbs\": 0.0,\n \"outgoing_bps\": {},\n \"total_incoming_mbs\": 0.0,\n \"total_outgoing_mbs\": 0.0,\n \"total_received\": 0,\n \"total_sent\": 0\n },\n \"finished_blobs\": 0\n },\n \"disk_space\": {\n \"running\": true,\n \"space_used\": \"0\"\n },\n \"ffmpeg_status\": {\n \"analyze_audio_volume\": true,\n \"available\": true,\n \"which\": \"/usr/bin/ffmpeg\"\n },\n \"file_manager\": {\n \"managed_files\": 0\n },\n \"installation_id\": \"7Heg2TFdygzcjQVvZBHK6hnXNWYMT43F9jyh2PoRyzHK3BeHMfjbL5AVvz4F9i8PxL\",\n \"is_running\": true,\n \"skipped_components\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\",\n \"libtorrent_component\"\n ],\n \"startup_status\": {\n \"blob_manager\": true,\n \"database\": true,\n \"disk_space\": true,\n \"exchange_rate_manager\": true,\n \"file_manager\": true,\n \"wallet\": true,\n \"wallet_server_payments\": true\n },\n \"wallet\": {\n \"available_servers\": 1,\n \"best_blockhash\": \"02ae257ee08c2368fa243b2ff29b4824fd178efbfb59f58d0e36c9edc69323a1\",\n \"blocks\": 206,\n \"blocks_behind\": 0,\n \"connected\": \"localhost:50002\",\n \"connected_features\": {\n \"daily_fee\": \"0\",\n \"description\": \"\",\n \"donation_address\": \"\",\n \"genesis_hash\": \"6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556\",\n \"hash_function\": \"sha256\",\n \"hosts\": {},\n \"payment_address\": \"\",\n \"protocol_max\": \"0.199.0\",\n \"protocol_min\": \"0.54.0\",\n \"pruning\": null,\n \"server_version\": \"0.105.0\",\n \"trending_algorithm\": \"variable_decay\"\n },\n \"headers_synchronization_progress\": 100,\n \"known_servers\": 0,\n \"servers\": [\n {\n \"availability\": true,\n \"host\": \"localhost\",\n \"latency\": 0.002615762990899384,\n \"port\": 50002\n }\n ]\n },\n \"wallet_server_payments\": {\n \"max_fee\": \"0.0\",\n \"running\": false\n }\n }\n}"
"output": "{\n \"jsonrpc\": \"2.0\",\n \"result\": {\n \"blob_manager\": {\n \"connections\": {\n \"incoming_bps\": {},\n \"max_incoming_mbs\": 0.0,\n \"max_outgoing_mbs\": 0.0,\n \"outgoing_bps\": {},\n \"total_incoming_mbs\": 0.0,\n \"total_outgoing_mbs\": 0.0,\n \"total_received\": 0,\n \"total_sent\": 0\n },\n \"finished_blobs\": 0\n },\n \"disk_space\": {\n \"running\": true,\n \"space_used\": \"0\"\n },\n \"ffmpeg_status\": {\n \"analyze_audio_volume\": true,\n \"available\": true,\n \"which\": \"/usr/bin/ffmpeg\"\n },\n \"file_manager\": {\n \"managed_files\": 0\n },\n \"installation_id\": \"7Heg2TFdygzcjQVvZBHK6hnXNWYMT43F9jyh2PoRyzHK3BeHMfjbL5AVvz4F9i8PxL\",\n \"is_running\": true,\n \"skipped_components\": [\n \"dht\",\n \"upnp\",\n \"hash_announcer\",\n \"peer_protocol_server\",\n \"libtorrent_component\"\n ],\n \"startup_status\": {\n \"blob_manager\": true,\n \"database\": true,\n \"disk_space\": true,\n \"exchange_rate_manager\": true,\n \"file_manager\": true,\n \"wallet\": true,\n \"wallet_server_payments\": true\n },\n \"wallet\": {\n \"available_servers\": 1,\n \"best_blockhash\": \"02ae257ee08c2368fa243b2ff29b4824fd178efbfb59f58d0e36c9edc69323a1\",\n \"blocks\": 206,\n \"blocks_behind\": 0,\n \"connected\": \"localhost:50002\",\n \"connected_features\": {\n \"daily_fee\": \"0\",\n \"description\": \"\",\n \"donation_address\": \"\",\n \"genesis_hash\": \"6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556\",\n \"hash_function\": \"sha256\",\n \"hosts\": {},\n \"payment_address\": \"\",\n \"protocol_max\": \"0.199.0\",\n \"protocol_min\": \"0.54.0\",\n \"pruning\": null,\n \"server_version\": \"0.105.0\",\n \"trending_algorithm\": \"fast_ar\"\n },\n \"headers_synchronization_progress\": 100,\n \"known_servers\": 0,\n \"servers\": [\n {\n \"availability\": true,\n \"host\": \"localhost\",\n \"latency\": 0.002615762990899384,\n \"port\": 50002\n }\n ]\n },\n \"wallet_server_payments\": {\n \"max_fee\": \"0.0\",\n \"running\": false\n }\n }\n}"
}
]
},
@@ -5062,4 +5062,4 @@
}
]
}
}
}
+9 -26
View File
@@ -85,22 +85,19 @@ class IterativeFinder:
self.protocol = protocol
self.key = key
self.bottom_out_limit = bottom_out_limit
self.max_results = max_results
self.max_results = max(constants.K, max_results)
self.exclude = exclude or []
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set['KademliaPeer'] = set()
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
self.closest_k_ids: typing.List[bytes] = []
self.iteration_queue = asyncio.Queue(loop=self.loop)
self.running_probes: typing.Set[asyncio.Task] = set()
self.iteration_count = 0
self.bottom_out_count = 0
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
@@ -139,14 +136,16 @@ class IterativeFinder:
return []
def _is_closer(self, peer: 'KademliaPeer') -> bool:
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
if len(self.closest_k_ids) < self.max_results:
return True
return self.distance.is_closer(peer.node_id, self.closest_k_ids[self.max_results - 1])
def _add_active(self, peer):
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
self.active.add(peer)
if self._is_closer(peer):
self.prev_closest_peer = self.closest_peer
self.closest_peer = peer
self.closest_k_ids.append(peer.node_id)
self.closest_k_ids.sort(key=lambda sorting_peer: self.distance(self.key))
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
self._add_active(peer)
@@ -285,7 +284,7 @@ class IterativeNodeFinder(IterativeFinder):
and self.peer_manager.peer_is_good(peer) is not False
]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))]
to_yield = not_yet_yielded[:self.max_results]
if to_yield:
self.yielded_peers.update(to_yield)
self.iteration_queue.put_nowait(to_yield)
@@ -297,17 +296,7 @@ class IterativeNodeFinder(IterativeFinder):
if found:
log.debug("found")
return self.put_result(self.active, finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
self.bottom_out_count = 0
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active, True)
return self.put_result(self.active.union(self.contacted), finish=True)
class IterativeValueFinder(IterativeFinder):
@@ -359,7 +348,6 @@ class IterativeValueFinder(IterativeFinder):
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
for compact_addr in response.found_compact_addresses]
to_yield = []
self.bottom_out_count = 0
for blob_peer in blob_peers:
if blob_peer not in self.blob_peers:
self.blob_peers.add(blob_peer)
@@ -371,11 +359,6 @@ class IterativeValueFinder(IterativeFinder):
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
self.iteration_queue.put_nowait(None)
def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key):
+5 -5
View File
@@ -1524,10 +1524,11 @@ class BlockProcessor:
self.db.tx_counts.append(self.tx_count)
cached_max_reorg_depth = self.daemon.cached_height() - self.env.reorg_limit
if height >= cached_max_reorg_depth:
self.db.prefix_db.touched_or_deleted.stage_put(
key_args=(height,), value_args=(self.touched_claim_hashes, self.removed_claim_hashes)
)
# if height >= cached_max_reorg_depth:
self.db.prefix_db.touched_or_deleted.stage_put(
key_args=(height,), value_args=(self.touched_claim_hashes, self.removed_claim_hashes)
)
self.height = height
self.db.headers.append(block.header)
@@ -1556,7 +1557,6 @@ class BlockProcessor:
now = time.time()
self.db.wall_time += now - self.db.last_flush
self.db.last_flush = now
self.db.write_db_state()
def clear_after_advance_or_reorg(self):
@@ -11,6 +11,7 @@ from lbry.wallet.server.db.elasticsearch.constants import ALL_FIELDS
async def get_recent_claims(env, index_name='claims', db=None):
log = logging.getLogger()
need_open = db is None
db = db or LevelDB(env)
try:
@@ -20,6 +21,7 @@ async def get_recent_claims(env, index_name='claims', db=None):
return
if need_open:
await db.initialize_caches()
log.info(f"catching up ES ({db.es_sync_height}) to leveldb height: {db.db_height}")
cnt = 0
touched_claims = set()
deleted_claims = set()
+1 -1
View File
@@ -890,7 +890,7 @@ class LBRYElectrumX(SessionBase):
'donation_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': 'variable_decay'
'trending_algorithm': 'fast_ar'
})
async def server_features_async(self):
+26 -5
View File
@@ -1,6 +1,8 @@
import asyncio
import argparse
import logging
import csv
from io import StringIO
from typing import Optional
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest, Gauge
@@ -24,8 +26,9 @@ PEERS = Gauge(
class SimpleMetrics:
def __init__(self, port):
def __init__(self, port, node):
self.prometheus_port = port
self.dht_node: Node = node
async def handle_metrics_get_request(self, request: web.Request):
try:
@@ -37,9 +40,27 @@ class SimpleMetrics:
log.exception('could not generate prometheus data')
raise
async def handle_peers_csv(self, request: web.Request):
out = StringIO()
writer = csv.DictWriter(out, fieldnames=["ip", "port", "dht_id"])
writer.writeheader()
for peer in self.dht_node.protocol.routing_table.get_peers():
writer.writerow({"ip": peer.address, "port": peer.udp_port, "dht_id": peer.node_id.hex()})
return web.Response(text=out.getvalue(), content_type='text/csv')
async def handle_blobs_csv(self, request: web.Request):
out = StringIO()
writer = csv.DictWriter(out, fieldnames=["blob_hash"])
writer.writeheader()
for blob in self.dht_node.protocol.data_store.keys():
writer.writerow({"blob_hash": blob.hex()})
return web.Response(text=out.getvalue(), content_type='text/csv')
async def start(self):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
prom_app.router.add_get('/peers.csv', self.handle_peers_csv)
prom_app.router.add_get('/blobs.csv', self.handle_blobs_csv)
metrics_runner = web.AppRunner(prom_app)
await metrics_runner.setup()
prom_site = web.TCPSite(metrics_runner, "0.0.0.0", self.prometheus_port)
@@ -47,9 +68,6 @@ class SimpleMetrics:
async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional[str], prometheus_port: int):
if prometheus_port > 0:
metrics = SimpleMetrics(prometheus_port)
await metrics.start()
loop = asyncio.get_event_loop()
conf = Config()
storage = SQLiteStorage(conf, db_file_path, loop, loop.time)
@@ -63,6 +81,9 @@ async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional
loop, PeerManager(loop), generate_id(), port, port, 3333, None,
storage=storage
)
if prometheus_port > 0:
metrics = SimpleMetrics(prometheus_port, node)
await metrics.start()
node.start(host, nodes)
while True:
await asyncio.sleep(10)
@@ -82,6 +103,6 @@ if __name__ == '__main__':
parser.add_argument("--bootstrap_node", default=None, type=str,
help="Node to connect for bootstraping this node. Leave unset to use the default ones. "
"Format: host:port Example: lbrynet1.lbry.com:4444")
parser.add_argument("--prometheus_port", default=0, type=int, help="Port for Prometheus metrics. 0 to disable. Default: 0")
parser.add_argument("--metrics_port", default=0, type=int, help="Port for Prometheus and raw CSV metrics. 0 to disable. Default: 0")
args = parser.parse_args()
asyncio.run(main(args.host, args.port, args.db_file, args.bootstrap_node, args.prometheus_port))
+2 -2
View File
@@ -33,7 +33,7 @@ class NetworkTests(IntegrationTestCase):
'donation_address': '',
'daily_fee': '0',
'server_version': lbry.__version__,
'trending_algorithm': 'variable_decay',
'trending_algorithm': 'fast_ar',
}, await self.ledger.network.get_server_features())
# await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
@@ -58,7 +58,7 @@ class NetworkTests(IntegrationTestCase):
'donation_address': donation_address,
'daily_fee': '42',
'server_version': lbry.__version__,
'trending_algorithm': 'variable_decay',
'trending_algorithm': 'fast_ar',
}, await self.ledger.network.get_server_features())