Compare commits

..
Author SHA1 Message Date
Thomas Zarebczan 1b2ea4eb80 Fix: missed one + test
fix
2019-07-18 18:35:52 -04:00
Thomas Zarebczan a408b113bc fix: effective amount calc
We don't care what the activate height is..that's kept track of separately. We may want to change the name of effective amount, but this doesn't change the understanding of it. If the takeover time had passed, the effective amount is set correctly. It's effective regardless of activation.
2019-07-18 16:19:12 -04:00
31 changed files with 376 additions and 1135 deletions

No files matched your search

+1 -2
View File
@@ -78,8 +78,7 @@ jobs:
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
deploy:
provider: releases
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
api_key: $GITHUB_OAUTH_TOKEN
file: lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
@@ -20,11 +20,14 @@ class ServerConnection {
channel = IOWebSocketChannel.connect(this.url);
int tick = 1;
channel.stream.listen((message) {
var data = json.decode(message);
Map data = json.decode(message);
print(data);
Map commands = data['commands'] ?? {};
_loadDataController.add(
ServerLoadDataPoint.from_map(
tick, data
ServerLoadDataPoint(
tick,
APICallMetrics.from_map(commands['search'] ?? {}),
APICallMetrics.from_map(commands['resolve'] ?? {})
)
);
tick++;
@@ -49,106 +52,48 @@ class ServerConnection {
}
class TimeStats {
final int avg, min, max;
// percentiles
final int five;
final int twenty_five;
final int fifty;
final int seventy_five;
final int ninety_five;
TimeStats(
this.avg, this.min, this.five, this.twenty_five,
this.fifty, this.seventy_five, this.ninety_five,
this.max
);
TimeStats.from_list(List l): this(
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
);
TimeStats.from_zeros(): this(
0, 0, 0, 0, 0, 0, 0, 0
);
factory TimeStats.from_list_or_zeros(List l) =>
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
}
class APICallMetrics {
// total requests received
final int receive_count;
// sum of these is total responses made
final int cache_response_count;
final int query_response_count;
final int intrp_response_count;
final int error_response_count;
// stacked values for chart
final int cache_response_stack;
final int query_response_stack;
final int intrp_response_stack;
final int error_response_stack;
// millisecond timings for non-cache responses
final TimeStats response;
final TimeStats interrupt;
final TimeStats error;
// response, interrupt and error each also report the python, wait and sql stats:
final TimeStats python;
final TimeStats wait;
final TimeStats sql;
// extended timings for individual sql executions
final TimeStats individual_sql;
final int individual_sql_count;
// actual queries
final List<String> errored_queries;
final List<String> interrupted_queries;
final int started;
final int finished;
final int total_time;
final int execution_time;
final int query_time;
final int query_count;
final int cache_hit;
final int avg_wait_time;
final int avg_total_time;
final int avg_execution_time;
final int avg_query_time_per_search;
final int avg_query_time_per_query;
APICallMetrics(
this.receive_count,
this.cache_response_count, this.query_response_count,
this.intrp_response_count, this.error_response_count,
this.response, this.interrupt, this.error,
this.python, this.wait, this.sql,
this.individual_sql, this.individual_sql_count,
this.errored_queries, this.interrupted_queries
):
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
query_response_stack=query_response_count+intrp_response_count+error_response_count,
intrp_response_stack=intrp_response_count+error_response_count,
error_response_stack=error_response_count;
this.started, this.finished, this.total_time, this.execution_time,
this.query_time, this.query_count, this.cache_hit):
avg_wait_time=finished > 0 ? ((total_time - (execution_time + query_time))/finished).round() : 0,
avg_total_time=finished > 0 ? (total_time/finished).round() : 0,
avg_execution_time=finished > 0 ? (execution_time/finished).round() : 0,
avg_query_time_per_search=finished > 0 ? (query_time/finished).round() : 0,
avg_query_time_per_query=query_count > 0 ? (query_time/query_count).round() : 0;
APICallMetrics.from_map(Map data): this(
data["receive_count"] ?? 0,
data["cache_response_count"] ?? 0,
data["query_response_count"] ?? 0,
data["intrp_response_count"] ?? 0,
data["error_response_count"] ?? 0,
TimeStats.from_list_or_zeros(data["response"]),
TimeStats.from_list_or_zeros(data["interrupt"]),
TimeStats.from_list_or_zeros(data["error"]),
TimeStats.from_list_or_zeros(data["python"]),
TimeStats.from_list_or_zeros(data["wait"]),
TimeStats.from_list_or_zeros(data["sql"]),
TimeStats.from_list_or_zeros(data["individual_sql"]),
data["individual_sql_count"] ?? 0,
List<String>.from(data["errored_queries"] ?? const []),
List<String>.from(data["interrupted_queries"] ?? const []),
data['started'] ?? 0,
data['finished'] ?? 0,
data['total_time'] ?? 0,
data['execution_time'] ?? 0,
data['query_time'] ?? 0,
data['query_count'] ?? 0,
data['cache_hit'] ?? 0,
);
}
class ServerLoadDataPoint {
final int tick;
final int sessions;
final APICallMetrics search;
final APICallMetrics resolve;
const ServerLoadDataPoint(
this.tick, this.sessions, this.search, this.resolve
);
ServerLoadDataPoint.from_map(int tick, Map data): this(
tick, (data['status'] ?? const {})['sessions'] ?? 0,
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
);
ServerLoadDataPoint.empty(): this(
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
);
ServerLoadDataPoint(this.tick, this.search, this.resolve);
ServerLoadDataPoint.empty():
tick = 0,
search=APICallMetrics.from_map({}),
resolve=APICallMetrics.from_map({});
}
@@ -13,12 +13,7 @@ class ServerCharts extends StatelessWidget {
Widget build(BuildContext context) {
var server = Provider.of<Server>(context, listen: false);
return ListView(children: <Widget>[
SizedBox(height: 300.0, child: APILoadChart(
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
)),
//SizedBox(height: 300.0, child: APILoadChart(
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
//)),
SizedBox(height: 300.0, child: ServerLoadChart(server)),
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
@@ -27,20 +22,16 @@ class ServerCharts extends StatelessWidget {
}
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
class APILoadChart extends StatefulWidget {
class ServerLoadChart extends StatefulWidget {
final Server server;
final String name;
final APIGetter getter;
APILoadChart(this.server, this.name, this.getter);
ServerLoadChart(this.server);
@override
State<StatefulWidget> createState() => APILoadChartState();
State<StatefulWidget> createState() => ServerLoadChartState();
}
class APILoadChartState extends State<APILoadChart> {
class ServerLoadChartState extends State<ServerLoadChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@@ -48,73 +39,54 @@ class APILoadChartState extends State<APILoadChart> {
void initState() {
super.initState();
seriesData = [
/*
charts.Series<ServerLoadDataPoint, int>(
id: 'Received',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
strokeWidthPxFn: (_, __) => 4.0,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
data: widget.server.serverLoadData,
),*/
charts.Series<ServerLoadDataPoint, int>(
id: 'Cache',
colorFn: (_, __) =>
charts.MaterialPalette.green.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Query',
colorFn: (_, __) =>
charts.MaterialPalette.blue.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Interrupts',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Errors',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
data: widget.server.serverLoadData,
),
/*
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Interrupted',
colorFn: (_, __) =>
charts.MaterialPalette.pink.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Errored',
colorFn: (_, __) =>
charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} From Cache',
id: 'Search Cache',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
measureFn: (ServerLoadDataPoint load, _) => load.search.cache_hit,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Search Finish',
colorFn: (_, __) =>
charts.MaterialPalette.deepOrange.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.finished,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Search Start',
colorFn: (_, __) =>
charts.MaterialPalette.deepOrange.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.started,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Cache',
colorFn: (_, __) => charts.MaterialPalette.cyan.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.cache_hit,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Finish',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.finished,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Start',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.started,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
data: widget.server.serverLoadData,
),
*/
];
}
@@ -122,9 +94,7 @@ class APILoadChartState extends State<APILoadChart> {
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData,
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
)
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
@@ -147,41 +117,27 @@ class ServerPerformanceChartState extends State<ServerPerformanceChart> {
void initState() {
super.initState();
seriesData = [
charts.Series<ServerLoadDataPoint, int>(
id: 'Waiting 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Waiting',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_wait_time,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Executing',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'SQLite 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_execution_time,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. SQLite',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_query_time_per_search,
data: widget.server.serverLoadData,
),
)
];
}
@@ -304,13 +260,11 @@ class BetterLineChart extends charts.LineChart {
final int itemCount;
final Object lastItem;
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
BetterLineChart(List<charts.Series<dynamic, int>> seriesList):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(
seriesList,
animate: false,
defaultRenderer: renderer,
behaviors: [charts.SeriesLegend()],
domainAxis: charts.NumericAxisSpec(
viewport: new charts.NumericExtents(
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.3"
__version__ = "0.38.1"
version = tuple(__version__.split('.'))
-3
View File
@@ -551,9 +551,6 @@ class Config(CLIConfig):
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
+1 -1
View File
@@ -6,7 +6,7 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
def clean_tags(tags: List[str]):
-2
View File
@@ -214,9 +214,7 @@ class CommandTestCase(IntegrationTestCase):
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
height = self.ledger.headers.height
await self.generate(1)
await self.on_header(height + 1)
await self.on_transaction_dict(claim)
return claim
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+2 -46
View File
@@ -3,11 +3,10 @@ import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL, normalize_name
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
@@ -73,50 +72,7 @@ class MainNetLedger(BaseLedger):
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return await self.validate_resolutions(result)
async def validate_resolutions(self, resolutions: dict):
provables = {}
for url, txo in resolutions.items():
if isinstance(txo, dict) and 'error' in txo:
continue
parsed_url = URL.parse(url)
# cases we check: names without sequence, claim id or any modifier
# except: @channel/name as the signature is enough, instead we check '@channel'
if parsed_url.has_channel:
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
if not provables:
return resolutions
root_hash = self.headers.claim_trie_root.decode()
names = list(provables.keys())
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
for proof, name in zip(proofs, names):
all_failed = True
for url, txid in provables[name]:
if not proof.get('txhash'):
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
continue
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
if txid != expected_txid:
resolutions[url] = {
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
}
else:
all_failed = False
if not all_failed:
try:
verify_proof(proof, root_hash, name)
continue
except InvalidProofError as error:
for url, _ in provables[name]:
resolution = resolutions[url]
if isinstance(resolution, dict) and 'error' in resolution:
continue
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
return resolutions
return result
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
-3
View File
@@ -9,8 +9,5 @@ class Network(BaseNetwork):
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def get_name_proofs(self, block_hash, *names):
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
@@ -79,10 +79,6 @@ class LBRYBlockProcessor(BlockProcessor):
raise
finally:
self.sql.commit()
if self.db.first_sync and self.height == self.daemon.cached_height():
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
-207
View File
@@ -11,210 +11,3 @@ STREAM_TYPES = {
'binary': 5,
'model': 6
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
+39 -77
View File
@@ -9,20 +9,14 @@ from contextvars import ContextVar
from functools import wraps
from dataclasses import dataclass
from torba.client.basedatabase import query, interpolate
from torba.client.basedatabase import query
from lbry.schema.url import URL, normalize_name
from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
class SQLiteOperationalError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query errored')
self.metrics = metrics
from .common import CLAIM_TYPES, STREAM_TYPES
class SQLiteInterruptedError(sqlite3.OperationalError):
@@ -56,7 +50,7 @@ SEARCH_PARAMS = {
ORDER_FIELDS = {
'name', 'claim_hash'
'name',
} | INTEGER_PARAMS
@@ -97,6 +91,7 @@ ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
db = sqlite3.connect(_path, isolation_level=None, uri=True)
db.row_factory = sqlite3.Row
ctx.set(
@@ -119,16 +114,21 @@ def measure(func):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
metric = {}
state.metrics.setdefault(func.__name__, []).append(metric)
metric = state.metrics.setdefault(func.__name__, {
'calls': 0, 'total': 0, 'isolated': 0, 'errors': 0
})
state.stack.append([])
start = time.perf_counter()
try:
return func(*args, **kwargs)
except:
metric['errors'] += 1
raise
finally:
elapsed = int((time.perf_counter()-start)*1000)
metric['total'] = elapsed
metric['isolated'] = (elapsed-sum(state.stack.pop()))
metric['calls'] += 1
metric['total'] += elapsed
metric['isolated'] += (elapsed-sum(state.stack.pop()))
if state.stack:
state.stack[-1].append(elapsed)
return wrapper
@@ -167,17 +167,16 @@ def execute_query(sql, values) -> List:
try:
return context.db.execute(sql, values).fetchall()
except sqlite3.OperationalError as err:
plain_sql = interpolate(sql, values)
if context.is_tracking_metrics:
context.metrics['execute_query'][-1]['sql'] = plain_sql
if str(err) == "interrupted":
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
query_str = sql
for k in sorted(values.keys(), reverse=True):
query_str = query_str.replace(f":{k}", str(values[k]) if not k.startswith("$") else f"'{values[k]}'")
context.log.warning("interrupted slow sqlite query:\n%s", query_str)
raise SQLiteInterruptedError(context.metrics)
context.log.exception('failed running query', exc_info=err)
raise SQLiteOperationalError(context.metrics)
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
@measure
def get_claims(cols, for_count=False, **constraints) -> List:
if 'order_by' in constraints:
sql_order_by = []
for order_by in constraints['order_by']:
@@ -232,6 +231,14 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'public_key_id' in constraints:
constraints['claim.public_key_hash'] = sqlite3.Binary(
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
if 'channel_hash' in constraints:
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_ids' in constraints:
@@ -300,18 +307,7 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)
return sql, values
def get_claims(cols, for_count=False, **constraints) -> List:
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
sql, values = _get_claims(cols, for_count, **constraints)
return execute_query(sql, values)
@@ -334,7 +330,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
constraints['order_by'] = ["height", "^name"]
txo_rows = _search(**constraints)
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
extra_txo_rows = []
@@ -408,7 +404,7 @@ def resolve_url(raw_url):
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount', '^height']
query['order_by'] = ['effective_amount']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
@@ -424,6 +420,7 @@ def resolve_url(raw_url):
return channel
@measure
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
@@ -432,61 +429,26 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
if attr == 'tag':
common_tags = any_items & COMMON_TAGS.keys()
if common_tags:
any_items -= common_tags
if len(common_tags) < 5:
for item in common_tags:
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
elif len(common_tags) >= 5:
constraints.update({
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
})
values = ', '.join(
f':$any_common_tag{i}' for i in range(len(common_tags))
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
if for_count:
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
constraints[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
+28 -44
View File
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
)
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -76,12 +76,30 @@ class SQLDB:
trending_global integer not null default 0
);
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_height_idx on claim (height);
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
"""
CREATE_SUPPORT_TABLE = """
@@ -92,6 +110,7 @@ class SQLDB:
claim_hash bytes not null,
amount integer not null
);
create index if not exists support_txo_hash_idx on support (txo_hash);
create index if not exists support_claim_hash_idx on support (claim_hash, height);
"""
@@ -101,6 +120,7 @@ class SQLDB:
claim_hash bytes not null,
height integer not null
);
create index if not exists tag_tag_idx on tag (tag);
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
@@ -113,38 +133,6 @@ class SQLDB:
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
TAG_INDEXES = '\n'.join(
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
for tag_value, tag_key in COMMON_TAGS.items()
)
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_CLAIM_TABLE +
@@ -269,7 +257,7 @@ class SQLDB:
if tags:
self.db.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
return claims
@@ -546,15 +534,10 @@ class SQLDB:
""", claim_hashes)
def _update_effective_amount(self, height, claim_hashes=None):
self.execute(
f"UPDATE claim SET effective_amount = amount + support_amount "
f"WHERE activation_height = {height}"
)
if claim_hashes:
self.execute(
f"UPDATE claim SET effective_amount = amount + support_amount "
f"WHERE activation_height < {height} "
f" AND claim_hash IN ({','.join('?' for _ in claim_hashes)})",
f"WHERE claim_hash IN ({','.join('?' for _ in claim_hashes)})",
claim_hashes
)
@@ -582,12 +565,13 @@ class SQLDB:
claimtrie.claim_hash AS current_winner,
MAX(winner.effective_amount)
FROM (
SELECT normalized, claim_hash, effective_amount FROM claim
SELECT normalized, claim_hash, effective_amount, activation_height FROM claim
WHERE normalized IN (
SELECT normalized FROM claim WHERE activation_height={height} {claim_hashes_sql}
) {deleted_names_sql}
ORDER BY effective_amount DESC, height ASC, tx_position ASC
) AS winner LEFT JOIN claimtrie USING (normalized)
WHERE winner.activation_height<={height}
GROUP BY winner.normalized
HAVING current_winner IS NULL OR current_winner <> winner.claim_hash
""", changed_claim_hashes+deleted_names)
-135
View File
@@ -1,135 +0,0 @@
import time
import math
from typing import Tuple
def calculate_elapsed(start) -> int:
return int((time.perf_counter() - start) * 1000)
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
if not data:
return 0, 0, 0, 0, 0, 0, 0, 0
data.sort()
size = len(data)
return (
int(sum(data) / size),
data[0],
data[math.ceil(size * .05) - 1],
data[math.ceil(size * .25) - 1],
data[math.ceil(size * .50) - 1],
data[math.ceil(size * .75) - 1],
data[math.ceil(size * .95) - 1],
data[-1]
)
def remove_select_list(sql) -> str:
return sql[sql.index('FROM'):]
class APICallMetrics:
def __init__(self, name):
self.name = name
# total requests received
self.receive_count = 0
self.cache_response_count = 0
# millisecond timings for query based responses
self.query_response_times = []
self.query_intrp_times = []
self.query_error_times = []
self.query_python_times = []
self.query_wait_times = []
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
self.individual_sql_times = [] # every SQL query run on server
# actual queries
self.errored_queries = set()
self.interrupted_queries = set()
def to_json_and_reset(self):
return {
# total requests received
"receive_count": self.receive_count,
# sum of these is total responses made
"cache_response_count": self.cache_response_count,
"query_response_count": len(self.query_response_times),
"intrp_response_count": len(self.query_intrp_times),
"error_response_count": len(self.query_error_times),
# millisecond timings for non-cache responses
"response": calculate_avg_percentiles(self.query_response_times),
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
"error": calculate_avg_percentiles(self.query_error_times),
# response, interrupt and error each also report the python, wait and sql stats:
"python": calculate_avg_percentiles(self.query_python_times),
"wait": calculate_avg_percentiles(self.query_wait_times),
"sql": calculate_avg_percentiles(self.query_sql_times),
# extended timings for individual sql executions
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
"individual_sql_count": len(self.individual_sql_times),
# actual queries
"errored_queries": list(self.errored_queries),
"interrupted_queries": list(self.interrupted_queries),
}
def start(self):
self.receive_count += 1
def cache_response(self):
self.cache_response_count += 1
def _add_query_timings(self, request_total_time, metrics):
if metrics and 'execute_query' in metrics:
sub_process_total = metrics[self.name][0]['total']
individual_query_times = [f['total'] for f in metrics['execute_query']]
aggregated_query_time = sum(individual_query_times)
self.individual_sql_times.extend(individual_query_times)
self.query_sql_times.append(aggregated_query_time)
self.query_python_times.append(sub_process_total - aggregated_query_time)
self.query_wait_times.append(request_total_time - sub_process_total)
@staticmethod
def _add_queries(query_set, metrics):
if metrics and 'execute_query' in metrics:
for execute_query in metrics['execute_query']:
if 'sql' in execute_query:
query_set.add(remove_select_list(execute_query['sql']))
def query_response(self, start, metrics):
self.query_response_times.append(calculate_elapsed(start))
self._add_query_timings(self.query_response_times[-1], metrics)
def query_interrupt(self, start, metrics):
self.query_intrp_times.append(calculate_elapsed(start))
self._add_queries(self.interrupted_queries, metrics)
self._add_query_timings(self.query_intrp_times[-1], metrics)
def query_error(self, start, metrics):
self.query_error_times.append(calculate_elapsed(start))
self._add_queries(self.errored_queries, metrics)
self._add_query_timings(self.query_error_times[-1], metrics)
class ServerLoadData:
def __init__(self):
self._apis = {}
def for_api(self, name) -> APICallMetrics:
if name not in self._apis:
self._apis[name] = APICallMetrics(name)
return self._apis[name]
def to_json_and_reset(self, status):
try:
return {
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
'status': status
}
finally:
self._apis = {}
+137 -35
View File
@@ -4,9 +4,13 @@ import time
import base64
import asyncio
from binascii import hexlify
from weakref import WeakSet
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
from torba.rpc.jsonrpc import RPCError, JSONRPC
from torba.server.session import ElectrumX, SessionManager
from torba.server import util
@@ -14,8 +18,56 @@ from torba.server import util
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
class ResultCacheItem:
@@ -43,8 +95,9 @@ class LBRYSessionManager(SessionManager):
super().__init__(*args, **kwargs)
self.query_executor = None
self.websocket = None
self.metrics = ServerLoadData()
self.metrics_loop = None
self.metrics_processor = None
self.command_metrics = {}
self.reader_metrics = {}
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
@@ -52,11 +105,69 @@ class LBRYSessionManager(SessionManager):
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
def get_command_tracking_info(self, command):
if command not in self.command_metrics:
self.command_metrics[command] = {
'cache_hit': 0,
'started': 0,
'finished': 0,
'total_time': 0,
'execution_time': 0,
'query_time': 0,
'query_count': 0,
'interrupted': 0,
'interrupted_query_values': [],
}
return self.command_metrics[command]
def cache_hit(self, command_name):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['cache_hit'] += 1
def start_command_tracking(self, command_name):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['started'] += 1
def finish_command_tracking(self, command_name, elapsed, metrics):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['finished'] += 1
command['total_time'] += elapsed
if 'execute_query' in metrics:
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
command['query_time'] += metrics['execute_query']['total']
command['query_count'] += metrics['execute_query']['calls']
for func_name, func_metrics in metrics.items():
reader = self.reader_metrics.setdefault(func_name, {})
for key in func_metrics:
if key not in reader:
reader[key] = func_metrics[key]
else:
reader[key] += func_metrics[key]
def interrupted_command_error(self, command_name, elapsed, metrics, kwargs):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['finished'] += 1
command['interrupted'] += 1
command['total_time'] += elapsed
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
command['query_time'] += metrics['execute_query']['total']
command['query_count'] += metrics['execute_query']['calls']
if len(command['interrupted_query_values']) < 100:
command['interrupted_query_values'].append(kwargs)
async def process_metrics(self):
while self.running:
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
commands, self.command_metrics = self.command_metrics, {}
reader, self.reader_metrics = self.reader_metrics, {}
if self.websocket is not None:
self.websocket.send_message(data)
self.websocket.send_message({
'commands': commands,
'reader': reader
})
await asyncio.sleep(1)
async def start_other(self):
@@ -75,12 +186,12 @@ class LBRYSessionManager(SessionManager):
if self.websocket is not None:
await self.websocket.start()
if self.env.track_metrics:
self.metrics_loop = asyncio.create_task(self.process_metrics())
self.metrics_processor = asyncio.create_task(self.process_metrics())
async def stop_other(self):
self.running = False
if self.env.track_metrics:
self.metrics_loop.cancel()
self.metrics_processor.cancel()
if self.websocket is not None:
await self.websocket.stop()
self.query_executor.shutdown()
@@ -104,56 +215,50 @@ class LBRYElectrumX(ElectrumX):
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': self.get_server_height,
}
self.request_handlers.update(handlers)
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
start = time.perf_counter()
async def run_in_executor(self, name, func, kwargs):
start = None
if self.env.track_metrics:
self.session_mgr.start_command_tracking(name)
start = time.perf_counter()
try:
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
except reader.SQLiteInterruptedError as error:
metrics.query_interrupt(start, error.metrics)
self.session_mgr.interrupted_command_error(
name, int((time.perf_counter() - start) * 1000), error.metrics, kwargs
)
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics.query_error(start, error.metrics)
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except:
metrics.query_error(start, {})
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
if self.env.track_metrics:
(result, metrics_data) = result
metrics.query_response(start, metrics_data)
elapsed = int((time.perf_counter() - start) * 1000)
(result, metrics) = result
self.session_mgr.finish_command_tracking(name, elapsed, metrics)
return base64.b64encode(result).decode()
async def run_and_cache_query(self, query_name, function, kwargs):
if self.env.track_metrics:
metrics = self.session_mgr.metrics.for_api(query_name)
else:
metrics = APICallMetrics(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
self.session_mgr.cache_hit(query_name)
return cache_item.result
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
metrics, function, kwargs
result = cache_item.result
if result is None:
result = cache_item.result = await self.run_in_executor(
query_name, function, kwargs
)
else:
metrics.cache_response()
return cache_item.result
self.session_mgr.cache_hit(query_name)
return result
async def claimtrie_search(self, **kwargs):
if kwargs:
@@ -166,9 +271,6 @@ class LBRYElectrumX(ElectrumX):
async def get_server_height(self):
return self.bp.height
def claimtrie_getnameproofs(self, block_hash, *names):
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
async def transaction_get_height(self, tx_hash):
self.assert_tx_hash(tx_hash)
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
-55
View File
@@ -1,55 +0,0 @@
import asyncio
from weakref import WeakSet
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
+19 -48
View File
@@ -1,10 +1,10 @@
import sys
import os
import time
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.server.db.reader import search_to_bytes, initializer
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
@@ -24,25 +24,6 @@ DEFAULT_ANY_TAGS = [
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
@@ -66,10 +47,10 @@ ORDER_BY = [
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
any_tags_combinations = [DEFAULT_ANY_TAGS, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_totals in [True]:
for offset in [0, 100]:
for offset in [0, 20, 40, 60, 80, 100, 1000, 2000, 3000]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
@@ -90,12 +71,13 @@ def get_args(limit=20):
def _search(kwargs):
start = time.time()
msg = f"offset={kwargs['offset']}, limit={kwargs['limit']}, no_totals={kwargs['no_totals']}, not_tags={kwargs.get('not_tags')}, any_tags={kwargs.get('any_tags')}, order_by={kwargs['order_by']}"
try:
search_to_bytes(kwargs)
t = time.time() - start
return t, kwargs
return t, f"{t} - {msg}"
except Exception as err:
return -1, f"failed: error={str(type(err))}({str(err)})"
return -1, f"failed: error={str(type(err))}({str(err)}) - {msg}"
async def search(executor, kwargs):
@@ -115,32 +97,21 @@ async def main(db_path, max_query_time):
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
for ts, constraints in results:
if ts >= max_query_time:
sql = interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints))
print(f"Query took {int(ts * 1000)}ms\n{sql}")
times = {msg: ts for ts, msg in results}
log.info("\n".join(sorted(filter(lambda msg: times[msg] > max_query_time, times.keys()), key=lambda msg: times[msg])))
finally:
query_executor.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
args = sys.argv[1:]
if len(args) >= 1:
db_path = args[0]
else:
db_path = os.path.expanduser('~/claims.db')
if len(args) >= 2:
max_query_time = float(args[1])
else:
max_query_time = -3
asyncio.run(main(db_path, max_query_time))
-107
View File
@@ -1,107 +0,0 @@
import time
import asyncio
import random
from argparse import ArgumentParser
from torba.client.basenetwork import ClientSession
class AgentSmith(ClientSession):
async def do_nefarious_things(self):
await self.send_request('blockchain.claimtrie.search', {
'no_totals': True,
'offset': random.choice(range(0, 300, 20)),
'limit': 20,
'any_tags': (
random.choice([[
random.choice(['gaming', 'games', 'game']) +
random.choice(['entertainment', 'playthrough', 'funny']) +
random.choice(['xbox', 'xbox one', 'xbox news'])
], [
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
random.choice(['news', 'sighting', 'sightings'])
], [
random.choice(['art', 'automotive']),
random.choice(['blockchain', 'economics', 'food']),
random.choice(['funny', 'learnings', 'nature']),
random.choice(['news', 'science', 'technology'])
]
])
),
'not_tags': random.choice([[], [
'porn', 'mature', 'xxx', 'nsfw'
]]),
'order_by': random.choice([
['release_time'],
['trending_global', 'trending_mixed'],
['effective_amount']
])
})
class AgentSmithProgram:
def __init__(self, host, port):
self.host, self.port = host, port
self.agent_smiths = []
async def make_one_more_of_them(self):
smith = AgentSmith(network=None, server=(self.host, self.port))
await smith.create_connection()
self.agent_smiths.append(smith)
async def coordinate_nefarious_activity(self):
start = time.perf_counter()
await asyncio.gather(
*(s.do_nefarious_things() for s in self.agent_smiths),
return_exceptions=True
)
return time.perf_counter() - start
def __len__(self):
return len(self.agent_smiths)
async def delete_one_smith(self):
if self.agent_smiths:
await self.agent_smiths.pop().close()
async def delete_program(self):
await asyncio.gather(*(
s.close() for s in self.agent_smiths
))
async def main(host, port):
smiths = AgentSmithProgram(host, port)
await smiths.make_one_more_of_them()
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
ease_off = 0
for i in range(1000):
await asyncio.sleep(1)
if activity.done() and activity.result() < .9:
print('more, more, more...')
await asyncio.gather(*(
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
))
else:
print('!!!!!!!!!!!!!!')
print('IS NEO LOSING?')
print('!!!!!!!!!!!!!!')
await asyncio.gather(*(
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
))
print(f'coordinate all {len(smiths)} smiths to action')
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
print('finishing up any remaining actions')
await activity
print('neo has won, deleting agents...')
await smiths.delete_program()
print('done.')
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--host', dest='host', default='localhost', type=str)
parser.add_argument('--port', dest='port', default=50001, type=int)
args = parser.parse_args()
asyncio.run(main(args.host, args.port))
+15 -16
View File
@@ -1,10 +1,9 @@
import uvloop, asyncio, time, sys, logging
from concurrent.futures import ProcessPoolExecutor
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
db_path = '../../../lbryconf/wallet-server/claims.db'
db_path = '/tmp/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
@@ -17,26 +16,26 @@ async def run_times(executor, iterations, show=True):
'no_totals': True,
'offset': 0,
'limit': 20,
'fee_amount': '<1',
#'all_tags': ['funny'],
'any_tags': [
'ufos', 'city fix'
'crypto',
'outdoors',
'cars',
'automotive'
],
'not_tags': [
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
'nsfw', 'xxx', 'mature'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query'][0]['total'] for r in timings]
timings = [r[1]['execute_query']['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t}ms')
@@ -45,16 +44,16 @@ async def run_times(executor, iterations, show=True):
async def main():
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', default_query_timout, True)
)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
await run_times(executor, 4, show=False)
await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
await run_times(executor, 2**9)
await run_times(executor, 2**11)
await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
@@ -89,7 +89,6 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim], **kwargs)
async def assertFindsClaims(self, claims, **kwargs):
kwargs.setdefault('order_by', ['height', '^name'])
results = await self.claim_search(**kwargs)
self.assertEqual(len(claims), len(results))
for claim, result in zip(claims, results):
@@ -166,15 +165,15 @@ class ClaimSearchCommand(ClaimTestCase):
await self.create_channel()
await self.create_lots_of_streams()
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page_size=20, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams)
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[:6])
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page=2, page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[6:])
@@ -15,7 +15,6 @@ class TestSessionBloat(IntegrationTestCase):
async def test_session_bloat_from_socket_timeout(self):
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
await self.conductor.start_spv()
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
+1 -1
View File
@@ -11,7 +11,7 @@ class TestTagNormalization(unittest.TestCase):
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'T\'ag')
tag('tag', 'Tag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
@@ -1,60 +0,0 @@
import time
import unittest
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
class TestPercentileCalculation(unittest.TestCase):
def test_calculate_percentiles(self):
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
self.assertEqual(calculate_avg_percentiles(
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
class TestCollectingMetrics(unittest.TestCase):
def test_happy_path(self):
self.maxDiff = None
load = ServerLoadData()
search = load.for_api('search')
self.assertEqual(search.name, 'search')
search.start()
search.cache_response()
search.cache_response()
metrics = {
'search': [{'total': 40}],
'execute_query': [
{'total': 20},
{'total': 10}
]
}
for x in range(5):
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
metrics['execute_query'][0]['total'] = 10
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
search.query_interrupt(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.052, {})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
"receive_count": 1,
"cache_response_count": 2,
"query_response_count": 5,
"intrp_response_count": 1,
"error_response_count": 2,
"response": (53, 51, 51, 52, 53, 54, 55, 55),
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
"error": (51, 50, 50, 50, 50, 52, 52, 52),
"python": (12, 10, 10, 10, 10, 20, 20, 20),
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
"individual_sql_count": 14,
"errored_queries": ['FROM claim where something=1'],
"interrupted_queries": ['FROM claim where something=1'],
}}})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
+7 -7
View File
@@ -167,35 +167,35 @@ class TestClaimtrie(TestSQLDB):
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[],
accepted=[('Claim B', 20*COIN, 0, 1031)]
accepted=[('Claim B', 20*COIN, 20*COIN, 1031)]
)
advance(1010, [self.get_support(stream, 14*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[],
accepted=[('Claim B', 20*COIN, 0, 1031)]
accepted=[('Claim B', 20*COIN, 20*COIN, 1031)]
)
advance(1020, [self.get_stream('Claim C', 50*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[],
accepted=[
('Claim B', 20*COIN, 0, 1031),
('Claim C', 50*COIN, 0, 1051)]
('Claim B', 20*COIN, 20*COIN, 1031),
('Claim C', 50*COIN, 50*COIN, 1051)]
)
advance(1031, [])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
accepted=[('Claim C', 50*COIN, 0, 1051)]
accepted=[('Claim C', 50*COIN, 50*COIN, 1051)]
)
advance(1040, [self.get_stream('Claim D', 300*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
accepted=[
('Claim C', 50*COIN, 0, 1051),
('Claim D', 300*COIN, 0, 1072)]
('Claim C', 50*COIN, 50*COIN, 1051),
('Claim D', 300*COIN, 300*COIN, 1072)]
)
advance(1051, [])
state(
@@ -53,45 +53,35 @@ class ReconnectTests(IntegrationTestCase):
class ServerPickingTestCase(AsyncioTestCase):
async def _make_fake_server(self, latency=1.0, port=1):
async def _make_fake_server(self, latency=1.0, port=1337):
# local fake server with artificial latency
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
return {"height": 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
proto = RPCSession()
async def __handler(_):
await asyncio.sleep(latency)
return {'height': 1}
async def _make_bad_server(self, port=42420):
async def echo(reader, writer):
while True:
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
proto.handle_request = __handler
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
# fast but unhealthy, should be discarded
await self._make_bad_server(),
('localhost', 1),
('example.that.doesnt.resolve', 9000),
await self._make_fake_server(latency=1.2, port=1340),
await self._make_fake_server(latency=0.5, port=1337),
await self._make_fake_server(latency=0.7, port=1339),
await self._make_fake_server(latency=1.5, port=1340),
await self._make_fake_server(latency=0.1, port=1337),
await self._make_fake_server(latency=1.0, port=1339),
await self._make_fake_server(latency=0.5, port=1338),
],
'connect_timeout': 3
'connect_timeout': 30
})
network = BaseNetwork(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=3)
await asyncio.wait_for(network.on_connected.first, timeout=1)
self.assertTrue(network.is_connected)
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
# ensure we are connected to all of them
self.assertEqual(len(network.session_pool.sessions), 4)
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
self.assertEqual(len(network.session_pool.sessions), 3)
+2 -21
View File
@@ -1,10 +1,11 @@
import unittest
import sqlite3
from functools import wraps
from torba.client.wallet import Wallet
from torba.client.constants import COIN
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
from torba.client.hash import sha256
from torba.testcase import AsyncioTestCase
@@ -166,26 +167,6 @@ class TestQueryBuilder(unittest.TestCase):
("select * from foo LIMIT 20 OFFSET 10", {})
)
def test_query_interpolation(self):
self.maxDiff = None
# tests that interpolation replaces longer keys first
self.assertEqual(
interpolate(*query(
"select * from foo",
a__not='b', b__in='select * from blah where c=:$c',
d__any={'one__like': 'o', 'two': 2},
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
ahash=memoryview(sha256(b'hello world')),
limit=10, order_by='b', **{'$c': 3})
),
"select * from foo WHERE a != 'b' AND "
"b IN (select * from blah where c=3) AND "
"(one LIKE 'o' OR two = 2) AND "
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
"ORDER BY b LIMIT 10",
)
class TestQueries(AsyncioTestCase):
-14
View File
@@ -1,7 +1,6 @@
import logging
import asyncio
from asyncio import wrap_future
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
@@ -192,19 +191,6 @@ def query(select, **constraints):
return ' '.join(sql), values
def interpolate(sql, values):
for k in sorted(values.keys(), reverse=True):
value = values[k]
if isinstance(value, memoryview):
value = f"X'{hexlify(bytes(value)).decode()}'"
elif isinstance(value, str):
value = f"'{value}'"
else:
value = str(value)
sql = sql.replace(f":{k}", value)
return sql
def rows_to_dict(rows, fields):
if rows:
return [dict(zip(fields, r)) for r in rows]
+8 -14
View File
@@ -40,11 +40,7 @@ class ClientSession(BaseClientSession):
# TODO: change to 'ping' on newer protocol (above 1.2)
while not self.is_closing():
if (time() - self.last_send) > self.max_seconds_idle:
try:
await self.send_request('server.banner')
except:
self.abort()
raise
await self.send_request('server.banner')
await asyncio.sleep(self.max_seconds_idle//3)
async def create_connection(self, timeout=6):
@@ -227,14 +223,10 @@ class SessionPool:
self._dead_servers = []
async def ensure_connection(self, session):
self._dead_servers.append(session)
self.sessions.remove(session)
if not session.is_closing():
return
try:
if session.is_closing():
await session.create_connection(self.timeout)
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
self.sessions.append(session)
self._dead_servers.remove(session)
return await session.create_connection(self.timeout)
except asyncio.TimeoutError:
log.warning("Timeout connecting to %s:%d", *session.server)
except asyncio.CancelledError: # pylint: disable=try-except-raise
@@ -246,9 +238,11 @@ class SessionPool:
log.warning("Could not connect to %s:%d", *session.server)
else:
log.exception("Connecting to %s:%d raised an exception:", *session.server)
self._dead_servers.append(session)
self.sessions.remove(session)
async def get_online_sessions(self):
self._lost_master.set()
while not self.online:
self._lost_master.set()
await asyncio.sleep(0.5)
await asyncio.sleep(0.1)
return self.sessions
+5 -8
View File
@@ -206,8 +206,7 @@ class SPVNode:
'HOST': self.hostname,
'TCP_PORT': str(self.port),
'SESSION_TIMEOUT': str(self.session_timeout),
'MAX_QUERY_WORKERS': '0',
'INDIVIDUAL_TAG_INDEXES': '',
'MAX_QUERY_WORKERS': '0'
}
# TODO: don't use os.environ
os.environ.update(conf)
@@ -266,7 +265,6 @@ class BlockchainNode:
self.data_path = None
self.protocol = None
self.transport = None
self.blockchain_address = None
self._block_expected = 0
self.hostname = 'localhost'
self.peerport = 9246 + 2 # avoid conflict with default peer port
@@ -330,7 +328,7 @@ class BlockchainNode:
self.daemon_bin,
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
f'-port={self.peerport}'
)
self.log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(
@@ -365,10 +363,9 @@ class BlockchainNode:
self.log.info(out.decode().strip())
return out.decode().strip()
async def generate(self, blocks, address=None):
def generate(self, blocks):
self._block_expected += blocks
address = await self.get_raw_change_address()
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
return self._cli_cmnd('generate', str(blocks))
def invalidate_block(self, blockhash):
return self._cli_cmnd('invalidateblock', blockhash)
@@ -377,7 +374,7 @@ class BlockchainNode:
return self._cli_cmnd('getblockhash', str(block))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress', 'legacy')
return self._cli_cmnd('getrawchangeaddress')
async def get_balance(self):
return float(await self._cli_cmnd('getbalance'))
-1
View File
@@ -38,7 +38,6 @@ class Env:
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
self.websocket_port = self.integer('WEBSOCKET_PORT', None)