Compare commits

...
67 Commits
Author SHA1 Message Date
Victor Shyba 593422f70c niko 2019-08-23 21:50:11 -03:00
Alex Grin de12e25450 v0.38.6 2019-07-29 15:11:18 -04:00
Oleg Silkin 2545ebbece Adds unittests & fixes comment_client functions 2019-07-29 14:55:42 -04:00
Oleg Silkin 14ec6ce43b Adds unit-testing for delete comment signatures 2019-07-29 14:55:42 -04:00
Oleg Silkin 0cac1e65aa Imports comment_client and uses dot notation for rest of modules 2019-07-29 14:55:42 -04:00
Oleg Silkin 1e4d57ab83 Simplifies signing function into one to make API symmetric 2019-07-29 14:55:42 -04:00
Oleg Silkin 49b42b7e03 Simplifies 2019-07-29 14:55:42 -04:00
Oleg Silkin 89c0b5dbdf adds comment abandon function 2019-07-29 14:55:42 -04:00
Lex Berezhny 8138675808 added index for claim_type searches 2019-07-29 14:55:07 -04:00
Jack Robison 726129836a Merge pull request #2323 from lbryio/refactor_ttfb
refactor and fix ttfb for new api
2019-07-28 22:28:02 -04:00
Jack Robison 356b3c85e5 use app default order_by 2019-07-28 21:42:04 -04:00
Jack Robison 1a0680ead9 update claim_search_performance.py, add new test cases shown by time_to_first_byte.py 2019-07-28 21:22:07 -04:00
Jack Robison c876d891fa use perf_counter 2019-07-28 21:18:29 -04:00
Jack Robison a08232f164 fix claim search kwargs in time to first byte script 2019-07-28 21:18:13 -04:00
Victor Shyba 0f8cf46626 refactor and fix ttfb for new api 2019-07-28 17:33:37 -04:00
Alex Grin 95017b5fc8 v0.38.5 2019-07-25 13:22:54 -04:00
Jack Robison 1374d337d8 Merge pull request #2348 from lbryio/probemon
Probe if peers can be connected to during blob downloads to more quickly skip bad peers
2019-07-25 13:21:42 -04:00
Jack Robison 35e7cc685f fix logging error 2019-07-24 22:12:51 -04:00
Victor Shyba 2ac4883bda cleanup and fix prober 2019-07-24 22:32:20 -03:00
Victor Shyba 5a7212aebb gotta probe 'em all 2019-07-24 20:26:38 -03:00
Lex Berezhny 3f39cc2955 v0.38.4 2019-07-24 18:02:03 -04:00
Lex Berezhny 8a5bfe884a save channel keys into wallet before broadcasting 2019-07-24 18:00:06 -04:00
Lex Berezhny 046dd53913 add index to filter by fee_amount and order by release_time 2019-07-24 14:27:59 -04:00
Alex Grintsvayg d2f174635c add script to monitor wallet servers for slow queries 2019-07-24 13:36:22 -04:00
Lex Berezhny 01a760d8f0 v0.38.3 2019-07-23 17:44:38 -04:00
Victor Shyba 9215d2c9f9 log that checkpoint succeeded 2019-07-23 17:44:13 -04:00
Victor Shyba 0bb09ae91e use checkpoint to speed up 2019-07-23 17:44:13 -04:00
Victor Shyba afeff61268 fix repair and tests 2019-07-23 17:44:13 -04:00
Victor Shyba 6f5a595975 check from final size instead of partials 2019-07-23 17:44:13 -04:00
Victor Shyba 8a69ec062e download headers to disk and repair instead of connecting each 2019-07-23 17:44:13 -04:00
Victor Shyba 45fb673e49 dont fail when s3 is out of reach 2019-07-23 17:44:13 -04:00
Lex Berezhny 93b5f1809e revert to v0.38.2 so we can release v0.38.3 again 2019-07-23 17:43:04 -04:00
Alex Grin ff05eaae48 v0.38.3 2019-07-23 11:14:20 -04:00
Alex Grintsvayg b48b67ed10 fix aws key 2019-07-23 11:14:00 -04:00
Alex Grintsvayg 3ead6f63b6 add lbry-ci bot auth token 2019-07-23 11:04:10 -04:00
Alex Grin 77235f0ad8 v0.38.2 2019-07-22 17:45:43 -04:00
Alex Grintsvayg f6a80ba929 Revert "v0.38.2"
This reverts commit 295ceb90d5.
2019-07-22 17:45:12 -04:00
Alex Grin 295ceb90d5 v0.38.2 2019-07-22 13:31:20 -04:00
Victor Shyba df41a82b23 fix integration test 2019-07-22 11:31:45 -04:00
Victor Shyba 34500e07c6 change picking logic to probe the session before considering it a good session 2019-07-22 11:31:45 -04:00
Alex Grintsvayg 1e6097e3b2 more spv dns entries 2019-07-22 11:24:12 -04:00
Lex Berezhny 521facedc4 script to find max server load 2019-07-22 00:15:05 -04:00
Lex Berezhny de80a2ba97 fixed 0 results when only filtering by lots of common tags 2019-07-21 19:44:10 -04:00
Lex Berezhny 3afd615447 dont run common tag optimization strategy if there are 5 or more common tags provided 2019-07-20 15:45:15 -04:00
Lex Berezhny 87e52bec0a successfuly run indexes after full wallet server sync 2019-07-20 15:45:15 -04:00
Lex Berezhny b89f04b1f1 fix claim integration tests 2019-07-20 15:45:15 -04:00
Lex Berezhny e4f802e5f4 when resolving inside channel break a tie of claims with same effective_amount by using height 2019-07-20 15:45:15 -04:00
Lex Berezhny 16fc2c456f moved executing create index queries outside of a tx 2019-07-20 15:45:15 -04:00
Lex Berezhny ef7656ae98 removed some dead code 2019-07-20 15:45:15 -04:00
Lex Berezhny ad2df4871e popular tags are indexed and use different search strategy than less popular tags 2019-07-20 15:45:15 -04:00
Jack Robison 8d5f50b695 improve claim search test script to show sql for nearly interrupted queries 2019-07-19 14:17:21 -04:00
Lex Berezhny bbc056eef0 added better stats to sqlite_perf_test.py 2019-07-19 11:26:50 -04:00
Jack Robison 82fd3c632e another claim search performance test case 2019-07-19 10:50:37 -04:00
Jack Robison dbae939808 claim search performance test 2019-07-19 10:42:38 -04:00
Jack Robison c2dcacb0de more test cases 2019-07-19 10:37:15 -04:00
Lex Berezhny 8f45618cd0 added improved effective_amount index covering release_time 2019-07-18 23:24:00 -04:00
Lex Berezhny 29a87c850c fix sqlite_perf_test 2019-07-18 19:54:38 -04:00
Lex Berezhny 395518ff76 ver updates to new metrics 2019-07-18 19:54:38 -04:00
Lex Berezhny 08da763327 more metrics cleanup 2019-07-18 19:54:38 -04:00
Lex Berezhny d7ec3c182d tuples 2019-07-18 19:54:38 -04:00
Lex Berezhny ffe6418392 fix metrics json response test 2019-07-18 19:54:38 -04:00
Lex Berezhny 88adf3ee27 oops 2019-07-18 19:54:38 -04:00
Lex Berezhny f406db85dc pylint again 2019-07-18 19:54:38 -04:00
Lex Berezhny d5f8af471a pylint 2019-07-18 19:54:38 -04:00
Lex Berezhny 046459a72d refactored websockets and metrics 2019-07-18 19:54:38 -04:00
Lex Berezhny 49b0f59790 added metrics module 2019-07-18 19:54:38 -04:00
Lex Berezhny 71551058ce query interpolation 2019-07-18 19:54:38 -04:00
43 changed files with 1810 additions and 742 deletions

No files matched your search

+2 -1
View File
@@ -78,7 +78,8 @@ jobs:
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
file: lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
@@ -20,14 +20,11 @@ class ServerConnection {
channel = IOWebSocketChannel.connect(this.url);
int tick = 1;
channel.stream.listen((message) {
Map data = json.decode(message);
var data = json.decode(message);
print(data);
Map commands = data['commands'] ?? {};
_loadDataController.add(
ServerLoadDataPoint(
tick,
APICallMetrics.from_map(commands['search'] ?? {}),
APICallMetrics.from_map(commands['resolve'] ?? {})
ServerLoadDataPoint.from_map(
tick, data
)
);
tick++;
@@ -52,48 +49,106 @@ class ServerConnection {
}
class TimeStats {
final int avg, min, max;
// percentiles
final int five;
final int twenty_five;
final int fifty;
final int seventy_five;
final int ninety_five;
TimeStats(
this.avg, this.min, this.five, this.twenty_five,
this.fifty, this.seventy_five, this.ninety_five,
this.max
);
TimeStats.from_list(List l): this(
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
);
TimeStats.from_zeros(): this(
0, 0, 0, 0, 0, 0, 0, 0
);
factory TimeStats.from_list_or_zeros(List l) =>
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
}
class APICallMetrics {
final int started;
final int finished;
final int total_time;
final int execution_time;
final int query_time;
final int query_count;
final int cache_hit;
final int avg_wait_time;
final int avg_total_time;
final int avg_execution_time;
final int avg_query_time_per_search;
final int avg_query_time_per_query;
// total requests received
final int receive_count;
// sum of these is total responses made
final int cache_response_count;
final int query_response_count;
final int intrp_response_count;
final int error_response_count;
// stacked values for chart
final int cache_response_stack;
final int query_response_stack;
final int intrp_response_stack;
final int error_response_stack;
// millisecond timings for non-cache responses
final TimeStats response;
final TimeStats interrupt;
final TimeStats error;
// response, interrupt and error each also report the python, wait and sql stats:
final TimeStats python;
final TimeStats wait;
final TimeStats sql;
// extended timings for individual sql executions
final TimeStats individual_sql;
final int individual_sql_count;
// actual queries
final List<String> errored_queries;
final List<String> interrupted_queries;
APICallMetrics(
this.started, this.finished, this.total_time, this.execution_time,
this.query_time, this.query_count, this.cache_hit):
avg_wait_time=finished > 0 ? ((total_time - (execution_time + query_time))/finished).round() : 0,
avg_total_time=finished > 0 ? (total_time/finished).round() : 0,
avg_execution_time=finished > 0 ? (execution_time/finished).round() : 0,
avg_query_time_per_search=finished > 0 ? (query_time/finished).round() : 0,
avg_query_time_per_query=query_count > 0 ? (query_time/query_count).round() : 0;
this.receive_count,
this.cache_response_count, this.query_response_count,
this.intrp_response_count, this.error_response_count,
this.response, this.interrupt, this.error,
this.python, this.wait, this.sql,
this.individual_sql, this.individual_sql_count,
this.errored_queries, this.interrupted_queries
):
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
query_response_stack=query_response_count+intrp_response_count+error_response_count,
intrp_response_stack=intrp_response_count+error_response_count,
error_response_stack=error_response_count;
APICallMetrics.from_map(Map data): this(
data['started'] ?? 0,
data['finished'] ?? 0,
data['total_time'] ?? 0,
data['execution_time'] ?? 0,
data['query_time'] ?? 0,
data['query_count'] ?? 0,
data['cache_hit'] ?? 0,
data["receive_count"] ?? 0,
data["cache_response_count"] ?? 0,
data["query_response_count"] ?? 0,
data["intrp_response_count"] ?? 0,
data["error_response_count"] ?? 0,
TimeStats.from_list_or_zeros(data["response"]),
TimeStats.from_list_or_zeros(data["interrupt"]),
TimeStats.from_list_or_zeros(data["error"]),
TimeStats.from_list_or_zeros(data["python"]),
TimeStats.from_list_or_zeros(data["wait"]),
TimeStats.from_list_or_zeros(data["sql"]),
TimeStats.from_list_or_zeros(data["individual_sql"]),
data["individual_sql_count"] ?? 0,
List<String>.from(data["errored_queries"] ?? const []),
List<String>.from(data["interrupted_queries"] ?? const []),
);
}
class ServerLoadDataPoint {
final int tick;
final int sessions;
final APICallMetrics search;
final APICallMetrics resolve;
ServerLoadDataPoint(this.tick, this.search, this.resolve);
ServerLoadDataPoint.empty():
tick = 0,
search=APICallMetrics.from_map({}),
resolve=APICallMetrics.from_map({});
const ServerLoadDataPoint(
this.tick, this.sessions, this.search, this.resolve
);
ServerLoadDataPoint.from_map(int tick, Map data): this(
tick, (data['status'] ?? const {})['sessions'] ?? 0,
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
);
ServerLoadDataPoint.empty(): this(
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
);
}
@@ -13,7 +13,12 @@ class ServerCharts extends StatelessWidget {
Widget build(BuildContext context) {
var server = Provider.of<Server>(context, listen: false);
return ListView(children: <Widget>[
SizedBox(height: 300.0, child: ServerLoadChart(server)),
SizedBox(height: 300.0, child: APILoadChart(
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
)),
//SizedBox(height: 300.0, child: APILoadChart(
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
//)),
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
@@ -22,16 +27,20 @@ class ServerCharts extends StatelessWidget {
}
class ServerLoadChart extends StatefulWidget {
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
class APILoadChart extends StatefulWidget {
final Server server;
ServerLoadChart(this.server);
final String name;
final APIGetter getter;
APILoadChart(this.server, this.name, this.getter);
@override
State<StatefulWidget> createState() => ServerLoadChartState();
State<StatefulWidget> createState() => APILoadChartState();
}
class ServerLoadChartState extends State<ServerLoadChart> {
class APILoadChartState extends State<APILoadChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@@ -39,54 +48,73 @@ class ServerLoadChartState extends State<ServerLoadChart> {
void initState() {
super.initState();
seriesData = [
/*
charts.Series<ServerLoadDataPoint, int>(
id: 'Search Cache',
id: 'Received',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
strokeWidthPxFn: (_, __) => 4.0,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
data: widget.server.serverLoadData,
),*/
charts.Series<ServerLoadDataPoint, int>(
id: 'Cache',
colorFn: (_, __) =>
charts.MaterialPalette.green.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Query',
colorFn: (_, __) =>
charts.MaterialPalette.blue.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Interrupts',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Errors',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
data: widget.server.serverLoadData,
),
/*
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Interrupted',
colorFn: (_, __) =>
charts.MaterialPalette.pink.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Errored',
colorFn: (_, __) =>
charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} From Cache',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.cache_hit,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Search Finish',
colorFn: (_, __) =>
charts.MaterialPalette.deepOrange.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.finished,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Search Start',
colorFn: (_, __) =>
charts.MaterialPalette.deepOrange.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.started,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Cache',
colorFn: (_, __) => charts.MaterialPalette.cyan.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.cache_hit,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Finish',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.finished,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Resolve Start',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.resolve.started,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
data: widget.server.serverLoadData,
),
*/
];
}
@@ -94,7 +122,9 @@ class ServerLoadChartState extends State<ServerLoadChart> {
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
builder: (BuildContext context, _) => BetterLineChart(seriesData,
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
)
);
}
}
@@ -117,27 +147,41 @@ class ServerPerformanceChartState extends State<ServerPerformanceChart> {
void initState() {
super.initState();
seriesData = [
charts.Series<ServerLoadDataPoint, int>(
id: 'Waiting 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Waiting',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_wait_time,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Executing',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_execution_time,
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'SQLite 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. SQLite',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_query_time_per_search,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
data: widget.server.serverLoadData,
)
),
];
}
@@ -260,11 +304,13 @@ class BetterLineChart extends charts.LineChart {
final int itemCount;
final Object lastItem;
BetterLineChart(List<charts.Series<dynamic, int>> seriesList):
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(
seriesList,
animate: false,
defaultRenderer: renderer,
behaviors: [charts.SeriesLegend()],
domainAxis: charts.NumericAxisSpec(
viewport: new charts.NumericExtents(
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.1"
__version__ = "0.38.6"
version = tuple(__version__.split('.'))
+5 -3
View File
@@ -187,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return await self._download_blob()
except OSError as e:
# i'm not sure how to fix this race condition - jack
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
@@ -220,7 +220,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
async def request_blob(loop: asyncio.BaseEventLoop, blob: typing.Optional['AbstractBlob'], address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
@@ -242,7 +242,9 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', addres
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or not blob.is_writeable():
connected_transport = protocol.transport
if blob is None or blob.get_is_verified() or not blob.is_writeable():
# blob is None happens when we are just opening a connection
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
+13 -9
View File
@@ -30,17 +30,19 @@ class BlobDownloader:
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
if len(self.active_connections) >= max_probes:
return False
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0,
just_probe: bool = False):
if blob.get_is_verified():
return
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
@@ -91,16 +93,18 @@ class BlobDownloader:
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
"running, %d peers, %d ignored, %d active, %s connections",
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
)
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if peer in self.ignored or peer in self.active_connections:
continue
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
just_probe = len(self.connections) == 0
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
self.active_connections[peer] = t
await self.new_peer_or_finished()
self.cleanup_active()
log.debug("downloaded %s", blob_hash[:8])
+3
View File
@@ -551,6 +551,9 @@ class Config(CLIConfig):
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
+50 -40
View File
@@ -1,3 +1,4 @@
import hashlib
import os
import asyncio
import logging
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
self.storage = None
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
HEADER_SIZE = 112
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
} if progress is not None and progress < 100 else {}
async def fetch_headers_from_s3(self):
local_header_size = self.headers.bytes_size
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
log.warning("s3 is more out of date than we are")
return
if response.content_length % HEADER_SIZE != 0:
final_size_after_download = response.content_length + local_header_size
if final_size_after_download % self.headers.header_size != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
buffer, header_size = b'', self.headers.header_size
async for chunk in response.content.iter_any():
chunk = buffer + chunk
remaining = len(chunk) % header_size
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
if not chunk:
continue
if not await self.headers.connect(len(self.headers), chunk):
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
return
self._headers_progress_percent = self._round_progress(
self.headers.bytes_size, final_size_after_download
)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while not response.content.at_eof():
local_header_size += fd.write(await response.content.readany())
self._headers_progress_percent = self._round_progress(
local_header_size, final_size_after_download
)
def local_header_file_size(self) -> int:
if os.path.isfile(self.headers_file):
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
return 0
async def get_downloadable_header_height(self) -> typing.Optional[int]:
try:
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return response.content_length // HEADER_SIZE
except OSError:
log.exception("Failed to download headers using https.")
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return -1
return response.content_length // self.headers.header_size
async def should_download_headers_from_s3(self) -> bool:
if self.conf.blockchain_name != "lbrycrd_main":
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
if not s3_headers_depth:
return False
local_height = self.headers.height
local_height = self.local_header_file_size() // self.headers.header_size
remote_height = await self.get_downloadable_header_height()
if remote_height is not None:
log.info("remote height: %i, local height: %i", remote_height, local_height)
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
return True
return False
def verify_checkpoint(self):
expected_hash, at_height = self.CHECKPOINT
if self.local_header_file_size() // self.headers.header_size < at_height:
return False
hash = hashlib.sha256()
chunk_size = self.headers.header_size * 1000
with open(self.headers_file, 'rb') as header_file:
data = header_file.read(chunk_size)
while data and header_file.tell() <= at_height * self.headers.header_size:
hash.update(data)
data = header_file.read(chunk_size)
return hash.hexdigest() == expected_hash
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
await self.headers.open()
await self.headers.repair()
if await self.should_download_headers_from_s3():
try:
try:
if await self.should_download_headers_from_s3():
self.is_downloading_headers = True
await self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
await self.headers.close()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
# fixme: workaround, this should happen before download but happens after because headers.connect fail
if not self.verify_checkpoint():
log.info("Checkpoint failed, verifying headers using slower method.")
await self.headers.open()
await self.headers.repair()
await self.headers.close()
else:
log.info("Header checkpoint verified.")
async def stop(self):
pass
+39 -8
View File
@@ -33,7 +33,7 @@ from lbry.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_
from lbry.extras.daemon.ComponentManager import RequiredCondition
from lbry.extras.daemon.ComponentManager import ComponentManager
from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder
from lbry.extras.daemon.comment_client import jsonrpc_post, sign_comment, is_comment_signed_by_channel
from lbry.extras.daemon import comment_client
from lbry.extras.daemon.undecorated import undecorated
from lbry.wallet.transaction import Transaction, Output, Input
from lbry.wallet.account import Account as LBCAccount
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(new_txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
]
}
"""
result = await jsonrpc_post(
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
"get_claim_comments",
claim_id=claim_id,
@@ -3432,7 +3432,7 @@ class Daemon(metaclass=JSONRPCServerType):
continue
resolve_response = await self.resolve([channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
@@ -3490,12 +3490,43 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
sign_comment(comment_body, channel)
response = await jsonrpc_post(self.conf.comment_server, 'create_comment', **comment_body)
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
if 'signature' in response:
response['is_claim_signature_valid'] = is_comment_signed_by_channel(response, channel)
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
return response
async def jsonrpc_comment_abandon(self, comment_id):
""""
Delete a comment published under your channel identity
Usage:
comment_delete (<comment_id> | --comment_id=<comment_id>)
Options:
--comment_id=<comment_id> : (str) The ID of the comment to be deleted.
Returns:
(dict) Object with the `comment_id` passed in as the key, and a flag indicating if it was deleted
{
<comment_id> (str): {
"deleted": (bool)
}
}
"""
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'deleted': False}}
channel = await self.get_channel_or_none(None, **channel)
abandon_comment_body.update({
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'delete_comment', abandon_comment_body)
async def broadcast_or_release(self, account, tx, blocking=False):
try:
+20 -10
View File
@@ -18,13 +18,18 @@ def get_encoded_signature(signature):
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
def is_comment_signed_by_channel(comment: dict, channel: Output):
def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
if type(channel) is Output:
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
channel.claim_hash,
comment['comment'].encode()
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
@@ -36,16 +41,21 @@ def is_comment_signed_by_channel(comment: dict, channel: Output):
return False
def sign_comment(comment: dict, channel: Output):
timestamp = str(int(time.time())).encode()
pieces = [timestamp, channel.claim_hash, comment['comment'].encode()]
def sign_comment(comment: dict, channel: Output, abandon=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment['signature'] = binascii.hexlify(signature).decode()
comment['signing_ts'] = timestamp.decode()
comment.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, **params) -> any:
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
headers = {'Content-Type': 'application/json'}
async with utils.aiohttp_request('POST', url, json=json_body, headers=headers) as response:
@@ -53,5 +63,5 @@ async def jsonrpc_post(url: str, method: str, **params) -> any:
result = await response.json()
return result['result'] if 'result' in result else result
except Exception as cte:
log.exception('Unable to decode respose from server: %s', cte)
log.exception('Unable to decode response from server: %s', cte)
return await response.text()
+1 -1
View File
@@ -6,7 +6,7 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
def clean_tags(tags: List[str]):
@@ -79,6 +79,10 @@ class LBRYBlockProcessor(BlockProcessor):
raise
finally:
self.sql.commit()
if self.db.first_sync and self.height == self.daemon.cached_height():
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
+207
View File
@@ -11,3 +11,210 @@ STREAM_TYPES = {
'binary': 5,
'model': 6
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
+77 -39
View File
@@ -9,14 +9,20 @@ from contextvars import ContextVar
from functools import wraps
from dataclasses import dataclass
from torba.client.basedatabase import query
from torba.client.basedatabase import query, interpolate
from lbry.schema.url import URL, normalize_name
from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
class SQLiteOperationalError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query errored')
self.metrics = metrics
class SQLiteInterruptedError(sqlite3.OperationalError):
@@ -50,7 +56,7 @@ SEARCH_PARAMS = {
ORDER_FIELDS = {
'name',
'name', 'claim_hash'
} | INTEGER_PARAMS
@@ -91,7 +97,6 @@ ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
db = sqlite3.connect(_path, isolation_level=None, uri=True)
db.row_factory = sqlite3.Row
ctx.set(
@@ -114,21 +119,16 @@ def measure(func):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
metric = state.metrics.setdefault(func.__name__, {
'calls': 0, 'total': 0, 'isolated': 0, 'errors': 0
})
metric = {}
state.metrics.setdefault(func.__name__, []).append(metric)
state.stack.append([])
start = time.perf_counter()
try:
return func(*args, **kwargs)
except:
metric['errors'] += 1
raise
finally:
elapsed = int((time.perf_counter()-start)*1000)
metric['calls'] += 1
metric['total'] += elapsed
metric['isolated'] += (elapsed-sum(state.stack.pop()))
metric['total'] = elapsed
metric['isolated'] = (elapsed-sum(state.stack.pop()))
if state.stack:
state.stack[-1].append(elapsed)
return wrapper
@@ -167,16 +167,17 @@ def execute_query(sql, values) -> List:
try:
return context.db.execute(sql, values).fetchall()
except sqlite3.OperationalError as err:
plain_sql = interpolate(sql, values)
if context.is_tracking_metrics:
context.metrics['execute_query'][-1]['sql'] = plain_sql
if str(err) == "interrupted":
query_str = sql
for k in sorted(values.keys(), reverse=True):
query_str = query_str.replace(f":{k}", str(values[k]) if not k.startswith("$") else f"'{values[k]}'")
context.log.warning("interrupted slow sqlite query:\n%s", query_str)
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
raise SQLiteInterruptedError(context.metrics)
context.log.exception('failed running query', exc_info=err)
raise SQLiteOperationalError(context.metrics)
@measure
def get_claims(cols, for_count=False, **constraints) -> List:
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'order_by' in constraints:
sql_order_by = []
for order_by in constraints['order_by']:
@@ -231,14 +232,6 @@ def get_claims(cols, for_count=False, **constraints) -> List:
if 'public_key_id' in constraints:
constraints['claim.public_key_hash'] = sqlite3.Binary(
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
if 'channel_hash' in constraints:
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_ids' in constraints:
@@ -307,7 +300,18 @@ def get_claims(cols, for_count=False, **constraints) -> List:
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)
return sql, values
def get_claims(cols, for_count=False, **constraints) -> List:
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
sql, values = _get_claims(cols, for_count, **constraints)
return execute_query(sql, values)
@@ -330,7 +334,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["height", "^name"]
constraints['order_by'] = ["claim_hash"]
txo_rows = _search(**constraints)
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
extra_txo_rows = []
@@ -404,7 +408,7 @@ def resolve_url(raw_url):
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount']
query['order_by'] = ['effective_amount', '^height']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
@@ -420,7 +424,6 @@ def resolve_url(raw_url):
return channel
@measure
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
@@ -429,26 +432,61 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
if attr == 'tag':
common_tags = any_items & COMMON_TAGS.keys()
if common_tags:
any_items -= common_tags
if len(common_tags) < 5:
for item in common_tags:
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
elif len(common_tags) >= 5:
constraints.update({
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
})
values = ', '.join(
f':$any_common_tag{i}' for i in range(len(common_tags))
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count:
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
constraints[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
+37 -25
View File
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
)
from .common import CLAIM_TYPES, STREAM_TYPES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -76,30 +76,12 @@ class SQLDB:
trending_global integer not null default 0
);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_height_idx on claim (height);
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
"""
CREATE_SUPPORT_TABLE = """
@@ -110,7 +92,6 @@ class SQLDB:
claim_hash bytes not null,
amount integer not null
);
create index if not exists support_txo_hash_idx on support (txo_hash);
create index if not exists support_claim_hash_idx on support (claim_hash, height);
"""
@@ -120,7 +101,6 @@ class SQLDB:
claim_hash bytes not null,
height integer not null
);
create index if not exists tag_tag_idx on tag (tag);
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
@@ -133,6 +113,38 @@ class SQLDB:
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
TAG_INDEXES = '\n'.join(
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
for tag_value, tag_key in COMMON_TAGS.items()
)
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_CLAIM_TABLE +
@@ -257,7 +269,7 @@ class SQLDB:
if tags:
self.db.executemany(
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
return claims
+135
View File
@@ -0,0 +1,135 @@
import time
import math
from typing import Tuple
def calculate_elapsed(start) -> int:
return int((time.perf_counter() - start) * 1000)
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
if not data:
return 0, 0, 0, 0, 0, 0, 0, 0
data.sort()
size = len(data)
return (
int(sum(data) / size),
data[0],
data[math.ceil(size * .05) - 1],
data[math.ceil(size * .25) - 1],
data[math.ceil(size * .50) - 1],
data[math.ceil(size * .75) - 1],
data[math.ceil(size * .95) - 1],
data[-1]
)
def remove_select_list(sql) -> str:
return sql[sql.index('FROM'):]
class APICallMetrics:
def __init__(self, name):
self.name = name
# total requests received
self.receive_count = 0
self.cache_response_count = 0
# millisecond timings for query based responses
self.query_response_times = []
self.query_intrp_times = []
self.query_error_times = []
self.query_python_times = []
self.query_wait_times = []
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
self.individual_sql_times = [] # every SQL query run on server
# actual queries
self.errored_queries = set()
self.interrupted_queries = set()
def to_json_and_reset(self):
return {
# total requests received
"receive_count": self.receive_count,
# sum of these is total responses made
"cache_response_count": self.cache_response_count,
"query_response_count": len(self.query_response_times),
"intrp_response_count": len(self.query_intrp_times),
"error_response_count": len(self.query_error_times),
# millisecond timings for non-cache responses
"response": calculate_avg_percentiles(self.query_response_times),
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
"error": calculate_avg_percentiles(self.query_error_times),
# response, interrupt and error each also report the python, wait and sql stats:
"python": calculate_avg_percentiles(self.query_python_times),
"wait": calculate_avg_percentiles(self.query_wait_times),
"sql": calculate_avg_percentiles(self.query_sql_times),
# extended timings for individual sql executions
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
"individual_sql_count": len(self.individual_sql_times),
# actual queries
"errored_queries": list(self.errored_queries),
"interrupted_queries": list(self.interrupted_queries),
}
def start(self):
self.receive_count += 1
def cache_response(self):
self.cache_response_count += 1
def _add_query_timings(self, request_total_time, metrics):
if metrics and 'execute_query' in metrics:
sub_process_total = metrics[self.name][0]['total']
individual_query_times = [f['total'] for f in metrics['execute_query']]
aggregated_query_time = sum(individual_query_times)
self.individual_sql_times.extend(individual_query_times)
self.query_sql_times.append(aggregated_query_time)
self.query_python_times.append(sub_process_total - aggregated_query_time)
self.query_wait_times.append(request_total_time - sub_process_total)
@staticmethod
def _add_queries(query_set, metrics):
if metrics and 'execute_query' in metrics:
for execute_query in metrics['execute_query']:
if 'sql' in execute_query:
query_set.add(remove_select_list(execute_query['sql']))
def query_response(self, start, metrics):
self.query_response_times.append(calculate_elapsed(start))
self._add_query_timings(self.query_response_times[-1], metrics)
def query_interrupt(self, start, metrics):
self.query_intrp_times.append(calculate_elapsed(start))
self._add_queries(self.interrupted_queries, metrics)
self._add_query_timings(self.query_intrp_times[-1], metrics)
def query_error(self, start, metrics):
self.query_error_times.append(calculate_elapsed(start))
self._add_queries(self.errored_queries, metrics)
self._add_query_timings(self.query_error_times[-1], metrics)
class ServerLoadData:
def __init__(self):
self._apis = {}
def for_api(self, name) -> APICallMetrics:
if name not in self._apis:
self._apis[name] = APICallMetrics(name)
return self._apis[name]
def to_json_and_reset(self, status):
try:
return {
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
'status': status
}
finally:
self._apis = {}
+31 -137
View File
@@ -4,13 +4,9 @@ import time
import base64
import asyncio
from binascii import hexlify
from weakref import WeakSet
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
from torba.rpc.jsonrpc import RPCError, JSONRPC
from torba.server.session import ElectrumX, SessionManager
from torba.server import util
@@ -18,56 +14,8 @@ from torba.server import util
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYDB
from lbry.wallet.server.db import reader
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
class ResultCacheItem:
@@ -95,9 +43,8 @@ class LBRYSessionManager(SessionManager):
super().__init__(*args, **kwargs)
self.query_executor = None
self.websocket = None
self.metrics_processor = None
self.command_metrics = {}
self.reader_metrics = {}
self.metrics = ServerLoadData()
self.metrics_loop = None
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
@@ -105,69 +52,11 @@ class LBRYSessionManager(SessionManager):
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
def get_command_tracking_info(self, command):
if command not in self.command_metrics:
self.command_metrics[command] = {
'cache_hit': 0,
'started': 0,
'finished': 0,
'total_time': 0,
'execution_time': 0,
'query_time': 0,
'query_count': 0,
'interrupted': 0,
'interrupted_query_values': [],
}
return self.command_metrics[command]
def cache_hit(self, command_name):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['cache_hit'] += 1
def start_command_tracking(self, command_name):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['started'] += 1
def finish_command_tracking(self, command_name, elapsed, metrics):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['finished'] += 1
command['total_time'] += elapsed
if 'execute_query' in metrics:
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
command['query_time'] += metrics['execute_query']['total']
command['query_count'] += metrics['execute_query']['calls']
for func_name, func_metrics in metrics.items():
reader = self.reader_metrics.setdefault(func_name, {})
for key in func_metrics:
if key not in reader:
reader[key] = func_metrics[key]
else:
reader[key] += func_metrics[key]
def interrupted_command_error(self, command_name, elapsed, metrics, kwargs):
if self.env.track_metrics:
command = self.get_command_tracking_info(command_name)
command['finished'] += 1
command['interrupted'] += 1
command['total_time'] += elapsed
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
command['query_time'] += metrics['execute_query']['total']
command['query_count'] += metrics['execute_query']['calls']
if len(command['interrupted_query_values']) < 100:
command['interrupted_query_values'].append(kwargs)
async def process_metrics(self):
while self.running:
commands, self.command_metrics = self.command_metrics, {}
reader, self.reader_metrics = self.reader_metrics, {}
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
if self.websocket is not None:
self.websocket.send_message({
'commands': commands,
'reader': reader
})
self.websocket.send_message(data)
await asyncio.sleep(1)
async def start_other(self):
@@ -186,12 +75,12 @@ class LBRYSessionManager(SessionManager):
if self.websocket is not None:
await self.websocket.start()
if self.env.track_metrics:
self.metrics_processor = asyncio.create_task(self.process_metrics())
self.metrics_loop = asyncio.create_task(self.process_metrics())
async def stop_other(self):
self.running = False
if self.env.track_metrics:
self.metrics_processor.cancel()
self.metrics_loop.cancel()
if self.websocket is not None:
await self.websocket.stop()
self.query_executor.shutdown()
@@ -220,45 +109,50 @@ class LBRYElectrumX(ElectrumX):
}
self.request_handlers.update(handlers)
async def run_in_executor(self, name, func, kwargs):
start = None
if self.env.track_metrics:
self.session_mgr.start_command_tracking(name)
start = time.perf_counter()
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
start = time.perf_counter()
try:
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
except reader.SQLiteInterruptedError as error:
self.session_mgr.interrupted_command_error(
name, int((time.perf_counter() - start) * 1000), error.metrics, kwargs
)
metrics.query_interrupt(start, error.metrics)
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics.query_error(start, error.metrics)
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except:
metrics.query_error(start, {})
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
if self.env.track_metrics:
elapsed = int((time.perf_counter() - start) * 1000)
(result, metrics) = result
self.session_mgr.finish_command_tracking(name, elapsed, metrics)
(result, metrics_data) = result
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
async def run_and_cache_query(self, query_name, function, kwargs):
if self.env.track_metrics:
metrics = self.session_mgr.metrics.for_api(query_name)
else:
metrics = APICallMetrics(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
self.session_mgr.cache_hit(query_name)
metrics.cache_response()
return cache_item.result
async with cache_item.lock:
result = cache_item.result
if result is None:
result = cache_item.result = await self.run_in_executor(
query_name, function, kwargs
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
metrics, function, kwargs
)
else:
self.session_mgr.cache_hit(query_name)
return result
metrics.cache_response()
return cache_item.result
async def claimtrie_search(self, **kwargs):
if kwargs:
+55
View File
@@ -0,0 +1,55 @@
import asyncio
from weakref import WeakSet
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
+104 -44
View File
@@ -1,15 +1,16 @@
import sys
import os
import time
import textwrap
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
log.setLevel(logging.CRITICAL)
DEFAULT_ANY_TAGS = [
'blockchain',
@@ -24,6 +25,25 @@ DEFAULT_ANY_TAGS = [
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
@@ -47,46 +67,48 @@ ORDER_BY = [
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, []]
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_totals in [True]:
for offset in [0, 20, 40, 60, 80, 100, 1000, 2000, 3000]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
args.append(kw)
print(len(args), "argument combinations")
for no_fee in [False, True]:
for claim_type in [None, 'stream', 'channel']:
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
if claim_type:
kw['claim_type'] = claim_type
if no_fee:
kw['fee_amount'] = 0
args.append(kw)
print(f"-- Trying {len(args)} argument combinations")
return args
def _search(kwargs):
start = time.time()
msg = f"offset={kwargs['offset']}, limit={kwargs['limit']}, no_totals={kwargs['no_totals']}, not_tags={kwargs.get('not_tags')}, any_tags={kwargs.get('any_tags')}, order_by={kwargs['order_by']}"
start = time.perf_counter()
error = None
try:
search_to_bytes(kwargs)
t = time.time() - start
return t, f"{t} - {msg}"
except Exception as err:
return -1, f"failed: error={str(type(err))}({str(err)}) - {msg}"
error = str(err)
return time.perf_counter() - start, kwargs, error
async def search(executor, kwargs):
try:
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
except Exception as err:
return f"failed (err={str(type(err))}({err}))- {kwargs}"
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
async def main(db_path, max_query_time):
@@ -97,21 +119,59 @@ async def main(db_path, max_query_time):
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
times = {msg: ts for ts, msg in results}
log.info("\n".join(sorted(filter(lambda msg: times[msg] > max_query_time, times.keys()), key=lambda msg: times[msg])))
query_times = [
{
'sql': interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints)),
'duration': ts,
'error': error
}
for ts, constraints, error in results
]
errored = [query_info for query_info in query_times if query_info['error']]
errors = {str(query_info['error']): [] for query_info in errored}
for error in errored:
errors[str(error['error'])].append(error['sql'])
slow = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] > (max_query_time / 2.0)
]
fast = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] <= (max_query_time / 2.0)
]
print(f"-- {len(fast)} queries were fast")
slow.sort(key=lambda query_info: query_info['duration'], reverse=True)
print(f"-- Failing queries:")
for error in errors:
print(f"-- Failure: \"{error}\"")
for failing_query in errors[error]:
print(f"{textwrap.dedent(failing_query)};\n")
print()
print(f"-- Slow queries:")
for slow_query in slow:
print(f"-- Query took {slow_query['duration']}\n{textwrap.dedent(slow_query['sql'])};\n")
finally:
query_executor.shutdown()
if __name__ == "__main__":
args = sys.argv[1:]
if len(args) >= 1:
db_path = args[0]
else:
db_path = os.path.expanduser('~/claims.db')
if len(args) >= 2:
max_query_time = float(args[1])
else:
max_query_time = -3
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.25, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
asyncio.run(main(db_path, max_query_time))
+107
View File
@@ -0,0 +1,107 @@
import time
import asyncio
import random
from argparse import ArgumentParser
from torba.client.basenetwork import ClientSession
class AgentSmith(ClientSession):
async def do_nefarious_things(self):
await self.send_request('blockchain.claimtrie.search', {
'no_totals': True,
'offset': random.choice(range(0, 300, 20)),
'limit': 20,
'any_tags': (
random.choice([[
random.choice(['gaming', 'games', 'game']) +
random.choice(['entertainment', 'playthrough', 'funny']) +
random.choice(['xbox', 'xbox one', 'xbox news'])
], [
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
random.choice(['news', 'sighting', 'sightings'])
], [
random.choice(['art', 'automotive']),
random.choice(['blockchain', 'economics', 'food']),
random.choice(['funny', 'learnings', 'nature']),
random.choice(['news', 'science', 'technology'])
]
])
),
'not_tags': random.choice([[], [
'porn', 'mature', 'xxx', 'nsfw'
]]),
'order_by': random.choice([
['release_time'],
['trending_global', 'trending_mixed'],
['effective_amount']
])
})
class AgentSmithProgram:
def __init__(self, host, port):
self.host, self.port = host, port
self.agent_smiths = []
async def make_one_more_of_them(self):
smith = AgentSmith(network=None, server=(self.host, self.port))
await smith.create_connection()
self.agent_smiths.append(smith)
async def coordinate_nefarious_activity(self):
start = time.perf_counter()
await asyncio.gather(
*(s.do_nefarious_things() for s in self.agent_smiths),
return_exceptions=True
)
return time.perf_counter() - start
def __len__(self):
return len(self.agent_smiths)
async def delete_one_smith(self):
if self.agent_smiths:
await self.agent_smiths.pop().close()
async def delete_program(self):
await asyncio.gather(*(
s.close() for s in self.agent_smiths
))
async def main(host, port):
smiths = AgentSmithProgram(host, port)
await smiths.make_one_more_of_them()
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
ease_off = 0
for i in range(1000):
await asyncio.sleep(1)
if activity.done() and activity.result() < .9:
print('more, more, more...')
await asyncio.gather(*(
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
))
else:
print('!!!!!!!!!!!!!!')
print('IS NEO LOSING?')
print('!!!!!!!!!!!!!!')
await asyncio.gather(*(
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
))
print(f'coordinate all {len(smiths)} smiths to action')
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
print('finishing up any remaining actions')
await activity
print('neo has won, deleting agents...')
await smiths.delete_program()
print('done.')
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--host', dest='host', default='localhost', type=str)
parser.add_argument('--port', dest='port', default=50001, type=int)
args = parser.parse_args()
asyncio.run(main(args.host, args.port))
+49
View File
@@ -0,0 +1,49 @@
import os, asyncio, aiohttp, json, slack, sqlparse
async def listen(slack_client, url):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
print(f"connecting to {url}")
try:
ws = await session.ws_connect(url)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
print(f"failed to connect to {url}")
return
print(f"connected to {url}")
async for msg in ws:
r = json.loads(msg.data)
try:
queries = r["api"]["search"]["interrupted_queries"]
except KeyError:
continue
for q in queries:
# clean = re.sub(r"\s+", " ", q)
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
print(f'{url}: {clean}')
response = await slack_client.chat_postMessage(
username=url,
icon_emoji=":hourglass_flowing_sand:",
channel='#clubhouse-de-obscure',
text="*Query timed out:* " + clean
)
if not response["ok"]:
print("SLACK ERROR:\n", response)
print()
async def main():
try:
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
except KeyError:
print("Error: SLACK_TOKEN env var required")
return
num_servers = 5
tasks = []
for i in range(1, num_servers+1):
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
await asyncio.gather(*tasks)
asyncio.run(main())
+16 -15
View File
@@ -1,9 +1,10 @@
import uvloop, asyncio, time, sys, logging
from concurrent.futures import ProcessPoolExecutor
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
db_path = '/tmp/wallet-server/claims.db'
db_path = '../../../lbryconf/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
@@ -16,26 +17,26 @@ async def run_times(executor, iterations, show=True):
'no_totals': True,
'offset': 0,
'limit': 20,
'fee_amount': '<1',
#'all_tags': ['funny'],
'any_tags': [
'crypto',
'outdoors',
'cars',
'automotive'
'ufos', 'city fix'
],
'not_tags': [
'nsfw', 'xxx', 'mature'
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query']['total'] for r in timings]
timings = [r[1]['execute_query'][0]['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t}ms')
@@ -44,16 +45,16 @@ async def run_times(executor, iterations, show=True):
async def main():
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', default_query_timout, True)
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
)
await run_times(executor, 4, show=False)
await run_times(executor, 1)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
await run_times(executor, 2**9)
await run_times(executor, 2**11)
await run_times(executor, 2**13)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
+86 -103
View File
@@ -1,43 +1,19 @@
import os
import sys
import json
import argparse
import asyncio
import aiohttp
import time
import aiohttp
from aiohttp import ClientConnectorError
from lbry import __version__
from lbry.blob.blob_file import MAX_BLOB_SIZE
from lbry.conf import Config
from lbry.schema.uri import parse_lbry_uri
from lbry.extras.daemon.client import daemon_rpc
from lbry.extras import system_info
def extract_uris(response):
uris = list()
for key in response:
for value in response[key]:
uris.append(value)
return uris
async def get_frontpage_uris():
session = aiohttp.ClientSession()
try:
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
if response.status != 200:
print("API returned non 200 code!!")
return
body = await response.json()
await session.close()
uris = extract_uris(body['data']['Uris'])
return uris
finally:
await session.close()
async def report_to_slack(output, webhook):
payload = {
"text": f"lbrynet {__version__} ({system_info.get_platform()['platform']}) time to first byte:\n{output}"
@@ -58,120 +34,127 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, uri, timeout):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
async def wait_for_done(conf, claim_name, timeout):
blobs_completed, last_completed = 0, time.perf_counter()
while True:
files = await daemon_rpc(conf, "file_list", claim_name=name)
file = files[0]
file = (await daemon_rpc(conf, "file_list", claim_name=claim_name))[0]
if file['status'] in ['finished', 'stopped']:
return True, file['blobs_completed'], file['blobs_in_stream']
if last_complete < int(file['blobs_completed']):
hang_count = 0
last_complete = int(file['blobs_completed'])
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > timeout:
elif blobs_completed < int(file['blobs_completed']):
blobs_completed, last_completed = int(file['blobs_completed']), time.perf_counter()
elif (time.perf_counter() - last_completed) > timeout:
return False, file['blobs_completed'], file['blobs_in_stream']
await asyncio.sleep(1.0)
async def main(uris=None, cmd_args=None):
if not uris:
uris = await get_frontpage_uris()
async def main(cmd_args=None):
print('Time to first byte started using parameters:')
for key, value in vars(cmd_args).items():
print(f"{key}: {value}")
conf = Config()
url_to_claim = {}
try:
await daemon_rpc(conf, 'status')
for page in range(1, cmd_args.download_pages + 1):
start = time.perf_counter()
kwargs = {
'page': page,
# 'claim_type': 'stream',
'order_by': ['trending_global', 'trending_mixed'],
'no_totals': True
}
# if not cmd_args.allow_fees:
# kwargs['fee_amount'] = 0
response = await daemon_rpc(
conf, 'claim_search', **kwargs
)
if 'error' in response or not response.get('items'):
print(f'Error getting claim list page {page}:')
print(response)
return 1
else:
url_to_claim.update({
claim['permanent_url']: claim for claim in response['items']
})
print(f'Claim search page {page} took: {time.perf_counter() - start}')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print(f"Checking {len(uris)} uris from the front page")
print("**********************************************")
resolvable = []
async def __resolve(name):
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
if 'error' not in resolved.get(name, {}):
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
await asyncio.gather(*(__resolve(name) for name in uris))
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
print(f"Attempting to download {len(url_to_claim)} claim_search streams")
first_byte_times = []
download_speeds = []
download_successes = []
failed_to_start = []
download_failures = []
failed_to = {}
for uri in resolvable:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.gather(*(
daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
for claim in url_to_claim.values() if not cmd_args.keep_files
))
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, uri, cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except Exception as e:
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
failed_to_start.append(uri)
for i, (url, claim) in enumerate(url_to_claim.items()):
start = time.perf_counter()
response = await daemon_rpc(conf, 'get', uri=url, save_file=not cmd_args.head_blob_only)
if 'error' in response:
print(f"{i + 1}/{len(url_to_claim)} - failed to start {url}: {response['error']}")
failed_to[url] = 'start'
if cmd_args.exit_on_error:
return
if cmd_args.delete_after_download or cmd_args.head_blob_only:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
continue
first_byte = time.perf_counter()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(url_to_claim)} - {first_byte - start} {url}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, claim['name'], cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(url)
else:
failed_to[url] = 'finish'
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.perf_counter() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {url} at "
f"{mbs}mb/s")
if not cmd_args.keep_files:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n"
if not cmd_args.head_blob_only:
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
result = f"Started {len(first_byte_times)} of {len(url_to_claim)} attempted front page streams\n"
if first_byte_times:
result += f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n"
if download_successes:
result += f"Downloaded {len(download_successes)}/{len(url_to_claim)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
if download_failures:
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
for reason in ('start', 'finish'):
failures = [url for url, why in failed_to.items() if reason == why]
if failures:
result += f"\nFailed to {reason}:\n" + "\n".join(failures)
print(result)
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
if webhook:
await report_to_slack(result, webhook)
return 0
if __name__ == "__main__":
parser = argparse.ArgumentParser()
#parser.add_argument("--data_dir")
#parser.add_argument("--wallet_dir")
#parser.add_argument("--download_directory")
parser.add_argument("--allow_fees", action='store_true')
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=10, type=int)
parser.add_argument("--delete_after_download", action='store_true')
parser.add_argument("--stall_download_timeout", default=0, type=int)
parser.add_argument("--keep_files", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
asyncio.run(main(cmd_args=parser.parse_args()))
parser.add_argument("--download_pages", type=int, default=10)
sys.exit(asyncio.run(main(cmd_args=parser.parse_args())) or 0)
@@ -89,6 +89,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim], **kwargs)
async def assertFindsClaims(self, claims, **kwargs):
kwargs.setdefault('order_by', ['height', '^name'])
results = await self.claim_search(**kwargs)
self.assertEqual(len(claims), len(results))
for claim, result in zip(claims, results):
@@ -165,15 +166,15 @@ class ClaimSearchCommand(ClaimTestCase):
await self.create_channel()
await self.create_lots_of_streams()
page = await self.claim_search(page_size=20, channel='@abc')
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams)
page = await self.claim_search(page_size=6, channel='@abc')
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[:6])
page = await self.claim_search(page=2, page_size=6, channel='@abc')
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[6:])
+87 -12
View File
@@ -1,3 +1,4 @@
import time
from math import ceil
from aiohttp import web
@@ -14,6 +15,19 @@ class MockedCommentServer:
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
}
COMMENT_SCHEMA = {
'comment': None,
'comment_id': None,
'claim_id': None,
'parent_id': None,
'channel_name': None,
'channel_id': None,
'signature': None,
'signing_ts': None,
'timestamp': None,
'channel_url': None,
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
@@ -23,13 +37,43 @@ class MockedCommentServer:
self.comments = []
self.comment_id = 0
def create_comment(self, **comment):
@classmethod
def _create_comment(cls, **kwargs):
schema = cls.COMMENT_SCHEMA.copy()
schema.update(**kwargs)
return schema
@staticmethod
def clean(d: dict):
return {k: v for k, v in d.items() if v}
def create_comment(self, channel_name=None, channel_id=None, **kwargs):
self.comment_id += 1
comment['comment_id'] = self.comment_id
if 'channel_id' in comment:
comment['channel_url'] = 'lbry://' + comment['channel_name'] + '#' + comment['channel_id']
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
comment = self._create_comment(
comment_id=str(comment_id),
channel_name=channel_name,
channel_id=channel_id,
channel_url=channel_url,
timestamp=str(int(time.time())),
**kwargs
)
self.comments.append(comment)
return comment
return self.clean(comment)
def delete_comment(self, comment_id: int, channel_id: str, **kwargs):
deleted = False
try:
if 0 <= comment_id <= len(self.comments) and self.comments[comment_id - 1]['channel_id'] == channel_id:
self.comments.pop(comment_id - 1)
deleted = True
finally:
return {
str(comment_id): {
'deleted': deleted
}
}
def get_claim_comments(self, page=1, page_size=50, **kwargs):
return {
@@ -37,22 +81,36 @@ class MockedCommentServer:
'page_size': page_size,
'total_pages': ceil(len(self.comments)/page_size),
'total_items': len(self.comments),
'items': (self.comments[::-1])[(page - 1) * page_size: page * page_size]
'items': [self.clean(c) for c in (self.comments[::-1])[(page - 1) * page_size: page * page_size]]
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[comment_id - 1]
return {
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
}
methods = {
'get_claim_comments': get_claim_comments,
'create_comment': create_comment,
'delete_comment': delete_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
if body['method'] in self.methods:
params = body.get('params', {})
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
try:
if body['method'] in self.methods:
params = body.get('params', {})
if 'comment_id' in params and type(params['comment_id']) is str:
params['comment_id'] = int(params['comment_id'])
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
except Exception:
response['error'] = self.ERRORS['UNKNOWN']
return response
async def start(self):
@@ -173,3 +231,20 @@ class CommentCommands(CommandTestCase):
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 28)
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], deleted)
self.assertTrue(deleted[comment['comment_id']]['deleted'])
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], deleted)
self.assertFalse(deleted[comment['comment_id']]['deleted'])
@@ -1,9 +1,11 @@
import asyncio
import os
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import lbry.wallet
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.Components import HeadersComponent
from torba.client.basenetwork import ClientSession
from torba.testcase import IntegrationTestCase
class TestSessionBloat(IntegrationTestCase):
@@ -15,6 +17,7 @@ class TestSessionBloat(IntegrationTestCase):
async def test_session_bloat_from_socket_timeout(self):
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
await self.conductor.start_spv()
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
@@ -28,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
class TestHeadersComponent(CommandTestCase):
LEDGER = lbry.wallet
async def asyncSetUp(self):
await super().asyncSetUp()
self.component_manager = self.daemon.component_manager
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
self.headers_component = HeadersComponent(self.component_manager)
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesnt raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -1,5 +1,5 @@
from torba.testcase import AsyncioTestCase
import hashlib
from lbry.extras.daemon.comment_client import sign_comment
from lbry.extras.daemon.comment_client import is_comment_signed_by_channel
@@ -14,7 +14,8 @@ class TestSigningComments(AsyncioTestCase):
'claim_id': claim.claim_id,
'channel_name': channel.claim_name,
'channel_id': channel.claim_id,
'comment': comment
'comment': comment,
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
}
def test01_successful_create_sign_and_validate_comment(self):
@@ -31,3 +32,28 @@ class TestSigningComments(AsyncioTestCase):
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
def test03_successful_sign_abandon_comment(self):
rswanson = get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
def test04_invalid_signature(self):
rswanson = get_channel('@RonSwanson')
jeanralphio = get_channel('@JeanRalphio')
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
chair_comment = self.create_claim_comment_body(
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
chair,
rswanson
)
sign_comment(chair_comment, rswanson)
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
+1 -1
View File
@@ -11,7 +11,7 @@ class TestTagNormalization(unittest.TestCase):
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'Tag')
tag('tag', 'T\'ag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
@@ -0,0 +1,60 @@
import time
import unittest
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
class TestPercentileCalculation(unittest.TestCase):
def test_calculate_percentiles(self):
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
self.assertEqual(calculate_avg_percentiles(
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
class TestCollectingMetrics(unittest.TestCase):
def test_happy_path(self):
self.maxDiff = None
load = ServerLoadData()
search = load.for_api('search')
self.assertEqual(search.name, 'search')
search.start()
search.cache_response()
search.cache_response()
metrics = {
'search': [{'total': 40}],
'execute_query': [
{'total': 20},
{'total': 10}
]
}
for x in range(5):
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
metrics['execute_query'][0]['total'] = 10
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
search.query_interrupt(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.052, {})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
"receive_count": 1,
"cache_response_count": 2,
"query_response_count": 5,
"intrp_response_count": 1,
"error_response_count": 2,
"response": (53, 51, 51, 52, 53, 54, 55, 55),
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
"error": (51, 50, 50, 50, 50, 52, 52, 52),
"python": (12, 10, 10, 10, 10, 20, 20, 20),
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
"individual_sql_count": 14,
"errored_queries": ['FROM claim where something=1'],
"interrupted_queries": ['FROM claim where something=1'],
}}})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
@@ -53,35 +53,45 @@ class ReconnectTests(IntegrationTestCase):
class ServerPickingTestCase(AsyncioTestCase):
async def _make_fake_server(self, latency=1.0, port=1337):
# local fake server with artificial latency
proto = RPCSession()
async def __handler(_):
await asyncio.sleep(latency)
return {'height': 1}
proto.handle_request = __handler
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
async def _make_fake_server(self, latency=1.0, port=1):
# local fake server with artificial latency
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
return {"height": 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def _make_bad_server(self, port=42420):
async def echo(reader, writer):
while True:
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
await self._make_fake_server(latency=1.5, port=1340),
await self._make_fake_server(latency=0.1, port=1337),
await self._make_fake_server(latency=1.0, port=1339),
await self._make_fake_server(latency=0.5, port=1338),
# fast but unhealthy, should be discarded
await self._make_bad_server(),
('localhost', 1),
('example.that.doesnt.resolve', 9000),
await self._make_fake_server(latency=1.2, port=1340),
await self._make_fake_server(latency=0.5, port=1337),
await self._make_fake_server(latency=0.7, port=1339),
],
'connect_timeout': 30
'connect_timeout': 3
})
network = BaseNetwork(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
await asyncio.wait_for(network.on_connected.first, timeout=3)
self.assertTrue(network.is_connected)
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
# ensure we are connected to all of them
self.assertEqual(len(network.session_pool.sessions), 4)
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
self.assertEqual(len(network.session_pool.sessions), 3)
+21 -2
View File
@@ -1,11 +1,10 @@
import unittest
import sqlite3
from functools import wraps
from torba.client.wallet import Wallet
from torba.client.constants import COIN
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
from torba.client.hash import sha256
from torba.testcase import AsyncioTestCase
@@ -167,6 +166,26 @@ class TestQueryBuilder(unittest.TestCase):
("select * from foo LIMIT 20 OFFSET 10", {})
)
def test_query_interpolation(self):
self.maxDiff = None
# tests that interpolation replaces longer keys first
self.assertEqual(
interpolate(*query(
"select * from foo",
a__not='b', b__in='select * from blah where c=:$c',
d__any={'one__like': 'o', 'two': 2},
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
ahash=memoryview(sha256(b'hello world')),
limit=10, order_by='b', **{'$c': 3})
),
"select * from foo WHERE a != 'b' AND "
"b IN (select * from blah where c=3) AND "
"(one LIKE 'o' OR two = 2) AND "
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
"ORDER BY b LIMIT 10",
)
class TestQueries(AsyncioTestCase):
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
# corrupt by appending
headers.io.seek(block_bytes(len(headers)))
headers.io.write(b"appending")
headers._size = None
await headers.repair()
self.assertEqual(headers.height, 1499)
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
+5 -1
View File
@@ -18,6 +18,9 @@ class MockNetwork:
self.get_transaction_called = []
self.is_connected = False
def retriable_call(self, function, *args, **kwargs):
return function(*args, **kwargs)
async def get_history(self, address):
self.get_history_called.append(address)
self.address = address
@@ -121,8 +124,9 @@ class TestSynchronization(LedgerTestCase):
)
class MocHeaderNetwork:
class MocHeaderNetwork(MockNetwork):
def __init__(self, responses):
super().__init__(None, None)
self.responses = responses
async def get_headers(self, height, blocks):
+14
View File
@@ -1,6 +1,7 @@
import logging
import asyncio
from asyncio import wrap_future
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
@@ -191,6 +192,19 @@ def query(select, **constraints):
return ' '.join(sql), values
def interpolate(sql, values):
for k in sorted(values.keys(), reverse=True):
value = values[k]
if isinstance(value, memoryview):
value = f"X'{hexlify(bytes(value)).decode()}'"
elif isinstance(value, str):
value = f"'{value}'"
else:
value = str(value)
sql = sql.replace(f":{k}", value)
return sql
def rows_to_dict(rows, fields):
if rows:
return [dict(zip(fields, r)) for r in rows]
+11 -7
View File
@@ -70,8 +70,10 @@ class BaseHeaders:
return True
def __getitem__(self, height) -> dict:
assert not isinstance(height, slice), \
"Slicing of header chain has not been implemented yet."
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self.deserialize(height, self.get_raw_header(height))
def get_raw_header(self, height) -> bytes:
@@ -115,7 +117,7 @@ class BaseHeaders:
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = None
self._size = self.io.tell() // self.header_size
added += written
if bail:
break
@@ -170,10 +172,12 @@ class BaseHeaders:
async def repair(self):
previous_header_hash = fail = None
self.io.seek(0)
batch_size = 10000
batch_size = 36
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
@@ -184,7 +188,7 @@ class BaseHeaders:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
@@ -205,7 +209,7 @@ class BaseHeaders:
end = min(len(headers), end + self.chunk_size * self.header_size)
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
assert len(headers) % self.header_size == 0
assert len(headers) % self.header_size == 0, len(headers)
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]
+18 -6
View File
@@ -287,7 +287,7 @@ class BaseLedger(metaclass=LedgerRegistry):
subscription_update = False
if not headers:
header_response = await self.network.get_headers(height, 2001)
header_response = await self.network.retriable_call(self.network.get_headers, height, 2001)
headers = header_response['hex']
if not headers:
@@ -394,7 +394,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if local_status == remote_status:
return
remote_history = await self.network.get_history(address)
remote_history = await self.network.retriable_call(self.network.get_history, address)
cache_tasks = []
synced_history = StringIO()
@@ -447,6 +447,17 @@ class BaseLedger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
local_status, local_history = await self.get_local_status_and_history(address)
if local_status != remote_status:
log.debug(
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
remote_status, len(remote_history), local_status, len(local_history)
)
log.debug("local: %s", local_history)
log.debug("remote: %s", remote_history)
else:
log.debug("Sync completed for: %s", address)
async def cache_transaction(self, txid, remote_height):
cache_item = self._tx_cache.get(txid)
if cache_item is None:
@@ -466,7 +477,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if tx is None:
# fetch from network
_raw = await self.network.get_transaction(txid)
_raw = await self.network.retriable_call(self.network.get_transaction, txid)
if _raw:
tx = self.transaction_class(unhexlify(_raw))
await self.maybe_verify_transaction(tx, remote_height)
@@ -486,8 +497,8 @@ class BaseLedger(metaclass=LedgerRegistry):
async def maybe_verify_transaction(self, tx, remote_height):
tx.height = remote_height
if 0 < remote_height <= len(self.headers):
merkle = await self.network.get_merkle(tx.id, remote_height)
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = self.headers[remote_height]
tx.position = merkle['pos']
@@ -501,6 +512,7 @@ class BaseLedger(metaclass=LedgerRegistry):
return None
def broadcast(self, tx):
# broadcast cant be a retriable call yet
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
@@ -522,4 +534,4 @@ class BaseLedger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
raise TimeoutError('Timed out waiting for transaction.')
raise asyncio.TimeoutError('Timed out waiting for transaction.')
+244 -144
View File
@@ -1,9 +1,8 @@
import logging
import asyncio
from asyncio import CancelledError
from time import time
from typing import List
import socket
from operator import itemgetter
from typing import Dict, Optional, Tuple
from time import perf_counter
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
@@ -14,67 +13,146 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, **kwargs):
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
self.framer.max_size = self.max_errors = 1 << 32
self.bw_limit = -1
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.ping_task = None
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
self._semaphore = asyncio.Semaphore(int(self.timeout))
@property
def available(self):
return not self.is_closing() and self._can_send.is_set() and self.response_time is not None
@property
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
if not self.transport:
return None
return self.transport.get_extra_info('peername')
async def send_timed_server_version_request(self, args=(), timeout=None):
timeout = timeout or self.timeout
log.debug("send version request to %s:%i", *self.server)
start = perf_counter()
result = await asyncio.wait_for(
super().send_request('server.version', args), timeout=timeout
)
current_response_time = perf_counter() - start
response_sum = (self.response_time or 0) * self._response_samples + current_response_time
self.response_time = response_sum / (self._response_samples + 1)
self._response_samples += 1
return result
async def send_request(self, method, args=()):
try:
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
except RPCError as e:
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
raise e
except asyncio.TimeoutError:
self.abort()
raise
self.pending_amount += 1
async with self._semaphore:
return await self._send_request(method, args)
async def ping_forever(self):
async def _send_request(self, method, args=()):
log.debug("send %s to %s:%i", method, *self.server)
try:
if method == 'server.version':
reply = await self.send_timed_server_version_request(args, self.timeout)
else:
reply = await asyncio.wait_for(
super().send_request(method, args), timeout=self.timeout
)
log.debug("got reply for %s from %s:%i", method, *self.server)
return reply
except RPCError as e:
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
raise e
except ConnectionError:
log.warning("connection to %s:%i lost", *self.server)
self.synchronous_close()
raise asyncio.CancelledError(f"connection to {self.server[0]}:{self.server[1]} lost")
except asyncio.TimeoutError:
log.info("timeout sending %s to %s:%i", method, *self.server)
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
raise
finally:
self.pending_amount -= 1
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
# TODO: change to 'ping' on newer protocol (above 1.2)
while not self.is_closing():
if (time() - self.last_send) > self.max_seconds_idle:
await self.send_request('server.banner')
await asyncio.sleep(self.max_seconds_idle//3)
retry_delay = default_delay = 1.0
while True:
try:
if self.is_closing():
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
pass
finally:
self.trigger_urgent_reconnect.clear()
async def ensure_server_version(self, required='1.2', timeout=3):
return await asyncio.wait_for(
self.send_request('server.version', [__version__, required]), timeout=timeout
)
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
start = perf_counter()
await asyncio.wait_for(connector.create_connection(), timeout=timeout)
self.ping_task = asyncio.create_task(self.ping_forever())
self.connection_latency = perf_counter() - start
async def handle_request(self, request):
controller = self.network.subscription_controllers[request.method]
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self.pending_amount = 0
self._on_disconnect_controller.add(True)
if self.ping_task:
self.ping_task.cancel()
class BaseNetwork:
def __init__(self, ledger):
self.config = ledger.config
self.client: ClientSession = None
self.session_pool: SessionPool = None
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.running = False
self.remote_height: int = 0
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
self._on_header_controller = StreamController()
self._on_header_controller = StreamController(merge_repeated_events=True)
self.on_header = self._on_header_controller.stream
self._on_status_controller = StreamController()
self._on_status_controller = StreamController(merge_repeated_events=True)
self.on_status = self._on_status_controller.stream
self.subscription_controllers = {
@@ -82,79 +160,72 @@ class BaseNetwork:
'blockchain.address.subscribe': self._on_status_controller,
}
async def switch_to_fastest(self):
try:
client = await asyncio.wait_for(self.session_pool.wait_for_fastest_session(), 30)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = None
for session in self.session_pool.sessions:
session.synchronous_close()
log.warning("not connected to any wallet servers")
return
current_client = self.client
self.client = client
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
try:
self._update_remote_height((await self.subscribe_headers(),))
log.info("Subscribed to headers: %s:%d", *self.client.server)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = current_client
return
self.session_pool.new_connection_event.clear()
return await self.session_pool.new_connection_event.wait()
async def start(self):
self.running = True
connect_timeout = self.config.get('connect_timeout', 6)
self.session_pool = SessionPool(network=self, timeout=connect_timeout)
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
while True:
try:
self.client = await self.pick_fastest_session()
if self.is_connected:
await self.ensure_server_version()
self._update_remote_height((await self.subscribe_headers(),))
log.info("Successfully connected to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
await self.client.on_disconnected.first
except CancelledError:
self.running = False
except asyncio.TimeoutError:
log.warning("Timed out while trying to find a server!")
except Exception: # pylint: disable=broad-except
log.exception("Exception while trying to find a server!")
if not self.running:
return
elif self.client:
await self.client.close()
self.client.connection.cancel_pending_requests()
while self.running:
await self.switch_to_fastest()
async def stop(self):
self.running = False
if self.session_pool:
self.session_pool.stop()
if self.is_connected:
disconnected = self.client.on_disconnected.first
await self.client.close()
await disconnected
self.session_pool.stop()
@property
def is_connected(self):
return self.client is not None and not self.client.is_closing()
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args):
if self.is_connected:
return self.client.send_request(list_or_method, args)
def rpc(self, list_or_method, args, session=None):
session = session or self.session_pool.fastest_session
if session:
return session.send_request(list_or_method, args)
else:
self.session_pool.trigger_nodelay_connect()
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def pick_fastest_session(self):
sessions = await self.session_pool.get_online_sessions()
done, pending = await asyncio.wait([
self.probe_session(session)
for session in sessions if not session.is_closing()
], return_when='FIRST_COMPLETED')
for task in pending:
task.cancel()
for session in done:
return await session
async def probe_session(self, session: ClientSession):
await session.send_request('server.banner')
return session
async def retriable_call(self, function, *args, **kwargs):
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
pass
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
def ensure_server_version(self, required='1.2'):
return self.rpc('server.version', [__version__, required])
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction])
def get_history(self, address):
return self.rpc('blockchain.address.get_history', [address])
def get_transaction(self, tx_hash):
return self.rpc('blockchain.transaction.get', [tx_hash])
@@ -167,82 +238,111 @@ class BaseNetwork:
def get_headers(self, height, count=10000):
return self.rpc('blockchain.block.headers', [height, count])
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True])
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
def get_history(self, address):
return self.rpc('blockchain.address.get_history', [address], session=self.client)
def subscribe_address(self, address):
return self.rpc('blockchain.address.subscribe', [address])
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], session=self.client)
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True], session=self.client)
async def subscribe_address(self, address):
try:
return await self.rpc('blockchain.address.subscribe', [address], session=self.client)
except asyncio.TimeoutError:
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
self.client.abort()
raise asyncio.CancelledError()
class SessionPool:
def __init__(self, network: BaseNetwork, timeout: float):
self.network = network
self.sessions: List[ClientSession] = []
self._dead_servers: List[ClientSession] = []
self.maintain_connections_task = None
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.timeout = timeout
# triggered when the master server is out, to speed up reconnect
self._lost_master = asyncio.Event()
self.new_connection_event = asyncio.Event()
@property
def online(self):
for session in self.sessions:
if not session.is_closing():
return True
return False
return any(not session.is_closing() for session in self.sessions)
@property
def available_sessions(self):
return [session for session in self.sessions if session.available]
@property
def fastest_session(self):
if not self.available_sessions:
return None
return min(
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions],
key=itemgetter(0)
)[1]
def _get_session_connect_callback(self, session: ClientSession):
loop = asyncio.get_event_loop()
def callback():
duplicate_connections = [
s for s in self.sessions
if s is not session and s.server_address_and_port == session.server_address_and_port
]
already_connected = None if not duplicate_connections else duplicate_connections[0]
if already_connected:
self.sessions.pop(session).cancel()
session.synchronous_close()
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
session.server[0], already_connected.server[0])
loop.call_later(3600, self._connect_session, session.server)
return
self.new_connection_event.set()
log.info("connected to %s:%i", *session.server)
return callback
def _connect_session(self, server: Tuple[str, int]):
session = None
for s in self.sessions:
if s.server == server:
session = s
break
if not session:
session = ClientSession(
network=self.network, server=server
)
session._on_connect_cb = self._get_session_connect_callback(session)
task = self.sessions.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
def start(self, default_servers):
self.sessions = [
ClientSession(network=self.network, server=server)
for server in default_servers
]
self.maintain_connections_task = asyncio.create_task(self.ensure_connections())
for server in default_servers:
self._connect_session(server)
def stop(self):
if self.maintain_connections_task:
self.maintain_connections_task.cancel()
for task in self.sessions.values():
task.cancel()
self.sessions.clear()
def ensure_connections(self):
for session in self.sessions:
if not session.is_closing():
session.abort()
self.sessions, self._dead_servers, self.maintain_connections_task = [], [], None
self._connect_session(session.server)
async def ensure_connections(self):
while True:
await asyncio.gather(*[
self.ensure_connection(session)
for session in self.sessions
], return_exceptions=True)
try:
await asyncio.wait_for(self._lost_master.wait(), timeout=3)
except asyncio.TimeoutError:
pass
self._lost_master.clear()
if not self.sessions:
self.sessions.extend(self._dead_servers)
self._dead_servers = []
def trigger_nodelay_connect(self):
# used when other parts of the system sees we might have internet back
# bypasses the retry interval
for session in self.sessions:
session.trigger_urgent_reconnect.set()
async def ensure_connection(self, session):
if not session.is_closing():
return
try:
return await session.create_connection(self.timeout)
except asyncio.TimeoutError:
log.warning("Timeout connecting to %s:%d", *session.server)
except asyncio.CancelledError: # pylint: disable=try-except-raise
raise
except socket.gaierror:
log.warning("Could not resolve IP for %s", session.server[0])
except Exception as err: # pylint: disable=broad-except
if 'Connect call failed' in str(err):
log.warning("Could not connect to %s:%d", *session.server)
else:
log.exception("Connecting to %s:%d raised an exception:", *session.server)
self._dead_servers.append(session)
self.sessions.remove(session)
async def get_online_sessions(self):
self._lost_master.set()
while not self.online:
await asyncio.sleep(0.1)
return self.sessions
async def wait_for_fastest_session(self):
while not self.fastest_session:
self.trigger_nodelay_connect()
self.new_connection_event.clear()
await self.new_connection_event.wait()
return self.fastest_session
+2 -1
View File
@@ -206,7 +206,8 @@ class SPVNode:
'HOST': self.hostname,
'TCP_PORT': str(self.port),
'SESSION_TIMEOUT': str(self.session_timeout),
'MAX_QUERY_WORKERS': '0'
'MAX_QUERY_WORKERS': '0',
'INDIVIDUAL_TAG_INDEXES': '',
}
# TODO: don't use os.environ
os.environ.update(conf)
+3 -3
View File
@@ -33,6 +33,7 @@ __all__ = ('JSONRPC', 'JSONRPCv1', 'JSONRPCv2', 'JSONRPCLoose',
import itertools
import json
import typing
import asyncio
from functools import partial
from numbers import Number
@@ -745,9 +746,8 @@ class JSONRPCConnection(object):
self._protocol = item
return self.receive_message(message)
def cancel_pending_requests(self):
"""Cancel all pending requests."""
exception = CancelledError()
def raise_pending_requests(self, exception):
exception = exception or asyncio.TimeoutError()
for request, event in self._requests.values():
event.result = exception
event.set()
+17 -8
View File
@@ -103,7 +103,7 @@ class SessionBase(asyncio.Protocol):
# Force-close a connection if a send doesn't succeed in this time
self.max_send_delay = 60
# Statistics. The RPC object also keeps its own statistics.
self.start_time = time.time()
self.start_time = time.perf_counter()
self.errors = 0
self.send_count = 0
self.send_size = 0
@@ -123,7 +123,7 @@ class SessionBase(asyncio.Protocol):
# A non-positive value means not to limit concurrency
if self.bw_limit <= 0:
return
now = time.time()
now = time.perf_counter()
# Reduce the recorded usage in proportion to the elapsed time
refund = (now - self.bw_time) * (self.bw_limit / 3600)
self.bw_charge = max(0, self.bw_charge - int(refund))
@@ -146,7 +146,7 @@ class SessionBase(asyncio.Protocol):
await asyncio.wait_for(self._can_send.wait(), secs)
except asyncio.TimeoutError:
self.abort()
raise asyncio.CancelledError(f'task timed out after {secs}s')
raise asyncio.TimeoutError(f'task timed out after {secs}s')
async def _send_message(self, message):
if not self._can_send.is_set():
@@ -156,7 +156,7 @@ class SessionBase(asyncio.Protocol):
self.send_size += len(framed_message)
self._using_bandwidth(len(framed_message))
self.send_count += 1
self.last_send = time.time()
self.last_send = time.perf_counter()
if self.verbosity >= 4:
self.logger.debug(f'Sending framed message {framed_message}')
self.transport.write(framed_message)
@@ -215,7 +215,8 @@ class SessionBase(asyncio.Protocol):
self._address = None
self.transport = None
self._task_group.cancel()
self._pm_task.cancel()
if self._pm_task:
self._pm_task.cancel()
# Release waiting tasks
self._can_send.set()
@@ -253,6 +254,7 @@ class SessionBase(asyncio.Protocol):
if self.transport:
self.transport.abort()
# TODO: replace with synchronous_close
async def close(self, *, force_after=30):
"""Close the connection and return when closed."""
self._close()
@@ -262,6 +264,11 @@ class SessionBase(asyncio.Protocol):
self.abort()
await self._pm_task
def synchronous_close(self):
self._close()
if self._pm_task and not self._pm_task.done():
self._pm_task.cancel()
class MessageSession(SessionBase):
"""Session class for protocols where messages are not tied to responses,
@@ -296,7 +303,7 @@ class MessageSession(SessionBase):
)
self._bump_errors()
else:
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -416,7 +423,7 @@ class RPCSession(SessionBase):
self.logger.warning(f'{e!r}')
continue
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -456,7 +463,7 @@ class RPCSession(SessionBase):
def connection_lost(self, exc):
# Cancel pending requests and message processing
self.connection.cancel_pending_requests()
self.connection.raise_pending_requests(exc)
super().connection_lost(exc)
# External API
@@ -473,6 +480,8 @@ class RPCSession(SessionBase):
async def send_request(self, method, args=()):
"""Send an RPC request over the network."""
if self.is_closing():
raise asyncio.TimeoutError("Trying to send request on a recently dropped connection.")
message, event = self.connection.send_request(Request(method, args))
await self._send_message(message)
await event.wait()
+1
View File
@@ -38,6 +38,7 @@ class Env:
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
+1 -1
View File
@@ -258,7 +258,7 @@ class SessionManager:
session_timeout = self.env.session_timeout
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.time() - session_timeout
stale_cutoff = time.perf_counter() - session_timeout
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions:
+8 -4
View File
@@ -45,10 +45,12 @@ class BroadcastSubscription:
class StreamController:
def __init__(self):
def __init__(self, merge_repeated_events=False):
self.stream = Stream(self)
self._first_subscription = None
self._last_subscription = None
self._last_event = None
self._merge_repeated = merge_repeated_events
@property
def has_listener(self):
@@ -76,8 +78,10 @@ class StreamController:
return f
def add(self, event):
skip = self._merge_repeated and event == self._last_event
self._last_event = event
return self._notify_and_ensure_future(
lambda subscription: subscription._add(event)
lambda subscription: None if skip else subscription._add(event)
)
def add_error(self, exception):
@@ -141,8 +145,8 @@ class Stream:
def first(self):
future = asyncio.get_event_loop().create_future()
subscription = self.listen(
lambda value: self._cancel_and_callback(subscription, future, value),
lambda exception: self._cancel_and_error(subscription, future, exception)
lambda value: not future.done() and self._cancel_and_callback(subscription, future, value),
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception)
)
return future