Compare commits
67
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
593422f70c | ||
|
|
de12e25450 | ||
|
|
2545ebbece | ||
|
|
14ec6ce43b | ||
|
|
0cac1e65aa | ||
|
|
1e4d57ab83 | ||
|
|
49b42b7e03 | ||
|
|
89c0b5dbdf | ||
|
|
8138675808 | ||
|
|
726129836a | ||
|
|
356b3c85e5 | ||
|
|
1a0680ead9 | ||
|
|
c876d891fa | ||
|
|
a08232f164 | ||
|
|
0f8cf46626 | ||
|
|
95017b5fc8 | ||
|
|
1374d337d8 | ||
|
|
35e7cc685f | ||
|
|
2ac4883bda | ||
|
|
5a7212aebb | ||
|
|
3f39cc2955 | ||
|
|
8a5bfe884a | ||
|
|
046dd53913 | ||
|
|
d2f174635c | ||
|
|
01a760d8f0 | ||
|
|
9215d2c9f9 | ||
|
|
0bb09ae91e | ||
|
|
afeff61268 | ||
|
|
6f5a595975 | ||
|
|
8a69ec062e | ||
|
|
45fb673e49 | ||
|
|
93b5f1809e | ||
|
|
ff05eaae48 | ||
|
|
b48b67ed10 | ||
|
|
3ead6f63b6 | ||
|
|
77235f0ad8 | ||
|
|
f6a80ba929 | ||
|
|
295ceb90d5 | ||
|
|
df41a82b23 | ||
|
|
34500e07c6 | ||
|
|
1e6097e3b2 | ||
|
|
521facedc4 | ||
|
|
de80a2ba97 | ||
|
|
3afd615447 | ||
|
|
87e52bec0a | ||
|
|
b89f04b1f1 | ||
|
|
e4f802e5f4 | ||
|
|
16fc2c456f | ||
|
|
ef7656ae98 | ||
|
|
ad2df4871e | ||
|
|
8d5f50b695 | ||
|
|
bbc056eef0 | ||
|
|
82fd3c632e | ||
|
|
dbae939808 | ||
|
|
c2dcacb0de | ||
|
|
8f45618cd0 | ||
|
|
29a87c850c | ||
|
|
395518ff76 | ||
|
|
08da763327 | ||
|
|
d7ec3c182d | ||
|
|
ffe6418392 | ||
|
|
88adf3ee27 | ||
|
|
f406db85dc | ||
|
|
d5f8af471a | ||
|
|
046459a72d | ||
|
|
49b0f59790 | ||
|
|
71551058ce |
No files matched your search
+2
-1
@@ -78,7 +78,8 @@ jobs:
|
||||
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
api_key:
|
||||
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
|
||||
file: lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
|
||||
@@ -20,14 +20,11 @@ class ServerConnection {
|
||||
channel = IOWebSocketChannel.connect(this.url);
|
||||
int tick = 1;
|
||||
channel.stream.listen((message) {
|
||||
Map data = json.decode(message);
|
||||
var data = json.decode(message);
|
||||
print(data);
|
||||
Map commands = data['commands'] ?? {};
|
||||
_loadDataController.add(
|
||||
ServerLoadDataPoint(
|
||||
tick,
|
||||
APICallMetrics.from_map(commands['search'] ?? {}),
|
||||
APICallMetrics.from_map(commands['resolve'] ?? {})
|
||||
ServerLoadDataPoint.from_map(
|
||||
tick, data
|
||||
)
|
||||
);
|
||||
tick++;
|
||||
@@ -52,48 +49,106 @@ class ServerConnection {
|
||||
}
|
||||
|
||||
|
||||
class TimeStats {
|
||||
final int avg, min, max;
|
||||
// percentiles
|
||||
final int five;
|
||||
final int twenty_five;
|
||||
final int fifty;
|
||||
final int seventy_five;
|
||||
final int ninety_five;
|
||||
TimeStats(
|
||||
this.avg, this.min, this.five, this.twenty_five,
|
||||
this.fifty, this.seventy_five, this.ninety_five,
|
||||
this.max
|
||||
);
|
||||
TimeStats.from_list(List l): this(
|
||||
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
|
||||
);
|
||||
TimeStats.from_zeros(): this(
|
||||
0, 0, 0, 0, 0, 0, 0, 0
|
||||
);
|
||||
factory TimeStats.from_list_or_zeros(List l) =>
|
||||
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
|
||||
}
|
||||
|
||||
|
||||
class APICallMetrics {
|
||||
final int started;
|
||||
final int finished;
|
||||
final int total_time;
|
||||
final int execution_time;
|
||||
final int query_time;
|
||||
final int query_count;
|
||||
final int cache_hit;
|
||||
final int avg_wait_time;
|
||||
final int avg_total_time;
|
||||
final int avg_execution_time;
|
||||
final int avg_query_time_per_search;
|
||||
final int avg_query_time_per_query;
|
||||
// total requests received
|
||||
final int receive_count;
|
||||
// sum of these is total responses made
|
||||
final int cache_response_count;
|
||||
final int query_response_count;
|
||||
final int intrp_response_count;
|
||||
final int error_response_count;
|
||||
// stacked values for chart
|
||||
final int cache_response_stack;
|
||||
final int query_response_stack;
|
||||
final int intrp_response_stack;
|
||||
final int error_response_stack;
|
||||
// millisecond timings for non-cache responses
|
||||
final TimeStats response;
|
||||
final TimeStats interrupt;
|
||||
final TimeStats error;
|
||||
// response, interrupt and error each also report the python, wait and sql stats:
|
||||
final TimeStats python;
|
||||
final TimeStats wait;
|
||||
final TimeStats sql;
|
||||
// extended timings for individual sql executions
|
||||
final TimeStats individual_sql;
|
||||
final int individual_sql_count;
|
||||
// actual queries
|
||||
final List<String> errored_queries;
|
||||
final List<String> interrupted_queries;
|
||||
APICallMetrics(
|
||||
this.started, this.finished, this.total_time, this.execution_time,
|
||||
this.query_time, this.query_count, this.cache_hit):
|
||||
avg_wait_time=finished > 0 ? ((total_time - (execution_time + query_time))/finished).round() : 0,
|
||||
avg_total_time=finished > 0 ? (total_time/finished).round() : 0,
|
||||
avg_execution_time=finished > 0 ? (execution_time/finished).round() : 0,
|
||||
avg_query_time_per_search=finished > 0 ? (query_time/finished).round() : 0,
|
||||
avg_query_time_per_query=query_count > 0 ? (query_time/query_count).round() : 0;
|
||||
this.receive_count,
|
||||
this.cache_response_count, this.query_response_count,
|
||||
this.intrp_response_count, this.error_response_count,
|
||||
this.response, this.interrupt, this.error,
|
||||
this.python, this.wait, this.sql,
|
||||
this.individual_sql, this.individual_sql_count,
|
||||
this.errored_queries, this.interrupted_queries
|
||||
):
|
||||
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
|
||||
query_response_stack=query_response_count+intrp_response_count+error_response_count,
|
||||
intrp_response_stack=intrp_response_count+error_response_count,
|
||||
error_response_stack=error_response_count;
|
||||
APICallMetrics.from_map(Map data): this(
|
||||
data['started'] ?? 0,
|
||||
data['finished'] ?? 0,
|
||||
data['total_time'] ?? 0,
|
||||
data['execution_time'] ?? 0,
|
||||
data['query_time'] ?? 0,
|
||||
data['query_count'] ?? 0,
|
||||
data['cache_hit'] ?? 0,
|
||||
data["receive_count"] ?? 0,
|
||||
data["cache_response_count"] ?? 0,
|
||||
data["query_response_count"] ?? 0,
|
||||
data["intrp_response_count"] ?? 0,
|
||||
data["error_response_count"] ?? 0,
|
||||
TimeStats.from_list_or_zeros(data["response"]),
|
||||
TimeStats.from_list_or_zeros(data["interrupt"]),
|
||||
TimeStats.from_list_or_zeros(data["error"]),
|
||||
TimeStats.from_list_or_zeros(data["python"]),
|
||||
TimeStats.from_list_or_zeros(data["wait"]),
|
||||
TimeStats.from_list_or_zeros(data["sql"]),
|
||||
TimeStats.from_list_or_zeros(data["individual_sql"]),
|
||||
data["individual_sql_count"] ?? 0,
|
||||
List<String>.from(data["errored_queries"] ?? const []),
|
||||
List<String>.from(data["interrupted_queries"] ?? const []),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadDataPoint {
|
||||
final int tick;
|
||||
final int sessions;
|
||||
final APICallMetrics search;
|
||||
final APICallMetrics resolve;
|
||||
ServerLoadDataPoint(this.tick, this.search, this.resolve);
|
||||
ServerLoadDataPoint.empty():
|
||||
tick = 0,
|
||||
search=APICallMetrics.from_map({}),
|
||||
resolve=APICallMetrics.from_map({});
|
||||
const ServerLoadDataPoint(
|
||||
this.tick, this.sessions, this.search, this.resolve
|
||||
);
|
||||
ServerLoadDataPoint.from_map(int tick, Map data): this(
|
||||
tick, (data['status'] ?? const {})['sessions'] ?? 0,
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
|
||||
);
|
||||
ServerLoadDataPoint.empty(): this(
|
||||
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,12 @@ class ServerCharts extends StatelessWidget {
|
||||
Widget build(BuildContext context) {
|
||||
var server = Provider.of<Server>(context, listen: false);
|
||||
return ListView(children: <Widget>[
|
||||
SizedBox(height: 300.0, child: ServerLoadChart(server)),
|
||||
SizedBox(height: 300.0, child: APILoadChart(
|
||||
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
|
||||
)),
|
||||
//SizedBox(height: 300.0, child: APILoadChart(
|
||||
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
|
||||
//)),
|
||||
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
|
||||
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
|
||||
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
|
||||
@@ -22,16 +27,20 @@ class ServerCharts extends StatelessWidget {
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadChart extends StatefulWidget {
|
||||
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
|
||||
|
||||
class APILoadChart extends StatefulWidget {
|
||||
final Server server;
|
||||
ServerLoadChart(this.server);
|
||||
final String name;
|
||||
final APIGetter getter;
|
||||
APILoadChart(this.server, this.name, this.getter);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => ServerLoadChartState();
|
||||
State<StatefulWidget> createState() => APILoadChartState();
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
class APILoadChartState extends State<APILoadChart> {
|
||||
|
||||
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
|
||||
|
||||
@@ -39,54 +48,73 @@ class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Cache',
|
||||
id: 'Received',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
strokeWidthPxFn: (_, __) => 4.0,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
|
||||
data: widget.server.serverLoadData,
|
||||
),*/
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Cache',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Query',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.blue.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Interrupts',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Errors',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Interrupted',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.pink.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Errored',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} From Cache',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.cache_hit,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Finish',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.deepOrange.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.finished,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Start',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.deepOrange.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.started,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Cache',
|
||||
colorFn: (_, __) => charts.MaterialPalette.cyan.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.cache_hit,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Finish',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.finished,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Start',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.started,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
*/
|
||||
];
|
||||
}
|
||||
|
||||
@@ -94,7 +122,9 @@ class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ServerLoadDataPoint>(
|
||||
stream: widget.server.serverLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData)
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData,
|
||||
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -117,27 +147,41 @@ class ServerPerformanceChartState extends State<ServerPerformanceChart> {
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Waiting 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Waiting',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_wait_time,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Executing',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_execution_time,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'SQLite 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. SQLite',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_query_time_per_search,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
)
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -260,11 +304,13 @@ class BetterLineChart extends charts.LineChart {
|
||||
final int itemCount;
|
||||
final Object lastItem;
|
||||
|
||||
BetterLineChart(List<charts.Series<dynamic, int>> seriesList):
|
||||
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
|
||||
itemCount = seriesList[0].data.length,
|
||||
lastItem = seriesList[0].data.last,
|
||||
super(
|
||||
seriesList,
|
||||
animate: false,
|
||||
defaultRenderer: renderer,
|
||||
behaviors: [charts.SeriesLegend()],
|
||||
domainAxis: charts.NumericAxisSpec(
|
||||
viewport: new charts.NumericExtents(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.38.1"
|
||||
__version__ = "0.38.6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -187,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
@@ -220,7 +220,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: typing.Optional['AbstractBlob'], address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
@@ -242,7 +242,9 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', addres
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
connected_transport = protocol.transport
|
||||
if blob is None or blob.get_is_verified() or not blob.is_writeable():
|
||||
# blob is None happens when we are just opening a connection
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
|
||||
@@ -30,17 +30,19 @@ class BlobDownloader:
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
|
||||
if len(self.active_connections) >= max_probes:
|
||||
return False
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0,
|
||||
just_probe: bool = False):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
@@ -91,16 +93,18 @@ class BlobDownloader:
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
"running, %d peers, %d ignored, %d active, %s connections",
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored or peer in self.active_connections:
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
just_probe = len(self.connections) == 0
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
|
||||
@@ -551,6 +551,9 @@ class Config(CLIConfig):
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
|
||||
self.storage = None
|
||||
|
||||
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
HEADER_SIZE = 112
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
|
||||
} if progress is not None and progress < 100 else {}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.headers.bytes_size
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if final_size_after_download % self.headers.header_size != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
buffer, header_size = b'', self.headers.header_size
|
||||
async for chunk in response.content.iter_any():
|
||||
chunk = buffer + chunk
|
||||
remaining = len(chunk) % header_size
|
||||
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
|
||||
if not chunk:
|
||||
continue
|
||||
if not await self.headers.connect(len(self.headers), chunk):
|
||||
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
|
||||
return
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
self.headers.bytes_size, final_size_after_download
|
||||
)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while not response.content.at_eof():
|
||||
local_header_size += fd.write(await response.content.readany())
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
local_header_size, final_size_after_download
|
||||
)
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
if os.path.isfile(self.headers_file):
|
||||
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
try:
|
||||
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return response.content_length // HEADER_SIZE
|
||||
except OSError:
|
||||
log.exception("Failed to download headers using https.")
|
||||
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return -1
|
||||
return response.content_length // self.headers.header_size
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.headers.height
|
||||
local_height = self.local_header_file_size() // self.headers.header_size
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
|
||||
return True
|
||||
return False
|
||||
|
||||
def verify_checkpoint(self):
|
||||
expected_hash, at_height = self.CHECKPOINT
|
||||
if self.local_header_file_size() // self.headers.header_size < at_height:
|
||||
return False
|
||||
hash = hashlib.sha256()
|
||||
chunk_size = self.headers.header_size * 1000
|
||||
with open(self.headers_file, 'rb') as header_file:
|
||||
data = header_file.read(chunk_size)
|
||||
while data and header_file.tell() <= at_height * self.headers.header_size:
|
||||
hash.update(data)
|
||||
data = header_file.read(chunk_size)
|
||||
return hash.hexdigest() == expected_hash
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
if await self.should_download_headers_from_s3():
|
||||
try:
|
||||
|
||||
try:
|
||||
if await self.should_download_headers_from_s3():
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
await self.headers.close()
|
||||
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
# fixme: workaround, this should happen before download but happens after because headers.connect fail
|
||||
if not self.verify_checkpoint():
|
||||
log.info("Checkpoint failed, verifying headers using slower method.")
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
await self.headers.close()
|
||||
else:
|
||||
log.info("Header checkpoint verified.")
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ from lbry.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_
|
||||
from lbry.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbry.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbry.extras.daemon.comment_client import jsonrpc_post, sign_comment, is_comment_signed_by_channel
|
||||
from lbry.extras.daemon import comment_client
|
||||
from lbry.extras.daemon.undecorated import undecorated
|
||||
from lbry.wallet.transaction import Transaction, Output, Input
|
||||
from lbry.wallet.account import Account as LBCAccount
|
||||
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(new_txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
]
|
||||
}
|
||||
"""
|
||||
result = await jsonrpc_post(
|
||||
result = await comment_client.jsonrpc_post(
|
||||
self.conf.comment_server,
|
||||
"get_claim_comments",
|
||||
claim_id=claim_id,
|
||||
@@ -3432,7 +3432,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
continue
|
||||
resolve_response = await self.resolve([channel_url])
|
||||
if isinstance(resolve_response[channel_url], Output):
|
||||
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
|
||||
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
|
||||
comment, resolve_response[channel_url]
|
||||
)
|
||||
else:
|
||||
@@ -3490,12 +3490,43 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
})
|
||||
sign_comment(comment_body, channel)
|
||||
response = await jsonrpc_post(self.conf.comment_server, 'create_comment', **comment_body)
|
||||
comment_client.sign_comment(comment_body, channel)
|
||||
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
|
||||
if 'signature' in response:
|
||||
response['is_claim_signature_valid'] = is_comment_signed_by_channel(response, channel)
|
||||
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
|
||||
return response
|
||||
|
||||
async def jsonrpc_comment_abandon(self, comment_id):
|
||||
""""
|
||||
Delete a comment published under your channel identity
|
||||
|
||||
Usage:
|
||||
comment_delete (<comment_id> | --comment_id=<comment_id>)
|
||||
|
||||
Options:
|
||||
--comment_id=<comment_id> : (str) The ID of the comment to be deleted.
|
||||
|
||||
Returns:
|
||||
(dict) Object with the `comment_id` passed in as the key, and a flag indicating if it was deleted
|
||||
{
|
||||
<comment_id> (str): {
|
||||
"deleted": (bool)
|
||||
}
|
||||
}
|
||||
"""
|
||||
abandon_comment_body = {'comment_id': comment_id}
|
||||
channel = await comment_client.jsonrpc_post(
|
||||
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
|
||||
)
|
||||
if 'error' in channel:
|
||||
return {comment_id: {'deleted': False}}
|
||||
channel = await self.get_channel_or_none(None, **channel)
|
||||
abandon_comment_body.update({
|
||||
'channel_id': channel.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
})
|
||||
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
|
||||
return await comment_client.jsonrpc_post(self.conf.comment_server, 'delete_comment', abandon_comment_body)
|
||||
|
||||
async def broadcast_or_release(self, account, tx, blocking=False):
|
||||
try:
|
||||
|
||||
@@ -18,13 +18,18 @@ def get_encoded_signature(signature):
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output):
|
||||
def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
||||
if type(channel) is Output:
|
||||
try:
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [
|
||||
comment['signing_ts'].encode(),
|
||||
channel.claim_hash,
|
||||
comment['comment'].encode()
|
||||
cid2hash(comment['channel_id']),
|
||||
signing_field.encode()
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(comment['signature']),
|
||||
@@ -36,16 +41,21 @@ def is_comment_signed_by_channel(comment: dict, channel: Output):
|
||||
return False
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output):
|
||||
timestamp = str(int(time.time())).encode()
|
||||
pieces = [timestamp, channel.claim_hash, comment['comment'].encode()]
|
||||
def sign_comment(comment: dict, channel: Output, abandon=False):
|
||||
timestamp = str(int(time.time()))
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
comment['signature'] = binascii.hexlify(signature).decode()
|
||||
comment['signing_ts'] = timestamp.decode()
|
||||
comment.update({
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
})
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body, headers=headers) as response:
|
||||
@@ -53,5 +63,5 @@ async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except Exception as cte:
|
||||
log.exception('Unable to decode respose from server: %s', cte)
|
||||
log.exception('Unable to decode response from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -6,7 +6,7 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
|
||||
|
||||
|
||||
def normalize_tag(tag: str):
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
|
||||
|
||||
|
||||
def clean_tags(tags: List[str]):
|
||||
|
||||
@@ -79,6 +79,10 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
raise
|
||||
finally:
|
||||
self.sql.commit()
|
||||
if self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
|
||||
@@ -11,3 +11,210 @@ STREAM_TYPES = {
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = [
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
]
|
||||
|
||||
COMMON_TAGS = {
|
||||
"gaming": "gaming",
|
||||
"people & blogs": "people_and_blogs",
|
||||
"pop culture": "pop_culture",
|
||||
"entertainment": "entertainment",
|
||||
"technology": "technology",
|
||||
"music": "music",
|
||||
"funny": "funny",
|
||||
"education": "education",
|
||||
"learning": "learning",
|
||||
"news": "news",
|
||||
"gameplay": "gameplay",
|
||||
"science & technology": "science_and_technology",
|
||||
"playstation 4": "playstation_4",
|
||||
"beliefs": "beliefs",
|
||||
"nature": "nature",
|
||||
"news & politics": "news_and_politics",
|
||||
"comedy": "comedy",
|
||||
"games": "games",
|
||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
||||
"film & animation": "film_and_animation",
|
||||
"game": "game",
|
||||
"howto & style": "howto_and_style",
|
||||
"weapons": "weapons",
|
||||
"blockchain": "blockchain",
|
||||
"video game": "video_game",
|
||||
"sports": "sports",
|
||||
"walkthrough": "walkthrough",
|
||||
"ps4live": "ps4live",
|
||||
"art": "art",
|
||||
"pc": "pc",
|
||||
"economics": "economics",
|
||||
"automotive": "automotive",
|
||||
"minecraft": "minecraft",
|
||||
"playthrough": "playthrough",
|
||||
"ps4share": "ps4share",
|
||||
"tutorial": "tutorial",
|
||||
"play": "play",
|
||||
"twitch": "twitch",
|
||||
"how to": "how_to",
|
||||
"ps4": "ps4",
|
||||
"bitcoin": "bitcoin",
|
||||
"fortnite": "fortnite",
|
||||
"commentary": "commentary",
|
||||
"lets play": "lets_play",
|
||||
"fun": "fun",
|
||||
"politics": "politics",
|
||||
"xbox": "xbox",
|
||||
"autos & vehicles": "autos_and_vehicles",
|
||||
"travel & events": "travel_and_events",
|
||||
"food": "food",
|
||||
"science": "science",
|
||||
"mature": "mature",
|
||||
"xbox one": "xbox_one",
|
||||
"liberal": "liberal",
|
||||
"democrat": "democrat",
|
||||
"progressive": "progressive",
|
||||
"survival": "survival",
|
||||
"nonprofits & activism": "nonprofits_and_activism",
|
||||
"cryptocurrency": "cryptocurrency",
|
||||
"playstation": "playstation",
|
||||
"nintendo": "nintendo",
|
||||
"government": "government",
|
||||
"steam": "steam",
|
||||
"podcast": "podcast",
|
||||
"horror": "horror",
|
||||
"conservative": "conservative",
|
||||
"reaction": "reaction",
|
||||
"trailer": "trailer",
|
||||
"love": "love",
|
||||
"cnn": "cnn",
|
||||
"republican": "republican",
|
||||
"gamer": "gamer",
|
||||
"political": "political",
|
||||
"hangoutsonair": "hangoutsonair",
|
||||
"hoa": "hoa",
|
||||
"msnbc": "msnbc",
|
||||
"cbs": "cbs",
|
||||
"donald trump": "donald_trump",
|
||||
"fiction": "fiction",
|
||||
"fox news": "fox_news",
|
||||
"anime": "anime",
|
||||
"crypto": "crypto",
|
||||
"ethereum": "ethereum",
|
||||
"call of duty": "call_of_duty",
|
||||
"multiplayer": "multiplayer",
|
||||
"android": "android",
|
||||
"epic": "epic",
|
||||
"rpg": "rpg",
|
||||
"adventure": "adventure",
|
||||
"secular talk": "secular_talk",
|
||||
"btc": "btc",
|
||||
"atheist": "atheist",
|
||||
"atheism": "atheism",
|
||||
"ps3": "ps3",
|
||||
"video games": "video_games",
|
||||
"cod": "cod",
|
||||
"agnostic": "agnostic",
|
||||
"movie": "movie",
|
||||
"online": "online",
|
||||
"fps": "fps",
|
||||
"mod": "mod",
|
||||
"reviews": "reviews",
|
||||
"sharefactory": "sharefactory",
|
||||
"world": "world",
|
||||
"space": "space",
|
||||
"hilarious": "hilarious",
|
||||
"stream": "stream",
|
||||
"lol": "lol",
|
||||
"sony": "sony",
|
||||
"god": "god",
|
||||
"lets": "lets",
|
||||
"dance": "dance",
|
||||
"pvp": "pvp",
|
||||
"tech": "tech",
|
||||
"zombies": "zombies",
|
||||
"pokemon": "pokemon",
|
||||
"fail": "fail",
|
||||
"xbox 360": "xbox_360",
|
||||
"film": "film",
|
||||
"unboxing": "unboxing",
|
||||
"animation": "animation",
|
||||
"travel": "travel",
|
||||
"money": "money",
|
||||
"wwe": "wwe",
|
||||
"how": "how",
|
||||
"mods": "mods",
|
||||
"pubg": "pubg",
|
||||
"indie": "indie",
|
||||
"strategy": "strategy",
|
||||
"history": "history",
|
||||
"rap": "rap",
|
||||
"ios": "ios",
|
||||
"sony computer entertainment": "sony_computer_entertainment",
|
||||
"mobile": "mobile",
|
||||
"trump": "trump",
|
||||
"flat earth": "flat_earth",
|
||||
"hack": "hack",
|
||||
"trap": "trap",
|
||||
"fox": "fox",
|
||||
"vlogging": "vlogging",
|
||||
"news radio": "news_radio",
|
||||
"humor": "humor",
|
||||
"facebook": "facebook",
|
||||
"edm": "edm",
|
||||
"fitness": "fitness",
|
||||
"vaping": "vaping",
|
||||
"hip hop": "hip_hop",
|
||||
"secular": "secular",
|
||||
"jesus": "jesus",
|
||||
"vape": "vape",
|
||||
"song": "song",
|
||||
"remix": "remix",
|
||||
"guitar": "guitar",
|
||||
"daily": "daily",
|
||||
"mining": "mining",
|
||||
"diy": "diy",
|
||||
"videogame": "videogame",
|
||||
"pets & animals": "pets_and_animals",
|
||||
"funny moments": "funny_moments",
|
||||
"religion": "religion",
|
||||
"death": "death",
|
||||
"media": "media",
|
||||
"nbc": "nbc",
|
||||
"war": "war",
|
||||
"freedom": "freedom",
|
||||
"viral": "viral",
|
||||
"meme": "meme",
|
||||
"family": "family",
|
||||
"gold": "gold",
|
||||
"photography": "photography",
|
||||
"chill": "chill",
|
||||
"zombie": "zombie",
|
||||
"computer": "computer",
|
||||
"sniper": "sniper",
|
||||
"bible": "bible",
|
||||
"linux": "linux",
|
||||
"overwatch": "overwatch",
|
||||
"pro": "pro",
|
||||
"dragon": "dragon",
|
||||
"litecoin": "litecoin",
|
||||
"gta": "gta",
|
||||
"iphone": "iphone",
|
||||
"house": "house",
|
||||
"bass": "bass",
|
||||
"bitcoin news": "bitcoin_news",
|
||||
"wii": "wii",
|
||||
"crash": "crash",
|
||||
"league of legends": "league_of_legends",
|
||||
"grand theft auto v": "grand_theft_auto_v",
|
||||
"mario": "mario",
|
||||
"mmorpg": "mmorpg",
|
||||
"satire": "satire",
|
||||
"fire": "fire",
|
||||
"racing": "racing",
|
||||
"apple": "apple",
|
||||
"health": "health",
|
||||
"instrumental": "instrumental",
|
||||
"destiny": "destiny",
|
||||
"truth": "truth",
|
||||
"race": "race"
|
||||
}
|
||||
@@ -9,14 +9,20 @@ from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from dataclasses import dataclass
|
||||
|
||||
from torba.client.basedatabase import query
|
||||
from torba.client.basedatabase import query, interpolate
|
||||
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
class SQLiteOperationalError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query errored')
|
||||
self.metrics = metrics
|
||||
|
||||
|
||||
class SQLiteInterruptedError(sqlite3.OperationalError):
|
||||
@@ -50,7 +56,7 @@ SEARCH_PARAMS = {
|
||||
|
||||
|
||||
ORDER_FIELDS = {
|
||||
'name',
|
||||
'name', 'claim_hash'
|
||||
} | INTEGER_PARAMS
|
||||
|
||||
|
||||
@@ -91,7 +97,6 @@ ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
|
||||
|
||||
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
|
||||
|
||||
db = sqlite3.connect(_path, isolation_level=None, uri=True)
|
||||
db.row_factory = sqlite3.Row
|
||||
ctx.set(
|
||||
@@ -114,21 +119,16 @@ def measure(func):
|
||||
state = ctx.get()
|
||||
if not state.is_tracking_metrics:
|
||||
return func(*args, **kwargs)
|
||||
metric = state.metrics.setdefault(func.__name__, {
|
||||
'calls': 0, 'total': 0, 'isolated': 0, 'errors': 0
|
||||
})
|
||||
metric = {}
|
||||
state.metrics.setdefault(func.__name__, []).append(metric)
|
||||
state.stack.append([])
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except:
|
||||
metric['errors'] += 1
|
||||
raise
|
||||
finally:
|
||||
elapsed = int((time.perf_counter()-start)*1000)
|
||||
metric['calls'] += 1
|
||||
metric['total'] += elapsed
|
||||
metric['isolated'] += (elapsed-sum(state.stack.pop()))
|
||||
metric['total'] = elapsed
|
||||
metric['isolated'] = (elapsed-sum(state.stack.pop()))
|
||||
if state.stack:
|
||||
state.stack[-1].append(elapsed)
|
||||
return wrapper
|
||||
@@ -167,16 +167,17 @@ def execute_query(sql, values) -> List:
|
||||
try:
|
||||
return context.db.execute(sql, values).fetchall()
|
||||
except sqlite3.OperationalError as err:
|
||||
plain_sql = interpolate(sql, values)
|
||||
if context.is_tracking_metrics:
|
||||
context.metrics['execute_query'][-1]['sql'] = plain_sql
|
||||
if str(err) == "interrupted":
|
||||
query_str = sql
|
||||
for k in sorted(values.keys(), reverse=True):
|
||||
query_str = query_str.replace(f":{k}", str(values[k]) if not k.startswith("$") else f"'{values[k]}'")
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", query_str)
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
|
||||
raise SQLiteInterruptedError(context.metrics)
|
||||
context.log.exception('failed running query', exc_info=err)
|
||||
raise SQLiteOperationalError(context.metrics)
|
||||
|
||||
|
||||
@measure
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'order_by' in constraints:
|
||||
sql_order_by = []
|
||||
for order_by in constraints['order_by']:
|
||||
@@ -231,14 +232,6 @@ def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['claim.public_key_hash'] = sqlite3.Binary(
|
||||
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = resolve_url(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
if 'channel_hash' in constraints:
|
||||
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
|
||||
if 'channel_ids' in constraints:
|
||||
@@ -307,7 +300,18 @@ def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
""", **constraints
|
||||
)
|
||||
return sql, values
|
||||
|
||||
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = resolve_url(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
sql, values = _get_claims(cols, for_count, **constraints)
|
||||
return execute_query(sql, values)
|
||||
|
||||
|
||||
@@ -330,7 +334,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["height", "^name"]
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
@@ -404,7 +408,7 @@ def resolve_url(raw_url):
|
||||
if channel is not None:
|
||||
if set(query) == {'name'}:
|
||||
# temporarily emulate is_controlling for claims in channel
|
||||
query['order_by'] = ['effective_amount']
|
||||
query['order_by'] = ['effective_amount', '^height']
|
||||
else:
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['channel_hash'] = channel['claim_hash']
|
||||
@@ -420,7 +424,6 @@ def resolve_url(raw_url):
|
||||
return channel
|
||||
|
||||
|
||||
@measure
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
@@ -429,26 +432,61 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
all_items = {item for item in all_items if item not in not_items}
|
||||
any_items = {item for item in any_items if item not in not_items}
|
||||
|
||||
any_queries = {}
|
||||
|
||||
if attr == 'tag':
|
||||
common_tags = any_items & COMMON_TAGS.keys()
|
||||
if common_tags:
|
||||
any_items -= common_tags
|
||||
if len(common_tags) < 5:
|
||||
for item in common_tags:
|
||||
index_name = COMMON_TAGS[item]
|
||||
any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag = '{item}'
|
||||
)
|
||||
"""
|
||||
elif len(common_tags) >= 5:
|
||||
constraints.update({
|
||||
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_common_tag{i}' for i in range(len(common_tags))
|
||||
)
|
||||
any_queries[f'#_any_common_tags'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if any_items:
|
||||
|
||||
constraints.update({
|
||||
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if len(any_queries) == 1:
|
||||
constraints.update(any_queries)
|
||||
elif len(any_queries) > 1:
|
||||
constraints[f'ORed_{attr}_queries__any'] = any_queries
|
||||
|
||||
if all_items:
|
||||
constraints[f'$all_{attr}_count'] = len(all_items)
|
||||
constraints.update({
|
||||
|
||||
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
|
||||
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
|
||||
)
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
@@ -76,30 +76,12 @@ class SQLDB:
|
||||
trending_global integer not null default 0
|
||||
);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
|
||||
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized);
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_height_idx on claim (height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -110,7 +92,6 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
amount integer not null
|
||||
);
|
||||
create index if not exists support_txo_hash_idx on support (txo_hash);
|
||||
create index if not exists support_claim_hash_idx on support (claim_hash, height);
|
||||
"""
|
||||
|
||||
@@ -120,7 +101,6 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
height integer not null
|
||||
);
|
||||
create index if not exists tag_tag_idx on tag (tag);
|
||||
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
|
||||
"""
|
||||
|
||||
@@ -133,6 +113,38 @@ class SQLDB:
|
||||
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
|
||||
"""
|
||||
|
||||
SEARCH_INDEXES = """
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
|
||||
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
TAG_INDEXES = '\n'.join(
|
||||
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
|
||||
for tag_value, tag_key in COMMON_TAGS.items()
|
||||
)
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_CLAIM_TABLE +
|
||||
@@ -257,7 +269,7 @@ class SQLDB:
|
||||
|
||||
if tags:
|
||||
self.db.executemany(
|
||||
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
)
|
||||
|
||||
return claims
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import time
|
||||
import math
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
def calculate_elapsed(start) -> int:
|
||||
return int((time.perf_counter() - start) * 1000)
|
||||
|
||||
|
||||
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
|
||||
if not data:
|
||||
return 0, 0, 0, 0, 0, 0, 0, 0
|
||||
data.sort()
|
||||
size = len(data)
|
||||
return (
|
||||
int(sum(data) / size),
|
||||
data[0],
|
||||
data[math.ceil(size * .05) - 1],
|
||||
data[math.ceil(size * .25) - 1],
|
||||
data[math.ceil(size * .50) - 1],
|
||||
data[math.ceil(size * .75) - 1],
|
||||
data[math.ceil(size * .95) - 1],
|
||||
data[-1]
|
||||
)
|
||||
|
||||
|
||||
def remove_select_list(sql) -> str:
|
||||
return sql[sql.index('FROM'):]
|
||||
|
||||
|
||||
class APICallMetrics:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
# total requests received
|
||||
self.receive_count = 0
|
||||
self.cache_response_count = 0
|
||||
|
||||
# millisecond timings for query based responses
|
||||
self.query_response_times = []
|
||||
self.query_intrp_times = []
|
||||
self.query_error_times = []
|
||||
|
||||
self.query_python_times = []
|
||||
self.query_wait_times = []
|
||||
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
|
||||
|
||||
self.individual_sql_times = [] # every SQL query run on server
|
||||
|
||||
# actual queries
|
||||
self.errored_queries = set()
|
||||
self.interrupted_queries = set()
|
||||
|
||||
def to_json_and_reset(self):
|
||||
return {
|
||||
# total requests received
|
||||
"receive_count": self.receive_count,
|
||||
# sum of these is total responses made
|
||||
"cache_response_count": self.cache_response_count,
|
||||
"query_response_count": len(self.query_response_times),
|
||||
"intrp_response_count": len(self.query_intrp_times),
|
||||
"error_response_count": len(self.query_error_times),
|
||||
# millisecond timings for non-cache responses
|
||||
"response": calculate_avg_percentiles(self.query_response_times),
|
||||
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
|
||||
"error": calculate_avg_percentiles(self.query_error_times),
|
||||
# response, interrupt and error each also report the python, wait and sql stats:
|
||||
"python": calculate_avg_percentiles(self.query_python_times),
|
||||
"wait": calculate_avg_percentiles(self.query_wait_times),
|
||||
"sql": calculate_avg_percentiles(self.query_sql_times),
|
||||
# extended timings for individual sql executions
|
||||
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
|
||||
"individual_sql_count": len(self.individual_sql_times),
|
||||
# actual queries
|
||||
"errored_queries": list(self.errored_queries),
|
||||
"interrupted_queries": list(self.interrupted_queries),
|
||||
}
|
||||
|
||||
def start(self):
|
||||
self.receive_count += 1
|
||||
|
||||
def cache_response(self):
|
||||
self.cache_response_count += 1
|
||||
|
||||
def _add_query_timings(self, request_total_time, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
sub_process_total = metrics[self.name][0]['total']
|
||||
individual_query_times = [f['total'] for f in metrics['execute_query']]
|
||||
aggregated_query_time = sum(individual_query_times)
|
||||
self.individual_sql_times.extend(individual_query_times)
|
||||
self.query_sql_times.append(aggregated_query_time)
|
||||
self.query_python_times.append(sub_process_total - aggregated_query_time)
|
||||
self.query_wait_times.append(request_total_time - sub_process_total)
|
||||
|
||||
@staticmethod
|
||||
def _add_queries(query_set, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
for execute_query in metrics['execute_query']:
|
||||
if 'sql' in execute_query:
|
||||
query_set.add(remove_select_list(execute_query['sql']))
|
||||
|
||||
def query_response(self, start, metrics):
|
||||
self.query_response_times.append(calculate_elapsed(start))
|
||||
self._add_query_timings(self.query_response_times[-1], metrics)
|
||||
|
||||
def query_interrupt(self, start, metrics):
|
||||
self.query_intrp_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.interrupted_queries, metrics)
|
||||
self._add_query_timings(self.query_intrp_times[-1], metrics)
|
||||
|
||||
def query_error(self, start, metrics):
|
||||
self.query_error_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.errored_queries, metrics)
|
||||
self._add_query_timings(self.query_error_times[-1], metrics)
|
||||
|
||||
|
||||
class ServerLoadData:
|
||||
|
||||
def __init__(self):
|
||||
self._apis = {}
|
||||
|
||||
def for_api(self, name) -> APICallMetrics:
|
||||
if name not in self._apis:
|
||||
self._apis[name] = APICallMetrics(name)
|
||||
return self._apis[name]
|
||||
|
||||
def to_json_and_reset(self, status):
|
||||
try:
|
||||
return {
|
||||
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
|
||||
'status': status
|
||||
}
|
||||
finally:
|
||||
self._apis = {}
|
||||
@@ -4,13 +4,9 @@ import time
|
||||
import base64
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from weakref import WeakSet
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
|
||||
from aiohttp.http_websocket import WSMsgType, WSCloseCode
|
||||
|
||||
from torba.rpc.jsonrpc import RPCError, JSONRPC
|
||||
from torba.server.session import ElectrumX, SessionManager
|
||||
from torba.server import util
|
||||
@@ -18,56 +14,8 @@ from torba.server import util
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYDB
|
||||
from lbry.wallet.server.db import reader
|
||||
|
||||
|
||||
class AdminWebSocket:
|
||||
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
self.app = Application()
|
||||
self.app['websockets'] = WeakSet()
|
||||
self.app.router.add_get('/', self.on_connect)
|
||||
self.app.on_shutdown.append(self.on_shutdown)
|
||||
self.runner = AppRunner(self.app)
|
||||
|
||||
async def on_status(self, _):
|
||||
if not self.app['websockets']:
|
||||
return
|
||||
self.send_message({
|
||||
'type': 'status',
|
||||
'height': self.manager.daemon.cached_height(),
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
for web_socket in self.app['websockets']:
|
||||
asyncio.create_task(web_socket.send_json(msg))
|
||||
|
||||
async def start(self):
|
||||
await self.runner.setup()
|
||||
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def on_connect(self, request):
|
||||
web_socket = WebSocketResponse()
|
||||
await web_socket.prepare(request)
|
||||
self.app['websockets'].add(web_socket)
|
||||
try:
|
||||
async for msg in web_socket:
|
||||
if msg.type == WSMsgType.TEXT:
|
||||
await self.on_status(None)
|
||||
elif msg.type == WSMsgType.ERROR:
|
||||
print('web socket connection closed with exception %s' %
|
||||
web_socket.exception())
|
||||
finally:
|
||||
self.app['websockets'].discard(web_socket)
|
||||
return web_socket
|
||||
|
||||
@staticmethod
|
||||
async def on_shutdown(app):
|
||||
for web_socket in set(app['websockets']):
|
||||
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
|
||||
|
||||
class ResultCacheItem:
|
||||
@@ -95,9 +43,8 @@ class LBRYSessionManager(SessionManager):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.query_executor = None
|
||||
self.websocket = None
|
||||
self.metrics_processor = None
|
||||
self.command_metrics = {}
|
||||
self.reader_metrics = {}
|
||||
self.metrics = ServerLoadData()
|
||||
self.metrics_loop = None
|
||||
self.running = False
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
@@ -105,69 +52,11 @@ class LBRYSessionManager(SessionManager):
|
||||
self.search_cache['search'] = lrucache(10000)
|
||||
self.search_cache['resolve'] = lrucache(10000)
|
||||
|
||||
def get_command_tracking_info(self, command):
|
||||
if command not in self.command_metrics:
|
||||
self.command_metrics[command] = {
|
||||
'cache_hit': 0,
|
||||
'started': 0,
|
||||
'finished': 0,
|
||||
'total_time': 0,
|
||||
'execution_time': 0,
|
||||
'query_time': 0,
|
||||
'query_count': 0,
|
||||
'interrupted': 0,
|
||||
'interrupted_query_values': [],
|
||||
}
|
||||
return self.command_metrics[command]
|
||||
|
||||
def cache_hit(self, command_name):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['cache_hit'] += 1
|
||||
|
||||
def start_command_tracking(self, command_name):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['started'] += 1
|
||||
|
||||
def finish_command_tracking(self, command_name, elapsed, metrics):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['finished'] += 1
|
||||
command['total_time'] += elapsed
|
||||
if 'execute_query' in metrics:
|
||||
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
|
||||
command['query_time'] += metrics['execute_query']['total']
|
||||
command['query_count'] += metrics['execute_query']['calls']
|
||||
for func_name, func_metrics in metrics.items():
|
||||
reader = self.reader_metrics.setdefault(func_name, {})
|
||||
for key in func_metrics:
|
||||
if key not in reader:
|
||||
reader[key] = func_metrics[key]
|
||||
else:
|
||||
reader[key] += func_metrics[key]
|
||||
|
||||
def interrupted_command_error(self, command_name, elapsed, metrics, kwargs):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['finished'] += 1
|
||||
command['interrupted'] += 1
|
||||
command['total_time'] += elapsed
|
||||
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
|
||||
command['query_time'] += metrics['execute_query']['total']
|
||||
command['query_count'] += metrics['execute_query']['calls']
|
||||
if len(command['interrupted_query_values']) < 100:
|
||||
command['interrupted_query_values'].append(kwargs)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
commands, self.command_metrics = self.command_metrics, {}
|
||||
reader, self.reader_metrics = self.reader_metrics, {}
|
||||
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
|
||||
if self.websocket is not None:
|
||||
self.websocket.send_message({
|
||||
'commands': commands,
|
||||
'reader': reader
|
||||
})
|
||||
self.websocket.send_message(data)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def start_other(self):
|
||||
@@ -186,12 +75,12 @@ class LBRYSessionManager(SessionManager):
|
||||
if self.websocket is not None:
|
||||
await self.websocket.start()
|
||||
if self.env.track_metrics:
|
||||
self.metrics_processor = asyncio.create_task(self.process_metrics())
|
||||
self.metrics_loop = asyncio.create_task(self.process_metrics())
|
||||
|
||||
async def stop_other(self):
|
||||
self.running = False
|
||||
if self.env.track_metrics:
|
||||
self.metrics_processor.cancel()
|
||||
self.metrics_loop.cancel()
|
||||
if self.websocket is not None:
|
||||
await self.websocket.stop()
|
||||
self.query_executor.shutdown()
|
||||
@@ -220,45 +109,50 @@ class LBRYElectrumX(ElectrumX):
|
||||
}
|
||||
self.request_handlers.update(handlers)
|
||||
|
||||
async def run_in_executor(self, name, func, kwargs):
|
||||
start = None
|
||||
if self.env.track_metrics:
|
||||
self.session_mgr.start_command_tracking(name)
|
||||
start = time.perf_counter()
|
||||
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
self.session_mgr.interrupted_command_error(
|
||||
name, int((time.perf_counter() - start) * 1000), error.metrics, kwargs
|
||||
)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics.query_error(start, error.metrics)
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except:
|
||||
metrics.query_error(start, {})
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
|
||||
if self.env.track_metrics:
|
||||
elapsed = int((time.perf_counter() - start) * 1000)
|
||||
(result, metrics) = result
|
||||
self.session_mgr.finish_command_tracking(name, elapsed, metrics)
|
||||
(result, metrics_data) = result
|
||||
metrics.query_response(start, metrics_data)
|
||||
|
||||
return base64.b64encode(result).decode()
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
if self.env.track_metrics:
|
||||
metrics = self.session_mgr.metrics.for_api(query_name)
|
||||
else:
|
||||
metrics = APICallMetrics(query_name)
|
||||
metrics.start()
|
||||
cache = self.session_mgr.search_cache[query_name]
|
||||
cache_key = str(kwargs)
|
||||
cache_item = cache.get(cache_key)
|
||||
if cache_item is None:
|
||||
cache_item = cache[cache_key] = ResultCacheItem()
|
||||
elif cache_item.result is not None:
|
||||
self.session_mgr.cache_hit(query_name)
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
async with cache_item.lock:
|
||||
result = cache_item.result
|
||||
if result is None:
|
||||
result = cache_item.result = await self.run_in_executor(
|
||||
query_name, function, kwargs
|
||||
if cache_item.result is None:
|
||||
cache_item.result = await self.run_in_executor(
|
||||
metrics, function, kwargs
|
||||
)
|
||||
else:
|
||||
self.session_mgr.cache_hit(query_name)
|
||||
return result
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import asyncio
|
||||
from weakref import WeakSet
|
||||
|
||||
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
|
||||
from aiohttp.http_websocket import WSMsgType, WSCloseCode
|
||||
|
||||
|
||||
class AdminWebSocket:
|
||||
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
self.app = Application()
|
||||
self.app['websockets'] = WeakSet()
|
||||
self.app.router.add_get('/', self.on_connect)
|
||||
self.app.on_shutdown.append(self.on_shutdown)
|
||||
self.runner = AppRunner(self.app)
|
||||
|
||||
async def on_status(self, _):
|
||||
if not self.app['websockets']:
|
||||
return
|
||||
self.send_message({
|
||||
'type': 'status',
|
||||
'height': self.manager.daemon.cached_height(),
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
for web_socket in self.app['websockets']:
|
||||
asyncio.create_task(web_socket.send_json(msg))
|
||||
|
||||
async def start(self):
|
||||
await self.runner.setup()
|
||||
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def on_connect(self, request):
|
||||
web_socket = WebSocketResponse()
|
||||
await web_socket.prepare(request)
|
||||
self.app['websockets'].add(web_socket)
|
||||
try:
|
||||
async for msg in web_socket:
|
||||
if msg.type == WSMsgType.TEXT:
|
||||
await self.on_status(None)
|
||||
elif msg.type == WSMsgType.ERROR:
|
||||
print('web socket connection closed with exception %s' %
|
||||
web_socket.exception())
|
||||
finally:
|
||||
self.app['websockets'].discard(web_socket)
|
||||
return web_socket
|
||||
|
||||
@staticmethod
|
||||
async def on_shutdown(app):
|
||||
for web_socket in set(app['websockets']):
|
||||
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
@@ -1,15 +1,16 @@
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import textwrap
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db.reader import search_to_bytes, initializer
|
||||
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
|
||||
from lbry.wallet.ledger import MainNetLedger
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.INFO)
|
||||
log.setLevel(logging.CRITICAL)
|
||||
|
||||
DEFAULT_ANY_TAGS = [
|
||||
'blockchain',
|
||||
@@ -24,6 +25,25 @@ DEFAULT_ANY_TAGS = [
|
||||
'nature'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE = [
|
||||
'gaming',
|
||||
'ufos'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE2 = [
|
||||
'city fix',
|
||||
'gaming'
|
||||
]
|
||||
|
||||
RARE_ANY_TAGS = [
|
||||
'city fix',
|
||||
'ufos',
|
||||
]
|
||||
|
||||
CITY_FIX = [
|
||||
'city fix'
|
||||
]
|
||||
|
||||
MATURE_TAGS = [
|
||||
'porn',
|
||||
'nsfw',
|
||||
@@ -47,46 +67,48 @@ ORDER_BY = [
|
||||
|
||||
def get_args(limit=20):
|
||||
args = []
|
||||
any_tags_combinations = [DEFAULT_ANY_TAGS, []]
|
||||
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
|
||||
not_tags_combinations = [MATURE_TAGS, []]
|
||||
for no_totals in [True]:
|
||||
for offset in [0, 20, 40, 60, 80, 100, 1000, 2000, 3000]:
|
||||
for any_tags in any_tags_combinations:
|
||||
for not_tags in not_tags_combinations:
|
||||
for order_by in ORDER_BY:
|
||||
kw = {
|
||||
'order_by': order_by,
|
||||
'offset': offset,
|
||||
'limit': limit,
|
||||
'no_totals': no_totals
|
||||
}
|
||||
if not_tags:
|
||||
kw['not_tags'] = not_tags
|
||||
if any_tags:
|
||||
kw['any_tags'] = any_tags
|
||||
args.append(kw)
|
||||
print(len(args), "argument combinations")
|
||||
for no_fee in [False, True]:
|
||||
for claim_type in [None, 'stream', 'channel']:
|
||||
for no_totals in [True]:
|
||||
for offset in [0, 100]:
|
||||
for any_tags in any_tags_combinations:
|
||||
for not_tags in not_tags_combinations:
|
||||
for order_by in ORDER_BY:
|
||||
kw = {
|
||||
'order_by': order_by,
|
||||
'offset': offset,
|
||||
'limit': limit,
|
||||
'no_totals': no_totals
|
||||
}
|
||||
if not_tags:
|
||||
kw['not_tags'] = not_tags
|
||||
if any_tags:
|
||||
kw['any_tags'] = any_tags
|
||||
if claim_type:
|
||||
kw['claim_type'] = claim_type
|
||||
if no_fee:
|
||||
kw['fee_amount'] = 0
|
||||
args.append(kw)
|
||||
print(f"-- Trying {len(args)} argument combinations")
|
||||
return args
|
||||
|
||||
|
||||
def _search(kwargs):
|
||||
start = time.time()
|
||||
msg = f"offset={kwargs['offset']}, limit={kwargs['limit']}, no_totals={kwargs['no_totals']}, not_tags={kwargs.get('not_tags')}, any_tags={kwargs.get('any_tags')}, order_by={kwargs['order_by']}"
|
||||
start = time.perf_counter()
|
||||
error = None
|
||||
try:
|
||||
search_to_bytes(kwargs)
|
||||
t = time.time() - start
|
||||
return t, f"{t} - {msg}"
|
||||
except Exception as err:
|
||||
return -1, f"failed: error={str(type(err))}({str(err)}) - {msg}"
|
||||
error = str(err)
|
||||
return time.perf_counter() - start, kwargs, error
|
||||
|
||||
|
||||
async def search(executor, kwargs):
|
||||
try:
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
executor, _search, kwargs
|
||||
)
|
||||
except Exception as err:
|
||||
return f"failed (err={str(type(err))}({err}))- {kwargs}"
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
executor, _search, kwargs
|
||||
)
|
||||
|
||||
|
||||
async def main(db_path, max_query_time):
|
||||
@@ -97,21 +119,59 @@ async def main(db_path, max_query_time):
|
||||
tasks = [search(query_executor, constraints) for constraints in get_args()]
|
||||
try:
|
||||
results = await asyncio.gather(*tasks)
|
||||
times = {msg: ts for ts, msg in results}
|
||||
log.info("\n".join(sorted(filter(lambda msg: times[msg] > max_query_time, times.keys()), key=lambda msg: times[msg])))
|
||||
query_times = [
|
||||
{
|
||||
'sql': interpolate(*_get_claims("""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
""", **constraints)),
|
||||
'duration': ts,
|
||||
'error': error
|
||||
}
|
||||
for ts, constraints, error in results
|
||||
]
|
||||
errored = [query_info for query_info in query_times if query_info['error']]
|
||||
errors = {str(query_info['error']): [] for query_info in errored}
|
||||
for error in errored:
|
||||
errors[str(error['error'])].append(error['sql'])
|
||||
slow = [
|
||||
query_info for query_info in query_times
|
||||
if not query_info['error'] and query_info['duration'] > (max_query_time / 2.0)
|
||||
]
|
||||
fast = [
|
||||
query_info for query_info in query_times
|
||||
if not query_info['error'] and query_info['duration'] <= (max_query_time / 2.0)
|
||||
]
|
||||
print(f"-- {len(fast)} queries were fast")
|
||||
slow.sort(key=lambda query_info: query_info['duration'], reverse=True)
|
||||
print(f"-- Failing queries:")
|
||||
for error in errors:
|
||||
print(f"-- Failure: \"{error}\"")
|
||||
for failing_query in errors[error]:
|
||||
print(f"{textwrap.dedent(failing_query)};\n")
|
||||
print()
|
||||
print(f"-- Slow queries:")
|
||||
for slow_query in slow:
|
||||
print(f"-- Query took {slow_query['duration']}\n{textwrap.dedent(slow_query['sql'])};\n")
|
||||
finally:
|
||||
query_executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
if len(args) >= 1:
|
||||
db_path = args[0]
|
||||
else:
|
||||
db_path = os.path.expanduser('~/claims.db')
|
||||
if len(args) >= 2:
|
||||
max_query_time = float(args[1])
|
||||
else:
|
||||
max_query_time = -3
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
|
||||
parser.add_argument('--max_time', dest='max_time', default=0.25, type=float)
|
||||
args = parser.parse_args()
|
||||
db_path = args.db_path
|
||||
max_query_time = args.max_time
|
||||
asyncio.run(main(db_path, max_query_time))
|
||||
@@ -0,0 +1,107 @@
|
||||
import time
|
||||
import asyncio
|
||||
import random
|
||||
from argparse import ArgumentParser
|
||||
from torba.client.basenetwork import ClientSession
|
||||
|
||||
|
||||
class AgentSmith(ClientSession):
|
||||
|
||||
async def do_nefarious_things(self):
|
||||
await self.send_request('blockchain.claimtrie.search', {
|
||||
'no_totals': True,
|
||||
'offset': random.choice(range(0, 300, 20)),
|
||||
'limit': 20,
|
||||
'any_tags': (
|
||||
random.choice([[
|
||||
random.choice(['gaming', 'games', 'game']) +
|
||||
random.choice(['entertainment', 'playthrough', 'funny']) +
|
||||
random.choice(['xbox', 'xbox one', 'xbox news'])
|
||||
], [
|
||||
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
|
||||
random.choice(['news', 'sighting', 'sightings'])
|
||||
], [
|
||||
random.choice(['art', 'automotive']),
|
||||
random.choice(['blockchain', 'economics', 'food']),
|
||||
random.choice(['funny', 'learnings', 'nature']),
|
||||
random.choice(['news', 'science', 'technology'])
|
||||
]
|
||||
])
|
||||
),
|
||||
'not_tags': random.choice([[], [
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
]]),
|
||||
'order_by': random.choice([
|
||||
['release_time'],
|
||||
['trending_global', 'trending_mixed'],
|
||||
['effective_amount']
|
||||
])
|
||||
})
|
||||
|
||||
|
||||
class AgentSmithProgram:
|
||||
|
||||
def __init__(self, host, port):
|
||||
self.host, self.port = host, port
|
||||
self.agent_smiths = []
|
||||
|
||||
async def make_one_more_of_them(self):
|
||||
smith = AgentSmith(network=None, server=(self.host, self.port))
|
||||
await smith.create_connection()
|
||||
self.agent_smiths.append(smith)
|
||||
|
||||
async def coordinate_nefarious_activity(self):
|
||||
start = time.perf_counter()
|
||||
await asyncio.gather(
|
||||
*(s.do_nefarious_things() for s in self.agent_smiths),
|
||||
return_exceptions=True
|
||||
)
|
||||
return time.perf_counter() - start
|
||||
|
||||
def __len__(self):
|
||||
return len(self.agent_smiths)
|
||||
|
||||
async def delete_one_smith(self):
|
||||
if self.agent_smiths:
|
||||
await self.agent_smiths.pop().close()
|
||||
|
||||
async def delete_program(self):
|
||||
await asyncio.gather(*(
|
||||
s.close() for s in self.agent_smiths
|
||||
))
|
||||
|
||||
|
||||
async def main(host, port):
|
||||
smiths = AgentSmithProgram(host, port)
|
||||
await smiths.make_one_more_of_them()
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
ease_off = 0
|
||||
for i in range(1000):
|
||||
await asyncio.sleep(1)
|
||||
if activity.done() and activity.result() < .9:
|
||||
print('more, more, more...')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
|
||||
))
|
||||
else:
|
||||
print('!!!!!!!!!!!!!!')
|
||||
print('IS NEO LOSING?')
|
||||
print('!!!!!!!!!!!!!!')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
|
||||
))
|
||||
print(f'coordinate all {len(smiths)} smiths to action')
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
print('finishing up any remaining actions')
|
||||
await activity
|
||||
print('neo has won, deleting agents...')
|
||||
await smiths.delete_program()
|
||||
print('done.')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('--host', dest='host', default='localhost', type=str)
|
||||
parser.add_argument('--port', dest='port', default=50001, type=int)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.host, args.port))
|
||||
@@ -0,0 +1,49 @@
|
||||
import os, asyncio, aiohttp, json, slack, sqlparse
|
||||
|
||||
|
||||
async def listen(slack_client, url):
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
|
||||
print(f"connecting to {url}")
|
||||
try:
|
||||
ws = await session.ws_connect(url)
|
||||
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
|
||||
print(f"failed to connect to {url}")
|
||||
return
|
||||
print(f"connected to {url}")
|
||||
|
||||
async for msg in ws:
|
||||
r = json.loads(msg.data)
|
||||
try:
|
||||
queries = r["api"]["search"]["interrupted_queries"]
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
for q in queries:
|
||||
# clean = re.sub(r"\s+", " ", q)
|
||||
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
|
||||
print(f'{url}: {clean}')
|
||||
response = await slack_client.chat_postMessage(
|
||||
username=url,
|
||||
icon_emoji=":hourglass_flowing_sand:",
|
||||
channel='#clubhouse-de-obscure',
|
||||
text="*Query timed out:* " + clean
|
||||
)
|
||||
if not response["ok"]:
|
||||
print("SLACK ERROR:\n", response)
|
||||
print()
|
||||
|
||||
|
||||
async def main():
|
||||
try:
|
||||
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
|
||||
except KeyError:
|
||||
print("Error: SLACK_TOKEN env var required")
|
||||
return
|
||||
|
||||
num_servers = 5
|
||||
tasks = []
|
||||
for i in range(1, num_servers+1):
|
||||
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,9 +1,10 @@
|
||||
import uvloop, asyncio, time, sys, logging
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.metrics import calculate_avg_percentiles
|
||||
|
||||
|
||||
db_path = '/tmp/wallet-server/claims.db'
|
||||
db_path = '../../../lbryconf/wallet-server/claims.db'
|
||||
default_query_timout = 0.25
|
||||
log = logging.getLogger(__name__)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
@@ -16,26 +17,26 @@ async def run_times(executor, iterations, show=True):
|
||||
'no_totals': True,
|
||||
'offset': 0,
|
||||
'limit': 20,
|
||||
'fee_amount': '<1',
|
||||
#'all_tags': ['funny'],
|
||||
'any_tags': [
|
||||
'crypto',
|
||||
'outdoors',
|
||||
'cars',
|
||||
'automotive'
|
||||
'ufos', 'city fix'
|
||||
],
|
||||
'not_tags': [
|
||||
'nsfw', 'xxx', 'mature'
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
],
|
||||
'order_by': [
|
||||
'release_time'
|
||||
]
|
||||
}
|
||||
) for _ in range(iterations)))
|
||||
timings = [r[1]['execute_query']['total'] for r in timings]
|
||||
timings = [r[1]['execute_query'][0]['total'] for r in timings]
|
||||
total = int((time.perf_counter() - start) * 100)
|
||||
if show:
|
||||
avg = sum(timings)/len(timings)
|
||||
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
|
||||
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
|
||||
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
|
||||
stats = calculate_avg_percentiles(timings)
|
||||
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
|
||||
sys.stdout.write(' sample:')
|
||||
for i, t in zip(range(10), timings[::-1]):
|
||||
sys.stdout.write(f' {t}ms')
|
||||
@@ -44,16 +45,16 @@ async def run_times(executor, iterations, show=True):
|
||||
|
||||
async def main():
|
||||
executor = ProcessPoolExecutor(
|
||||
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', default_query_timout, True)
|
||||
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
|
||||
)
|
||||
await run_times(executor, 4, show=False)
|
||||
await run_times(executor, 1)
|
||||
#await run_times(executor, 4, show=False)
|
||||
#await run_times(executor, 1)
|
||||
await run_times(executor, 2**3)
|
||||
await run_times(executor, 2**5)
|
||||
await run_times(executor, 2**7)
|
||||
await run_times(executor, 2**9)
|
||||
await run_times(executor, 2**11)
|
||||
await run_times(executor, 2**13)
|
||||
#await run_times(executor, 2**9)
|
||||
#await run_times(executor, 2**11)
|
||||
#await run_times(executor, 2**13)
|
||||
executor.shutdown(True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,43 +1,19 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import ClientConnectorError
|
||||
from lbry import __version__
|
||||
from lbry.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbry.conf import Config
|
||||
from lbry.schema.uri import parse_lbry_uri
|
||||
from lbry.extras.daemon.client import daemon_rpc
|
||||
from lbry.extras import system_info
|
||||
|
||||
|
||||
def extract_uris(response):
|
||||
uris = list()
|
||||
for key in response:
|
||||
for value in response[key]:
|
||||
uris.append(value)
|
||||
|
||||
return uris
|
||||
|
||||
|
||||
async def get_frontpage_uris():
|
||||
session = aiohttp.ClientSession()
|
||||
try:
|
||||
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
|
||||
if response.status != 200:
|
||||
print("API returned non 200 code!!")
|
||||
return
|
||||
body = await response.json()
|
||||
await session.close()
|
||||
uris = extract_uris(body['data']['Uris'])
|
||||
return uris
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def report_to_slack(output, webhook):
|
||||
payload = {
|
||||
"text": f"lbrynet {__version__} ({system_info.get_platform()['platform']}) time to first byte:\n{output}"
|
||||
@@ -58,120 +34,127 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(conf, uri, timeout):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
async def wait_for_done(conf, claim_name, timeout):
|
||||
blobs_completed, last_completed = 0, time.perf_counter()
|
||||
while True:
|
||||
files = await daemon_rpc(conf, "file_list", claim_name=name)
|
||||
file = files[0]
|
||||
file = (await daemon_rpc(conf, "file_list", claim_name=claim_name))[0]
|
||||
if file['status'] in ['finished', 'stopped']:
|
||||
return True, file['blobs_completed'], file['blobs_in_stream']
|
||||
if last_complete < int(file['blobs_completed']):
|
||||
hang_count = 0
|
||||
last_complete = int(file['blobs_completed'])
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > timeout:
|
||||
elif blobs_completed < int(file['blobs_completed']):
|
||||
blobs_completed, last_completed = int(file['blobs_completed']), time.perf_counter()
|
||||
elif (time.perf_counter() - last_completed) > timeout:
|
||||
return False, file['blobs_completed'], file['blobs_in_stream']
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
|
||||
async def main(uris=None, cmd_args=None):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
async def main(cmd_args=None):
|
||||
print('Time to first byte started using parameters:')
|
||||
for key, value in vars(cmd_args).items():
|
||||
print(f"{key}: {value}")
|
||||
conf = Config()
|
||||
url_to_claim = {}
|
||||
try:
|
||||
await daemon_rpc(conf, 'status')
|
||||
for page in range(1, cmd_args.download_pages + 1):
|
||||
start = time.perf_counter()
|
||||
kwargs = {
|
||||
'page': page,
|
||||
# 'claim_type': 'stream',
|
||||
'order_by': ['trending_global', 'trending_mixed'],
|
||||
'no_totals': True
|
||||
}
|
||||
|
||||
# if not cmd_args.allow_fees:
|
||||
# kwargs['fee_amount'] = 0
|
||||
|
||||
response = await daemon_rpc(
|
||||
conf, 'claim_search', **kwargs
|
||||
)
|
||||
if 'error' in response or not response.get('items'):
|
||||
print(f'Error getting claim list page {page}:')
|
||||
print(response)
|
||||
return 1
|
||||
else:
|
||||
url_to_claim.update({
|
||||
claim['permanent_url']: claim for claim in response['items']
|
||||
})
|
||||
print(f'Claim search page {page} took: {time.perf_counter() - start}')
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
print("Could not connect to daemon")
|
||||
return 1
|
||||
print(f"Checking {len(uris)} uris from the front page")
|
||||
print("**********************************************")
|
||||
|
||||
resolvable = []
|
||||
async def __resolve(name):
|
||||
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
|
||||
if 'error' not in resolved.get(name, {}):
|
||||
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
|
||||
resolvable.append(name)
|
||||
else:
|
||||
print(f"{name} has a fee, skipping it")
|
||||
else:
|
||||
print(f"failed to resolve {name}: {resolved[name]['error']}")
|
||||
await asyncio.gather(*(__resolve(name) for name in uris))
|
||||
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
|
||||
print(f"Attempting to download {len(url_to_claim)} claim_search streams")
|
||||
|
||||
first_byte_times = []
|
||||
download_speeds = []
|
||||
download_successes = []
|
||||
failed_to_start = []
|
||||
download_failures = []
|
||||
failed_to = {}
|
||||
|
||||
for uri in resolvable:
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
await asyncio.gather(*(
|
||||
daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
|
||||
for claim in url_to_claim.values() if not cmd_args.keep_files
|
||||
))
|
||||
|
||||
for i, uri in enumerate(resolvable):
|
||||
start = time.time()
|
||||
try:
|
||||
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
if not cmd_args.head_blob_only:
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
|
||||
conf, uri, cmd_args.stall_download_timeout
|
||||
)
|
||||
if downloaded:
|
||||
download_successes.append(uri)
|
||||
else:
|
||||
download_failures.append(uri)
|
||||
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
|
||||
download_speeds.append(mbs)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
|
||||
f"{mbs}mb/s")
|
||||
except Exception as e:
|
||||
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
|
||||
failed_to_start.append(uri)
|
||||
for i, (url, claim) in enumerate(url_to_claim.items()):
|
||||
start = time.perf_counter()
|
||||
response = await daemon_rpc(conf, 'get', uri=url, save_file=not cmd_args.head_blob_only)
|
||||
if 'error' in response:
|
||||
print(f"{i + 1}/{len(url_to_claim)} - failed to start {url}: {response['error']}")
|
||||
failed_to[url] = 'start'
|
||||
if cmd_args.exit_on_error:
|
||||
return
|
||||
if cmd_args.delete_after_download or cmd_args.head_blob_only:
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
continue
|
||||
first_byte = time.perf_counter()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(url_to_claim)} - {first_byte - start} {url}")
|
||||
if not cmd_args.head_blob_only:
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
|
||||
conf, claim['name'], cmd_args.stall_download_timeout
|
||||
)
|
||||
if downloaded:
|
||||
download_successes.append(url)
|
||||
else:
|
||||
failed_to[url] = 'finish'
|
||||
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.perf_counter() - start) / 1000000, 2)
|
||||
download_speeds.append(mbs)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {url} at "
|
||||
f"{mbs}mb/s")
|
||||
if not cmd_args.keep_files:
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
print("**********************************************")
|
||||
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
|
||||
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
|
||||
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
|
||||
f"Variance: {variance(first_byte_times)}\n"
|
||||
if not cmd_args.head_blob_only:
|
||||
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
|
||||
result = f"Started {len(first_byte_times)} of {len(url_to_claim)} attempted front page streams\n"
|
||||
if first_byte_times:
|
||||
result += f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
|
||||
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
|
||||
f"Variance: {variance(first_byte_times)}\n"
|
||||
if download_successes:
|
||||
result += f"Downloaded {len(download_successes)}/{len(url_to_claim)}\n" \
|
||||
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
|
||||
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
|
||||
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
|
||||
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
|
||||
|
||||
if failed_to_start:
|
||||
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
|
||||
if download_failures:
|
||||
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
|
||||
for reason in ('start', 'finish'):
|
||||
failures = [url for url, why in failed_to.items() if reason == why]
|
||||
if failures:
|
||||
result += f"\nFailed to {reason}:\n" + "\n".join(failures)
|
||||
print(result)
|
||||
|
||||
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
if webhook:
|
||||
await report_to_slack(result, webhook)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
#parser.add_argument("--data_dir")
|
||||
#parser.add_argument("--wallet_dir")
|
||||
#parser.add_argument("--download_directory")
|
||||
parser.add_argument("--allow_fees", action='store_true')
|
||||
parser.add_argument("--exit_on_error", action='store_true')
|
||||
parser.add_argument("--stall_download_timeout", default=10, type=int)
|
||||
parser.add_argument("--delete_after_download", action='store_true')
|
||||
parser.add_argument("--stall_download_timeout", default=0, type=int)
|
||||
parser.add_argument("--keep_files", action='store_true')
|
||||
parser.add_argument("--head_blob_only", action='store_true')
|
||||
asyncio.run(main(cmd_args=parser.parse_args()))
|
||||
parser.add_argument("--download_pages", type=int, default=10)
|
||||
sys.exit(asyncio.run(main(cmd_args=parser.parse_args())) or 0)
|
||||
@@ -89,6 +89,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([claim], **kwargs)
|
||||
|
||||
async def assertFindsClaims(self, claims, **kwargs):
|
||||
kwargs.setdefault('order_by', ['height', '^name'])
|
||||
results = await self.claim_search(**kwargs)
|
||||
self.assertEqual(len(claims), len(results))
|
||||
for claim, result in zip(claims, results):
|
||||
@@ -165,15 +166,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.create_channel()
|
||||
await self.create_lots_of_streams()
|
||||
|
||||
page = await self.claim_search(page_size=20, channel='@abc')
|
||||
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams)
|
||||
|
||||
page = await self.claim_search(page_size=6, channel='@abc')
|
||||
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[:6])
|
||||
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc')
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[6:])
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import time
|
||||
from math import ceil
|
||||
|
||||
from aiohttp import web
|
||||
@@ -14,6 +15,19 @@ class MockedCommentServer:
|
||||
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
|
||||
}
|
||||
|
||||
COMMENT_SCHEMA = {
|
||||
'comment': None,
|
||||
'comment_id': None,
|
||||
'claim_id': None,
|
||||
'parent_id': None,
|
||||
'channel_name': None,
|
||||
'channel_id': None,
|
||||
'signature': None,
|
||||
'signing_ts': None,
|
||||
'timestamp': None,
|
||||
'channel_url': None,
|
||||
}
|
||||
|
||||
def __init__(self, port=2903):
|
||||
self.port = port
|
||||
self.app = web.Application(debug=True)
|
||||
@@ -23,13 +37,43 @@ class MockedCommentServer:
|
||||
self.comments = []
|
||||
self.comment_id = 0
|
||||
|
||||
def create_comment(self, **comment):
|
||||
@classmethod
|
||||
def _create_comment(cls, **kwargs):
|
||||
schema = cls.COMMENT_SCHEMA.copy()
|
||||
schema.update(**kwargs)
|
||||
return schema
|
||||
|
||||
@staticmethod
|
||||
def clean(d: dict):
|
||||
return {k: v for k, v in d.items() if v}
|
||||
|
||||
def create_comment(self, channel_name=None, channel_id=None, **kwargs):
|
||||
self.comment_id += 1
|
||||
comment['comment_id'] = self.comment_id
|
||||
if 'channel_id' in comment:
|
||||
comment['channel_url'] = 'lbry://' + comment['channel_name'] + '#' + comment['channel_id']
|
||||
comment_id = self.comment_id
|
||||
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
|
||||
comment = self._create_comment(
|
||||
comment_id=str(comment_id),
|
||||
channel_name=channel_name,
|
||||
channel_id=channel_id,
|
||||
channel_url=channel_url,
|
||||
timestamp=str(int(time.time())),
|
||||
**kwargs
|
||||
)
|
||||
self.comments.append(comment)
|
||||
return comment
|
||||
return self.clean(comment)
|
||||
|
||||
def delete_comment(self, comment_id: int, channel_id: str, **kwargs):
|
||||
deleted = False
|
||||
try:
|
||||
if 0 <= comment_id <= len(self.comments) and self.comments[comment_id - 1]['channel_id'] == channel_id:
|
||||
self.comments.pop(comment_id - 1)
|
||||
deleted = True
|
||||
finally:
|
||||
return {
|
||||
str(comment_id): {
|
||||
'deleted': deleted
|
||||
}
|
||||
}
|
||||
|
||||
def get_claim_comments(self, page=1, page_size=50, **kwargs):
|
||||
return {
|
||||
@@ -37,22 +81,36 @@ class MockedCommentServer:
|
||||
'page_size': page_size,
|
||||
'total_pages': ceil(len(self.comments)/page_size),
|
||||
'total_items': len(self.comments),
|
||||
'items': (self.comments[::-1])[(page - 1) * page_size: page * page_size]
|
||||
'items': [self.clean(c) for c in (self.comments[::-1])[(page - 1) * page_size: page * page_size]]
|
||||
}
|
||||
|
||||
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
|
||||
comment = self.comments[comment_id - 1]
|
||||
return {
|
||||
'channel_id': comment.get('channel_id'),
|
||||
'channel_name': comment.get('channel_name')
|
||||
}
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
'create_comment': create_comment,
|
||||
'delete_comment': delete_comment,
|
||||
'get_channel_from_comment_id': get_comment_channel_by_id,
|
||||
}
|
||||
|
||||
def process_json(self, body) -> dict:
|
||||
response = {'jsonrpc': '2.0', 'id': body['id']}
|
||||
if body['method'] in self.methods:
|
||||
params = body.get('params', {})
|
||||
result = self.methods[body['method']](self, **params)
|
||||
response['result'] = result
|
||||
else:
|
||||
response['error'] = self.ERRORS['INVALID_METHOD']
|
||||
try:
|
||||
if body['method'] in self.methods:
|
||||
params = body.get('params', {})
|
||||
if 'comment_id' in params and type(params['comment_id']) is str:
|
||||
params['comment_id'] = int(params['comment_id'])
|
||||
result = self.methods[body['method']](self, **params)
|
||||
response['result'] = result
|
||||
else:
|
||||
response['error'] = self.ERRORS['INVALID_METHOD']
|
||||
except Exception:
|
||||
response['error'] = self.ERRORS['UNKNOWN']
|
||||
return response
|
||||
|
||||
async def start(self):
|
||||
@@ -173,3 +231,20 @@ class CommentCommands(CommandTestCase):
|
||||
is_channel_signature_valid=True
|
||||
)
|
||||
self.assertIs(len(signed_comment_list['items']), 28)
|
||||
|
||||
async def test04_comment_abandons(self):
|
||||
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
|
||||
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
|
||||
claim_id=stream['claim_id'],
|
||||
channel_id=rswanson['claim_id']
|
||||
)
|
||||
self.assertIn('signature', comment)
|
||||
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], deleted)
|
||||
self.assertTrue(deleted[comment['comment_id']]['deleted'])
|
||||
|
||||
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], deleted)
|
||||
self.assertFalse(deleted[comment['comment_id']]['deleted'])
|
||||
@@ -1,9 +1,11 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.orchstr8 import Conductor
|
||||
from torba.testcase import IntegrationTestCase
|
||||
import lbry.wallet
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.extras.daemon.Components import HeadersComponent
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
|
||||
class TestSessionBloat(IntegrationTestCase):
|
||||
@@ -15,6 +17,7 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
|
||||
async def test_session_bloat_from_socket_timeout(self):
|
||||
await self.conductor.stop_spv()
|
||||
await self.ledger.stop()
|
||||
self.conductor.spv_node.session_timeout = 1
|
||||
await self.conductor.start_spv()
|
||||
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
|
||||
@@ -28,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertTrue(session.is_closing())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
|
||||
|
||||
|
||||
class TestHeadersComponent(CommandTestCase):
|
||||
|
||||
LEDGER = lbry.wallet
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.component_manager = self.daemon.component_manager
|
||||
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
|
||||
self.headers_component = HeadersComponent(self.component_manager)
|
||||
|
||||
async def test_cant_reach_host(self):
|
||||
HeadersComponent.HEADERS_URL = 'notthere/'
|
||||
os.unlink(self.headers_component.headers.path)
|
||||
# test is that this doesnt raise
|
||||
await self.headers_component.start()
|
||||
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
|
||||
self.assertEqual(await self.headers_component.get_status(), {})
|
||||
@@ -1,5 +1,5 @@
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
import hashlib
|
||||
from lbry.extras.daemon.comment_client import sign_comment
|
||||
from lbry.extras.daemon.comment_client import is_comment_signed_by_channel
|
||||
|
||||
@@ -14,7 +14,8 @@ class TestSigningComments(AsyncioTestCase):
|
||||
'claim_id': claim.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
'channel_id': channel.claim_id,
|
||||
'comment': comment
|
||||
'comment': comment,
|
||||
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
|
||||
}
|
||||
|
||||
def test01_successful_create_sign_and_validate_comment(self):
|
||||
@@ -31,3 +32,28 @@ class TestSigningComments(AsyncioTestCase):
|
||||
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
|
||||
sign_comment(comment, channel2)
|
||||
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
|
||||
|
||||
def test03_successful_sign_abandon_comment(self):
|
||||
rswanson = get_channel('@RonSwanson')
|
||||
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
|
||||
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
|
||||
sign_comment(comment_body, rswanson, abandon=True)
|
||||
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
|
||||
|
||||
def test04_invalid_signature(self):
|
||||
rswanson = get_channel('@RonSwanson')
|
||||
jeanralphio = get_channel('@JeanRalphio')
|
||||
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
|
||||
chair_comment = self.create_claim_comment_body(
|
||||
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
|
||||
chair,
|
||||
rswanson
|
||||
)
|
||||
sign_comment(chair_comment, rswanson)
|
||||
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
|
||||
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
|
||||
fake_abandon_signal = chair_comment.copy()
|
||||
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
|
||||
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
|
||||
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
|
||||
|
||||
@@ -11,7 +11,7 @@ class TestTagNormalization(unittest.TestCase):
|
||||
def test_normalize_tag(self):
|
||||
tag = self.assertNormalizedTag
|
||||
tag('', ' \t #!~')
|
||||
tag('tag', 'Tag')
|
||||
tag('tag', 'T\'ag')
|
||||
tag('t ag', '\tT \nAG ')
|
||||
tag('tag hash', '#tag~#hash!')
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import time
|
||||
import unittest
|
||||
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
|
||||
|
||||
|
||||
class TestPercentileCalculation(unittest.TestCase):
|
||||
|
||||
def test_calculate_percentiles(self):
|
||||
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
|
||||
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
|
||||
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
|
||||
self.assertEqual(calculate_avg_percentiles(
|
||||
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
|
||||
|
||||
|
||||
class TestCollectingMetrics(unittest.TestCase):
|
||||
|
||||
def test_happy_path(self):
|
||||
self.maxDiff = None
|
||||
load = ServerLoadData()
|
||||
search = load.for_api('search')
|
||||
self.assertEqual(search.name, 'search')
|
||||
search.start()
|
||||
search.cache_response()
|
||||
search.cache_response()
|
||||
metrics = {
|
||||
'search': [{'total': 40}],
|
||||
'execute_query': [
|
||||
{'total': 20},
|
||||
{'total': 10}
|
||||
]
|
||||
}
|
||||
for x in range(5):
|
||||
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
|
||||
metrics['execute_query'][0]['total'] = 10
|
||||
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
|
||||
search.query_interrupt(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.052, {})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
|
||||
"receive_count": 1,
|
||||
"cache_response_count": 2,
|
||||
"query_response_count": 5,
|
||||
"intrp_response_count": 1,
|
||||
"error_response_count": 2,
|
||||
"response": (53, 51, 51, 52, 53, 54, 55, 55),
|
||||
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
|
||||
"error": (51, 50, 50, 50, 50, 52, 52, 52),
|
||||
"python": (12, 10, 10, 10, 10, 20, 20, 20),
|
||||
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
|
||||
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
|
||||
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
|
||||
"individual_sql_count": 14,
|
||||
"errored_queries": ['FROM claim where something=1'],
|
||||
"interrupted_queries": ['FROM claim where something=1'],
|
||||
}}})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
|
||||
@@ -53,35 +53,45 @@ class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
async def _make_fake_server(self, latency=1.0, port=1337):
|
||||
# local fake server with artificial latency
|
||||
proto = RPCSession()
|
||||
async def __handler(_):
|
||||
await asyncio.sleep(latency)
|
||||
return {'height': 1}
|
||||
|
||||
proto.handle_request = __handler
|
||||
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
# local fake server with artificial latency
|
||||
class FakeSession(RPCSession):
|
||||
async def handle_request(self, request):
|
||||
await asyncio.sleep(latency)
|
||||
return {"height": 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
async def echo(reader, writer):
|
||||
while True:
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
await self._make_fake_server(latency=1.5, port=1340),
|
||||
await self._make_fake_server(latency=0.1, port=1337),
|
||||
await self._make_fake_server(latency=1.0, port=1339),
|
||||
await self._make_fake_server(latency=0.5, port=1338),
|
||||
# fast but unhealthy, should be discarded
|
||||
await self._make_bad_server(),
|
||||
('localhost', 1),
|
||||
('example.that.doesnt.resolve', 9000),
|
||||
await self._make_fake_server(latency=1.2, port=1340),
|
||||
await self._make_fake_server(latency=0.5, port=1337),
|
||||
await self._make_fake_server(latency=0.7, port=1339),
|
||||
],
|
||||
'connect_timeout': 30
|
||||
'connect_timeout': 3
|
||||
})
|
||||
|
||||
network = BaseNetwork(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=3)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
# ensure we are connected to all of them
|
||||
self.assertEqual(len(network.session_pool.sessions), 4)
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
|
||||
self.assertEqual(len(network.session_pool.sessions), 3)
|
||||
@@ -1,11 +1,10 @@
|
||||
import unittest
|
||||
import sqlite3
|
||||
from functools import wraps
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
|
||||
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
|
||||
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
|
||||
from torba.client.hash import sha256
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
@@ -167,6 +166,26 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
("select * from foo LIMIT 20 OFFSET 10", {})
|
||||
)
|
||||
|
||||
def test_query_interpolation(self):
|
||||
self.maxDiff = None
|
||||
# tests that interpolation replaces longer keys first
|
||||
self.assertEqual(
|
||||
interpolate(*query(
|
||||
"select * from foo",
|
||||
a__not='b', b__in='select * from blah where c=:$c',
|
||||
d__any={'one__like': 'o', 'two': 2},
|
||||
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
|
||||
ahash=memoryview(sha256(b'hello world')),
|
||||
limit=10, order_by='b', **{'$c': 3})
|
||||
),
|
||||
"select * from foo WHERE a != 'b' AND "
|
||||
"b IN (select * from blah where c=3) AND "
|
||||
"(one LIKE 'o' OR two = 2) AND "
|
||||
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
|
||||
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
|
||||
"ORDER BY b LIMIT 10",
|
||||
)
|
||||
|
||||
|
||||
class TestQueries(AsyncioTestCase):
|
||||
|
||||
|
||||
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
# corrupt by appending
|
||||
headers.io.seek(block_bytes(len(headers)))
|
||||
headers.io.write(b"appending")
|
||||
headers._size = None
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 1499)
|
||||
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
|
||||
|
||||
@@ -18,6 +18,9 @@ class MockNetwork:
|
||||
self.get_transaction_called = []
|
||||
self.is_connected = False
|
||||
|
||||
def retriable_call(self, function, *args, **kwargs):
|
||||
return function(*args, **kwargs)
|
||||
|
||||
async def get_history(self, address):
|
||||
self.get_history_called.append(address)
|
||||
self.address = address
|
||||
@@ -121,8 +124,9 @@ class TestSynchronization(LedgerTestCase):
|
||||
)
|
||||
|
||||
|
||||
class MocHeaderNetwork:
|
||||
class MocHeaderNetwork(MockNetwork):
|
||||
def __init__(self, responses):
|
||||
super().__init__(None, None)
|
||||
self.responses = responses
|
||||
|
||||
async def get_headers(self, height, blocks):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from asyncio import wrap_future
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
|
||||
@@ -191,6 +192,19 @@ def query(select, **constraints):
|
||||
return ' '.join(sql), values
|
||||
|
||||
|
||||
def interpolate(sql, values):
|
||||
for k in sorted(values.keys(), reverse=True):
|
||||
value = values[k]
|
||||
if isinstance(value, memoryview):
|
||||
value = f"X'{hexlify(bytes(value)).decode()}'"
|
||||
elif isinstance(value, str):
|
||||
value = f"'{value}'"
|
||||
else:
|
||||
value = str(value)
|
||||
sql = sql.replace(f":{k}", value)
|
||||
return sql
|
||||
|
||||
|
||||
def rows_to_dict(rows, fields):
|
||||
if rows:
|
||||
return [dict(zip(fields, r)) for r in rows]
|
||||
|
||||
@@ -70,8 +70,10 @@ class BaseHeaders:
|
||||
return True
|
||||
|
||||
def __getitem__(self, height) -> dict:
|
||||
assert not isinstance(height, slice), \
|
||||
"Slicing of header chain has not been implemented yet."
|
||||
if isinstance(height, slice):
|
||||
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return self.deserialize(height, self.get_raw_header(height))
|
||||
|
||||
def get_raw_header(self, height) -> bytes:
|
||||
@@ -115,7 +117,7 @@ class BaseHeaders:
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
self._size = self.io.tell() // self.header_size
|
||||
added += written
|
||||
if bail:
|
||||
break
|
||||
@@ -170,10 +172,12 @@ class BaseHeaders:
|
||||
|
||||
async def repair(self):
|
||||
previous_header_hash = fail = None
|
||||
self.io.seek(0)
|
||||
batch_size = 10000
|
||||
batch_size = 36
|
||||
for start_height in range(0, self.height, batch_size):
|
||||
self.io.seek(self.header_size * start_height)
|
||||
headers = self.io.read(self.header_size*batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
||||
height = header['block_height']
|
||||
if height:
|
||||
@@ -184,7 +188,7 @@ class BaseHeaders:
|
||||
fail = True
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
@@ -205,7 +209,7 @@ class BaseHeaders:
|
||||
end = min(len(headers), end + self.chunk_size * self.header_size)
|
||||
|
||||
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
|
||||
assert len(headers) % self.header_size == 0
|
||||
assert len(headers) % self.header_size == 0, len(headers)
|
||||
for idx in range(len(headers) // self.header_size):
|
||||
start, end = idx * self.header_size, (idx + 1) * self.header_size
|
||||
header = headers[start:end]
|
||||
|
||||
@@ -287,7 +287,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
subscription_update = False
|
||||
|
||||
if not headers:
|
||||
header_response = await self.network.get_headers(height, 2001)
|
||||
header_response = await self.network.retriable_call(self.network.get_headers, height, 2001)
|
||||
headers = header_response['hex']
|
||||
|
||||
if not headers:
|
||||
@@ -394,7 +394,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
if local_status == remote_status:
|
||||
return
|
||||
|
||||
remote_history = await self.network.get_history(address)
|
||||
remote_history = await self.network.retriable_call(self.network.get_history, address)
|
||||
|
||||
cache_tasks = []
|
||||
synced_history = StringIO()
|
||||
@@ -447,6 +447,17 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
if address_manager is not None:
|
||||
await address_manager.ensure_address_gap()
|
||||
|
||||
local_status, local_history = await self.get_local_status_and_history(address)
|
||||
if local_status != remote_status:
|
||||
log.debug(
|
||||
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
|
||||
remote_status, len(remote_history), local_status, len(local_history)
|
||||
)
|
||||
log.debug("local: %s", local_history)
|
||||
log.debug("remote: %s", remote_history)
|
||||
else:
|
||||
log.debug("Sync completed for: %s", address)
|
||||
|
||||
async def cache_transaction(self, txid, remote_height):
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
@@ -466,7 +477,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw = await self.network.get_transaction(txid)
|
||||
_raw = await self.network.retriable_call(self.network.get_transaction, txid)
|
||||
if _raw:
|
||||
tx = self.transaction_class(unhexlify(_raw))
|
||||
await self.maybe_verify_transaction(tx, remote_height)
|
||||
@@ -486,8 +497,8 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height):
|
||||
tx.height = remote_height
|
||||
if 0 < remote_height <= len(self.headers):
|
||||
merkle = await self.network.get_merkle(tx.id, remote_height)
|
||||
if 0 < remote_height < len(self.headers):
|
||||
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
|
||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||
header = self.headers[remote_height]
|
||||
tx.position = merkle['pos']
|
||||
@@ -501,6 +512,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
return None
|
||||
|
||||
def broadcast(self, tx):
|
||||
# broadcast cant be a retriable call yet
|
||||
return self.network.broadcast(hexlify(tx.raw).decode())
|
||||
|
||||
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
|
||||
@@ -522,4 +534,4 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
)) for address_record in records
|
||||
], timeout=timeout)
|
||||
if pending:
|
||||
raise TimeoutError('Timed out waiting for transaction.')
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
+244
-144
@@ -1,9 +1,8 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from asyncio import CancelledError
|
||||
from time import time
|
||||
from typing import List
|
||||
import socket
|
||||
from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
from time import perf_counter
|
||||
|
||||
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
|
||||
|
||||
@@ -14,67 +13,146 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
|
||||
def __init__(self, *args, network, server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.bw_limit = -1
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
self.ping_task = None
|
||||
self.response_time: Optional[float] = None
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
self._semaphore = asyncio.Semaphore(int(self.timeout))
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
return not self.is_closing() and self._can_send.is_set() and self.response_time is not None
|
||||
|
||||
@property
|
||||
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
|
||||
if not self.transport:
|
||||
return None
|
||||
return self.transport.get_extra_info('peername')
|
||||
|
||||
async def send_timed_server_version_request(self, args=(), timeout=None):
|
||||
timeout = timeout or self.timeout
|
||||
log.debug("send version request to %s:%i", *self.server)
|
||||
start = perf_counter()
|
||||
result = await asyncio.wait_for(
|
||||
super().send_request('server.version', args), timeout=timeout
|
||||
)
|
||||
current_response_time = perf_counter() - start
|
||||
response_sum = (self.response_time or 0) * self._response_samples + current_response_time
|
||||
self.response_time = response_sum / (self._response_samples + 1)
|
||||
self._response_samples += 1
|
||||
return result
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
try:
|
||||
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
|
||||
except RPCError as e:
|
||||
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
|
||||
raise e
|
||||
except asyncio.TimeoutError:
|
||||
self.abort()
|
||||
raise
|
||||
self.pending_amount += 1
|
||||
async with self._semaphore:
|
||||
return await self._send_request(method, args)
|
||||
|
||||
async def ping_forever(self):
|
||||
async def _send_request(self, method, args=()):
|
||||
log.debug("send %s to %s:%i", method, *self.server)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
reply = await self.send_timed_server_version_request(args, self.timeout)
|
||||
else:
|
||||
reply = await asyncio.wait_for(
|
||||
super().send_request(method, args), timeout=self.timeout
|
||||
)
|
||||
log.debug("got reply for %s from %s:%i", method, *self.server)
|
||||
return reply
|
||||
except RPCError as e:
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
raise e
|
||||
except ConnectionError:
|
||||
log.warning("connection to %s:%i lost", *self.server)
|
||||
self.synchronous_close()
|
||||
raise asyncio.CancelledError(f"connection to {self.server[0]}:{self.server[1]} lost")
|
||||
except asyncio.TimeoutError:
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
while not self.is_closing():
|
||||
if (time() - self.last_send) > self.max_seconds_idle:
|
||||
await self.send_request('server.banner')
|
||||
await asyncio.sleep(self.max_seconds_idle//3)
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required='1.2', timeout=3):
|
||||
return await asyncio.wait_for(
|
||||
self.send_request('server.version', [__version__, required]), timeout=timeout
|
||||
)
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
start = perf_counter()
|
||||
await asyncio.wait_for(connector.create_connection(), timeout=timeout)
|
||||
self.ping_task = asyncio.create_task(self.ping_forever())
|
||||
self.connection_latency = perf_counter() - start
|
||||
|
||||
async def handle_request(self, request):
|
||||
controller = self.network.subscription_controllers[request.method]
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_disconnect_controller.add(True)
|
||||
if self.ping_task:
|
||||
self.ping_task.cancel()
|
||||
|
||||
|
||||
class BaseNetwork:
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.config = ledger.config
|
||||
self.client: ClientSession = None
|
||||
self.session_pool: SessionPool = None
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
|
||||
self._on_connected_controller = StreamController()
|
||||
self.on_connected = self._on_connected_controller.stream
|
||||
|
||||
self._on_header_controller = StreamController()
|
||||
self._on_header_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_header = self._on_header_controller.stream
|
||||
|
||||
self._on_status_controller = StreamController()
|
||||
self._on_status_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_status = self._on_status_controller.stream
|
||||
|
||||
self.subscription_controllers = {
|
||||
@@ -82,79 +160,72 @@ class BaseNetwork:
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
}
|
||||
|
||||
async def switch_to_fastest(self):
|
||||
try:
|
||||
client = await asyncio.wait_for(self.session_pool.wait_for_fastest_session(), 30)
|
||||
except asyncio.TimeoutError:
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
self.client = None
|
||||
for session in self.session_pool.sessions:
|
||||
session.synchronous_close()
|
||||
log.warning("not connected to any wallet servers")
|
||||
return
|
||||
current_client = self.client
|
||||
self.client = client
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
self._on_connected_controller.add(True)
|
||||
try:
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except asyncio.TimeoutError:
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
self.client = current_client
|
||||
return
|
||||
self.session_pool.new_connection_event.clear()
|
||||
return await self.session_pool.new_connection_event.wait()
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
connect_timeout = self.config.get('connect_timeout', 6)
|
||||
self.session_pool = SessionPool(network=self, timeout=connect_timeout)
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
while True:
|
||||
try:
|
||||
self.client = await self.pick_fastest_session()
|
||||
if self.is_connected:
|
||||
await self.ensure_server_version()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
log.info("Successfully connected to SPV wallet server: %s:%d", *self.client.server)
|
||||
self._on_connected_controller.add(True)
|
||||
await self.client.on_disconnected.first
|
||||
except CancelledError:
|
||||
self.running = False
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out while trying to find a server!")
|
||||
except Exception: # pylint: disable=broad-except
|
||||
log.exception("Exception while trying to find a server!")
|
||||
if not self.running:
|
||||
return
|
||||
elif self.client:
|
||||
await self.client.close()
|
||||
self.client.connection.cancel_pending_requests()
|
||||
while self.running:
|
||||
await self.switch_to_fastest()
|
||||
|
||||
async def stop(self):
|
||||
self.running = False
|
||||
if self.session_pool:
|
||||
self.session_pool.stop()
|
||||
if self.is_connected:
|
||||
disconnected = self.client.on_disconnected.first
|
||||
await self.client.close()
|
||||
await disconnected
|
||||
self.session_pool.stop()
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client is not None and not self.client.is_closing()
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args):
|
||||
if self.is_connected:
|
||||
return self.client.send_request(list_or_method, args)
|
||||
def rpc(self, list_or_method, args, session=None):
|
||||
session = session or self.session_pool.fastest_session
|
||||
if session:
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def pick_fastest_session(self):
|
||||
sessions = await self.session_pool.get_online_sessions()
|
||||
done, pending = await asyncio.wait([
|
||||
self.probe_session(session)
|
||||
for session in sessions if not session.is_closing()
|
||||
], return_when='FIRST_COMPLETED')
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
for session in done:
|
||||
return await session
|
||||
|
||||
async def probe_session(self, session: ClientSession):
|
||||
await session.send_request('server.banner')
|
||||
return session
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
def ensure_server_version(self, required='1.2'):
|
||||
return self.rpc('server.version', [__version__, required])
|
||||
|
||||
def broadcast(self, raw_transaction):
|
||||
return self.rpc('blockchain.transaction.broadcast', [raw_transaction])
|
||||
|
||||
def get_history(self, address):
|
||||
return self.rpc('blockchain.address.get_history', [address])
|
||||
|
||||
def get_transaction(self, tx_hash):
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash])
|
||||
|
||||
@@ -167,82 +238,111 @@ class BaseNetwork:
|
||||
def get_headers(self, height, count=10000):
|
||||
return self.rpc('blockchain.block.headers', [height, count])
|
||||
|
||||
def subscribe_headers(self):
|
||||
return self.rpc('blockchain.headers.subscribe', [True])
|
||||
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
|
||||
def get_history(self, address):
|
||||
return self.rpc('blockchain.address.get_history', [address], session=self.client)
|
||||
|
||||
def subscribe_address(self, address):
|
||||
return self.rpc('blockchain.address.subscribe', [address])
|
||||
def broadcast(self, raw_transaction):
|
||||
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], session=self.client)
|
||||
|
||||
def subscribe_headers(self):
|
||||
return self.rpc('blockchain.headers.subscribe', [True], session=self.client)
|
||||
|
||||
async def subscribe_address(self, address):
|
||||
try:
|
||||
return await self.rpc('blockchain.address.subscribe', [address], session=self.client)
|
||||
except asyncio.TimeoutError:
|
||||
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
|
||||
self.client.abort()
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
|
||||
class SessionPool:
|
||||
|
||||
def __init__(self, network: BaseNetwork, timeout: float):
|
||||
self.network = network
|
||||
self.sessions: List[ClientSession] = []
|
||||
self._dead_servers: List[ClientSession] = []
|
||||
self.maintain_connections_task = None
|
||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
||||
self.timeout = timeout
|
||||
# triggered when the master server is out, to speed up reconnect
|
||||
self._lost_master = asyncio.Event()
|
||||
self.new_connection_event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def online(self):
|
||||
for session in self.sessions:
|
||||
if not session.is_closing():
|
||||
return True
|
||||
return False
|
||||
return any(not session.is_closing() for session in self.sessions)
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return [session for session in self.sessions if session.available]
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.available_sessions:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
def _get_session_connect_callback(self, session: ClientSession):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback():
|
||||
duplicate_connections = [
|
||||
s for s in self.sessions
|
||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
||||
]
|
||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
||||
if already_connected:
|
||||
self.sessions.pop(session).cancel()
|
||||
session.synchronous_close()
|
||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
||||
session.server[0], already_connected.server[0])
|
||||
loop.call_later(3600, self._connect_session, session.server)
|
||||
return
|
||||
self.new_connection_event.set()
|
||||
log.info("connected to %s:%i", *session.server)
|
||||
|
||||
return callback
|
||||
|
||||
def _connect_session(self, server: Tuple[str, int]):
|
||||
session = None
|
||||
for s in self.sessions:
|
||||
if s.server == server:
|
||||
session = s
|
||||
break
|
||||
if not session:
|
||||
session = ClientSession(
|
||||
network=self.network, server=server
|
||||
)
|
||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
||||
task = self.sessions.get(session, None)
|
||||
if not task or task.done():
|
||||
task = asyncio.create_task(session.ensure_session())
|
||||
task.add_done_callback(lambda _: self.ensure_connections())
|
||||
self.sessions[session] = task
|
||||
|
||||
def start(self, default_servers):
|
||||
self.sessions = [
|
||||
ClientSession(network=self.network, server=server)
|
||||
for server in default_servers
|
||||
]
|
||||
self.maintain_connections_task = asyncio.create_task(self.ensure_connections())
|
||||
for server in default_servers:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
if self.maintain_connections_task:
|
||||
self.maintain_connections_task.cancel()
|
||||
for task in self.sessions.values():
|
||||
task.cancel()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
for session in self.sessions:
|
||||
if not session.is_closing():
|
||||
session.abort()
|
||||
self.sessions, self._dead_servers, self.maintain_connections_task = [], [], None
|
||||
self._connect_session(session.server)
|
||||
|
||||
async def ensure_connections(self):
|
||||
while True:
|
||||
await asyncio.gather(*[
|
||||
self.ensure_connection(session)
|
||||
for session in self.sessions
|
||||
], return_exceptions=True)
|
||||
try:
|
||||
await asyncio.wait_for(self._lost_master.wait(), timeout=3)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
self._lost_master.clear()
|
||||
if not self.sessions:
|
||||
self.sessions.extend(self._dead_servers)
|
||||
self._dead_servers = []
|
||||
def trigger_nodelay_connect(self):
|
||||
# used when other parts of the system sees we might have internet back
|
||||
# bypasses the retry interval
|
||||
for session in self.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
|
||||
async def ensure_connection(self, session):
|
||||
if not session.is_closing():
|
||||
return
|
||||
try:
|
||||
return await session.create_connection(self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timeout connecting to %s:%d", *session.server)
|
||||
except asyncio.CancelledError: # pylint: disable=try-except-raise
|
||||
raise
|
||||
except socket.gaierror:
|
||||
log.warning("Could not resolve IP for %s", session.server[0])
|
||||
except Exception as err: # pylint: disable=broad-except
|
||||
if 'Connect call failed' in str(err):
|
||||
log.warning("Could not connect to %s:%d", *session.server)
|
||||
else:
|
||||
log.exception("Connecting to %s:%d raised an exception:", *session.server)
|
||||
self._dead_servers.append(session)
|
||||
self.sessions.remove(session)
|
||||
|
||||
async def get_online_sessions(self):
|
||||
self._lost_master.set()
|
||||
while not self.online:
|
||||
await asyncio.sleep(0.1)
|
||||
return self.sessions
|
||||
async def wait_for_fastest_session(self):
|
||||
while not self.fastest_session:
|
||||
self.trigger_nodelay_connect()
|
||||
self.new_connection_event.clear()
|
||||
await self.new_connection_event.wait()
|
||||
return self.fastest_session
|
||||
@@ -206,7 +206,8 @@ class SPVNode:
|
||||
'HOST': self.hostname,
|
||||
'TCP_PORT': str(self.port),
|
||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||
'MAX_QUERY_WORKERS': '0'
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
}
|
||||
# TODO: don't use os.environ
|
||||
os.environ.update(conf)
|
||||
|
||||
@@ -33,6 +33,7 @@ __all__ = ('JSONRPC', 'JSONRPCv1', 'JSONRPCv2', 'JSONRPCLoose',
|
||||
import itertools
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
from functools import partial
|
||||
from numbers import Number
|
||||
|
||||
@@ -745,9 +746,8 @@ class JSONRPCConnection(object):
|
||||
self._protocol = item
|
||||
return self.receive_message(message)
|
||||
|
||||
def cancel_pending_requests(self):
|
||||
"""Cancel all pending requests."""
|
||||
exception = CancelledError()
|
||||
def raise_pending_requests(self, exception):
|
||||
exception = exception or asyncio.TimeoutError()
|
||||
for request, event in self._requests.values():
|
||||
event.result = exception
|
||||
event.set()
|
||||
|
||||
@@ -103,7 +103,7 @@ class SessionBase(asyncio.Protocol):
|
||||
# Force-close a connection if a send doesn't succeed in this time
|
||||
self.max_send_delay = 60
|
||||
# Statistics. The RPC object also keeps its own statistics.
|
||||
self.start_time = time.time()
|
||||
self.start_time = time.perf_counter()
|
||||
self.errors = 0
|
||||
self.send_count = 0
|
||||
self.send_size = 0
|
||||
@@ -123,7 +123,7 @@ class SessionBase(asyncio.Protocol):
|
||||
# A non-positive value means not to limit concurrency
|
||||
if self.bw_limit <= 0:
|
||||
return
|
||||
now = time.time()
|
||||
now = time.perf_counter()
|
||||
# Reduce the recorded usage in proportion to the elapsed time
|
||||
refund = (now - self.bw_time) * (self.bw_limit / 3600)
|
||||
self.bw_charge = max(0, self.bw_charge - int(refund))
|
||||
@@ -146,7 +146,7 @@ class SessionBase(asyncio.Protocol):
|
||||
await asyncio.wait_for(self._can_send.wait(), secs)
|
||||
except asyncio.TimeoutError:
|
||||
self.abort()
|
||||
raise asyncio.CancelledError(f'task timed out after {secs}s')
|
||||
raise asyncio.TimeoutError(f'task timed out after {secs}s')
|
||||
|
||||
async def _send_message(self, message):
|
||||
if not self._can_send.is_set():
|
||||
@@ -156,7 +156,7 @@ class SessionBase(asyncio.Protocol):
|
||||
self.send_size += len(framed_message)
|
||||
self._using_bandwidth(len(framed_message))
|
||||
self.send_count += 1
|
||||
self.last_send = time.time()
|
||||
self.last_send = time.perf_counter()
|
||||
if self.verbosity >= 4:
|
||||
self.logger.debug(f'Sending framed message {framed_message}')
|
||||
self.transport.write(framed_message)
|
||||
@@ -215,7 +215,8 @@ class SessionBase(asyncio.Protocol):
|
||||
self._address = None
|
||||
self.transport = None
|
||||
self._task_group.cancel()
|
||||
self._pm_task.cancel()
|
||||
if self._pm_task:
|
||||
self._pm_task.cancel()
|
||||
# Release waiting tasks
|
||||
self._can_send.set()
|
||||
|
||||
@@ -253,6 +254,7 @@ class SessionBase(asyncio.Protocol):
|
||||
if self.transport:
|
||||
self.transport.abort()
|
||||
|
||||
# TODO: replace with synchronous_close
|
||||
async def close(self, *, force_after=30):
|
||||
"""Close the connection and return when closed."""
|
||||
self._close()
|
||||
@@ -262,6 +264,11 @@ class SessionBase(asyncio.Protocol):
|
||||
self.abort()
|
||||
await self._pm_task
|
||||
|
||||
def synchronous_close(self):
|
||||
self._close()
|
||||
if self._pm_task and not self._pm_task.done():
|
||||
self._pm_task.cancel()
|
||||
|
||||
|
||||
class MessageSession(SessionBase):
|
||||
"""Session class for protocols where messages are not tied to responses,
|
||||
@@ -296,7 +303,7 @@ class MessageSession(SessionBase):
|
||||
)
|
||||
self._bump_errors()
|
||||
else:
|
||||
self.last_recv = time.time()
|
||||
self.last_recv = time.perf_counter()
|
||||
self.recv_count += 1
|
||||
if self.recv_count % 10 == 0:
|
||||
await self._update_concurrency()
|
||||
@@ -416,7 +423,7 @@ class RPCSession(SessionBase):
|
||||
self.logger.warning(f'{e!r}')
|
||||
continue
|
||||
|
||||
self.last_recv = time.time()
|
||||
self.last_recv = time.perf_counter()
|
||||
self.recv_count += 1
|
||||
if self.recv_count % 10 == 0:
|
||||
await self._update_concurrency()
|
||||
@@ -456,7 +463,7 @@ class RPCSession(SessionBase):
|
||||
|
||||
def connection_lost(self, exc):
|
||||
# Cancel pending requests and message processing
|
||||
self.connection.cancel_pending_requests()
|
||||
self.connection.raise_pending_requests(exc)
|
||||
super().connection_lost(exc)
|
||||
|
||||
# External API
|
||||
@@ -473,6 +480,8 @@ class RPCSession(SessionBase):
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
"""Send an RPC request over the network."""
|
||||
if self.is_closing():
|
||||
raise asyncio.TimeoutError("Trying to send request on a recently dropped connection.")
|
||||
message, event = self.connection.send_request(Request(method, args))
|
||||
await self._send_message(message)
|
||||
await event.wait()
|
||||
|
||||
@@ -38,6 +38,7 @@ class Env:
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
|
||||
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
|
||||
|
||||
@@ -258,7 +258,7 @@ class SessionManager:
|
||||
session_timeout = self.env.session_timeout
|
||||
while True:
|
||||
await sleep(session_timeout // 10)
|
||||
stale_cutoff = time.time() - session_timeout
|
||||
stale_cutoff = time.perf_counter() - session_timeout
|
||||
stale_sessions = [session for session in self.sessions
|
||||
if session.last_recv < stale_cutoff]
|
||||
if stale_sessions:
|
||||
|
||||
@@ -45,10 +45,12 @@ class BroadcastSubscription:
|
||||
|
||||
class StreamController:
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, merge_repeated_events=False):
|
||||
self.stream = Stream(self)
|
||||
self._first_subscription = None
|
||||
self._last_subscription = None
|
||||
self._last_event = None
|
||||
self._merge_repeated = merge_repeated_events
|
||||
|
||||
@property
|
||||
def has_listener(self):
|
||||
@@ -76,8 +78,10 @@ class StreamController:
|
||||
return f
|
||||
|
||||
def add(self, event):
|
||||
skip = self._merge_repeated and event == self._last_event
|
||||
self._last_event = event
|
||||
return self._notify_and_ensure_future(
|
||||
lambda subscription: subscription._add(event)
|
||||
lambda subscription: None if skip else subscription._add(event)
|
||||
)
|
||||
|
||||
def add_error(self, exception):
|
||||
@@ -141,8 +145,8 @@ class Stream:
|
||||
def first(self):
|
||||
future = asyncio.get_event_loop().create_future()
|
||||
subscription = self.listen(
|
||||
lambda value: self._cancel_and_callback(subscription, future, value),
|
||||
lambda exception: self._cancel_and_error(subscription, future, exception)
|
||||
lambda value: not future.done() and self._cancel_and_callback(subscription, future, value),
|
||||
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception)
|
||||
)
|
||||
return future
|
||||
|
||||
|
||||
Reference in new issue
Block a user