Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7885f34221 | ||
|
|
ff05eaae48 | ||
|
|
b48b67ed10 | ||
|
|
3ead6f63b6 | ||
|
|
77235f0ad8 | ||
|
|
f6a80ba929 | ||
|
|
295ceb90d5 | ||
|
|
df41a82b23 | ||
|
|
34500e07c6 | ||
|
|
1e6097e3b2 | ||
|
|
521facedc4 | ||
|
|
de80a2ba97 | ||
|
|
3afd615447 | ||
|
|
87e52bec0a | ||
|
|
b89f04b1f1 | ||
|
|
e4f802e5f4 | ||
|
|
16fc2c456f | ||
|
|
ef7656ae98 | ||
|
|
ad2df4871e | ||
|
|
8d5f50b695 | ||
|
|
bbc056eef0 | ||
|
|
82fd3c632e | ||
|
|
dbae939808 | ||
|
|
c2dcacb0de | ||
|
|
8f45618cd0 | ||
|
|
29a87c850c | ||
|
|
395518ff76 | ||
|
|
08da763327 | ||
|
|
d7ec3c182d | ||
|
|
ffe6418392 | ||
|
|
88adf3ee27 | ||
|
|
f406db85dc | ||
|
|
d5f8af471a | ||
|
|
046459a72d | ||
|
|
49b0f59790 | ||
|
|
71551058ce |
No files matched your search
+2
-1
@@ -78,7 +78,8 @@ jobs:
|
||||
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
api_key:
|
||||
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
|
||||
file: lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
|
||||
@@ -20,14 +20,11 @@ class ServerConnection {
|
||||
channel = IOWebSocketChannel.connect(this.url);
|
||||
int tick = 1;
|
||||
channel.stream.listen((message) {
|
||||
Map data = json.decode(message);
|
||||
var data = json.decode(message);
|
||||
print(data);
|
||||
Map commands = data['commands'] ?? {};
|
||||
_loadDataController.add(
|
||||
ServerLoadDataPoint(
|
||||
tick,
|
||||
APICallMetrics.from_map(commands['search'] ?? {}),
|
||||
APICallMetrics.from_map(commands['resolve'] ?? {})
|
||||
ServerLoadDataPoint.from_map(
|
||||
tick, data
|
||||
)
|
||||
);
|
||||
tick++;
|
||||
@@ -52,48 +49,106 @@ class ServerConnection {
|
||||
}
|
||||
|
||||
|
||||
class TimeStats {
|
||||
final int avg, min, max;
|
||||
// percentiles
|
||||
final int five;
|
||||
final int twenty_five;
|
||||
final int fifty;
|
||||
final int seventy_five;
|
||||
final int ninety_five;
|
||||
TimeStats(
|
||||
this.avg, this.min, this.five, this.twenty_five,
|
||||
this.fifty, this.seventy_five, this.ninety_five,
|
||||
this.max
|
||||
);
|
||||
TimeStats.from_list(List l): this(
|
||||
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
|
||||
);
|
||||
TimeStats.from_zeros(): this(
|
||||
0, 0, 0, 0, 0, 0, 0, 0
|
||||
);
|
||||
factory TimeStats.from_list_or_zeros(List l) =>
|
||||
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
|
||||
}
|
||||
|
||||
|
||||
class APICallMetrics {
|
||||
final int started;
|
||||
final int finished;
|
||||
final int total_time;
|
||||
final int execution_time;
|
||||
final int query_time;
|
||||
final int query_count;
|
||||
final int cache_hit;
|
||||
final int avg_wait_time;
|
||||
final int avg_total_time;
|
||||
final int avg_execution_time;
|
||||
final int avg_query_time_per_search;
|
||||
final int avg_query_time_per_query;
|
||||
// total requests received
|
||||
final int receive_count;
|
||||
// sum of these is total responses made
|
||||
final int cache_response_count;
|
||||
final int query_response_count;
|
||||
final int intrp_response_count;
|
||||
final int error_response_count;
|
||||
// stacked values for chart
|
||||
final int cache_response_stack;
|
||||
final int query_response_stack;
|
||||
final int intrp_response_stack;
|
||||
final int error_response_stack;
|
||||
// millisecond timings for non-cache responses
|
||||
final TimeStats response;
|
||||
final TimeStats interrupt;
|
||||
final TimeStats error;
|
||||
// response, interrupt and error each also report the python, wait and sql stats:
|
||||
final TimeStats python;
|
||||
final TimeStats wait;
|
||||
final TimeStats sql;
|
||||
// extended timings for individual sql executions
|
||||
final TimeStats individual_sql;
|
||||
final int individual_sql_count;
|
||||
// actual queries
|
||||
final List<String> errored_queries;
|
||||
final List<String> interrupted_queries;
|
||||
APICallMetrics(
|
||||
this.started, this.finished, this.total_time, this.execution_time,
|
||||
this.query_time, this.query_count, this.cache_hit):
|
||||
avg_wait_time=finished > 0 ? ((total_time - (execution_time + query_time))/finished).round() : 0,
|
||||
avg_total_time=finished > 0 ? (total_time/finished).round() : 0,
|
||||
avg_execution_time=finished > 0 ? (execution_time/finished).round() : 0,
|
||||
avg_query_time_per_search=finished > 0 ? (query_time/finished).round() : 0,
|
||||
avg_query_time_per_query=query_count > 0 ? (query_time/query_count).round() : 0;
|
||||
this.receive_count,
|
||||
this.cache_response_count, this.query_response_count,
|
||||
this.intrp_response_count, this.error_response_count,
|
||||
this.response, this.interrupt, this.error,
|
||||
this.python, this.wait, this.sql,
|
||||
this.individual_sql, this.individual_sql_count,
|
||||
this.errored_queries, this.interrupted_queries
|
||||
):
|
||||
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
|
||||
query_response_stack=query_response_count+intrp_response_count+error_response_count,
|
||||
intrp_response_stack=intrp_response_count+error_response_count,
|
||||
error_response_stack=error_response_count;
|
||||
APICallMetrics.from_map(Map data): this(
|
||||
data['started'] ?? 0,
|
||||
data['finished'] ?? 0,
|
||||
data['total_time'] ?? 0,
|
||||
data['execution_time'] ?? 0,
|
||||
data['query_time'] ?? 0,
|
||||
data['query_count'] ?? 0,
|
||||
data['cache_hit'] ?? 0,
|
||||
data["receive_count"] ?? 0,
|
||||
data["cache_response_count"] ?? 0,
|
||||
data["query_response_count"] ?? 0,
|
||||
data["intrp_response_count"] ?? 0,
|
||||
data["error_response_count"] ?? 0,
|
||||
TimeStats.from_list_or_zeros(data["response"]),
|
||||
TimeStats.from_list_or_zeros(data["interrupt"]),
|
||||
TimeStats.from_list_or_zeros(data["error"]),
|
||||
TimeStats.from_list_or_zeros(data["python"]),
|
||||
TimeStats.from_list_or_zeros(data["wait"]),
|
||||
TimeStats.from_list_or_zeros(data["sql"]),
|
||||
TimeStats.from_list_or_zeros(data["individual_sql"]),
|
||||
data["individual_sql_count"] ?? 0,
|
||||
List<String>.from(data["errored_queries"] ?? const []),
|
||||
List<String>.from(data["interrupted_queries"] ?? const []),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadDataPoint {
|
||||
final int tick;
|
||||
final int sessions;
|
||||
final APICallMetrics search;
|
||||
final APICallMetrics resolve;
|
||||
ServerLoadDataPoint(this.tick, this.search, this.resolve);
|
||||
ServerLoadDataPoint.empty():
|
||||
tick = 0,
|
||||
search=APICallMetrics.from_map({}),
|
||||
resolve=APICallMetrics.from_map({});
|
||||
const ServerLoadDataPoint(
|
||||
this.tick, this.sessions, this.search, this.resolve
|
||||
);
|
||||
ServerLoadDataPoint.from_map(int tick, Map data): this(
|
||||
tick, (data['status'] ?? const {})['sessions'] ?? 0,
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
|
||||
);
|
||||
ServerLoadDataPoint.empty(): this(
|
||||
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,12 @@ class ServerCharts extends StatelessWidget {
|
||||
Widget build(BuildContext context) {
|
||||
var server = Provider.of<Server>(context, listen: false);
|
||||
return ListView(children: <Widget>[
|
||||
SizedBox(height: 300.0, child: ServerLoadChart(server)),
|
||||
SizedBox(height: 300.0, child: APILoadChart(
|
||||
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
|
||||
)),
|
||||
//SizedBox(height: 300.0, child: APILoadChart(
|
||||
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
|
||||
//)),
|
||||
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
|
||||
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
|
||||
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
|
||||
@@ -22,16 +27,20 @@ class ServerCharts extends StatelessWidget {
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadChart extends StatefulWidget {
|
||||
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
|
||||
|
||||
class APILoadChart extends StatefulWidget {
|
||||
final Server server;
|
||||
ServerLoadChart(this.server);
|
||||
final String name;
|
||||
final APIGetter getter;
|
||||
APILoadChart(this.server, this.name, this.getter);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => ServerLoadChartState();
|
||||
State<StatefulWidget> createState() => APILoadChartState();
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
class APILoadChartState extends State<APILoadChart> {
|
||||
|
||||
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
|
||||
|
||||
@@ -39,54 +48,73 @@ class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Cache',
|
||||
id: 'Received',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
strokeWidthPxFn: (_, __) => 4.0,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
|
||||
data: widget.server.serverLoadData,
|
||||
),*/
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Cache',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Query',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.blue.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Interrupts',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Errors',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Interrupted',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.pink.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Errored',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} From Cache',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.cache_hit,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Finish',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.deepOrange.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.finished,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Search Start',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.deepOrange.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.started,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Cache',
|
||||
colorFn: (_, __) => charts.MaterialPalette.cyan.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.cache_hit,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Finish',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.finished,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Resolve Start',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.resolve.started,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 1.0,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
*/
|
||||
];
|
||||
}
|
||||
|
||||
@@ -94,7 +122,9 @@ class ServerLoadChartState extends State<ServerLoadChart> {
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ServerLoadDataPoint>(
|
||||
stream: widget.server.serverLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData)
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData,
|
||||
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -117,27 +147,41 @@ class ServerPerformanceChartState extends State<ServerPerformanceChart> {
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Waiting 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Waiting',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_wait_time,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Executing',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.lighter,
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_execution_time,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'SQLite 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. SQLite',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.avg_query_time_per_search,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
)
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -260,11 +304,13 @@ class BetterLineChart extends charts.LineChart {
|
||||
final int itemCount;
|
||||
final Object lastItem;
|
||||
|
||||
BetterLineChart(List<charts.Series<dynamic, int>> seriesList):
|
||||
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
|
||||
itemCount = seriesList[0].data.length,
|
||||
lastItem = seriesList[0].data.last,
|
||||
super(
|
||||
seriesList,
|
||||
animate: false,
|
||||
defaultRenderer: renderer,
|
||||
behaviors: [charts.SeriesLegend()],
|
||||
domainAxis: charts.NumericAxisSpec(
|
||||
viewport: new charts.NumericExtents(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.38.1"
|
||||
__version__ = "0.38.3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -551,6 +551,9 @@ class Config(CLIConfig):
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
|
||||
@@ -6,7 +6,7 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
|
||||
|
||||
|
||||
def normalize_tag(tag: str):
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
|
||||
|
||||
|
||||
def clean_tags(tags: List[str]):
|
||||
|
||||
@@ -214,7 +214,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim)
|
||||
height = self.ledger.headers.height
|
||||
await self.generate(1)
|
||||
await self.on_header(height + 1)
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -3,10 +3,11 @@ import logging
|
||||
from binascii import unhexlify
|
||||
from typing import Tuple, List, Dict
|
||||
|
||||
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from torba.client.baseaccount import SingleKey
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.schema.url import URL
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.resolve import Resolver
|
||||
from lbry.wallet.account import Account
|
||||
@@ -72,7 +73,50 @@ class MainNetLedger(BaseLedger):
|
||||
result[url] = txo
|
||||
else:
|
||||
result[url] = {'error': f'{url} did not resolve to a claim'}
|
||||
return result
|
||||
return await self.validate_resolutions(result)
|
||||
|
||||
async def validate_resolutions(self, resolutions: dict):
|
||||
provables = {}
|
||||
for url, txo in resolutions.items():
|
||||
if isinstance(txo, dict) and 'error' in txo:
|
||||
continue
|
||||
parsed_url = URL.parse(url)
|
||||
# cases we check: names without sequence, claim id or any modifier
|
||||
# except: @channel/name as the signature is enough, instead we check '@channel'
|
||||
if parsed_url.has_channel:
|
||||
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
|
||||
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
|
||||
if not provables:
|
||||
return resolutions
|
||||
root_hash = self.headers.claim_trie_root.decode()
|
||||
names = list(provables.keys())
|
||||
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
|
||||
for proof, name in zip(proofs, names):
|
||||
all_failed = True
|
||||
for url, txid in provables[name]:
|
||||
if not proof.get('txhash'):
|
||||
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
|
||||
continue
|
||||
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
|
||||
if txid != expected_txid:
|
||||
resolutions[url] = {
|
||||
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
|
||||
}
|
||||
else:
|
||||
all_failed = False
|
||||
if not all_failed:
|
||||
try:
|
||||
verify_proof(proof, root_hash, name)
|
||||
continue
|
||||
except InvalidProofError as error:
|
||||
for url, _ in provables[name]:
|
||||
resolution = resolutions[url]
|
||||
if isinstance(resolution, dict) and 'error' in resolution:
|
||||
continue
|
||||
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
|
||||
return resolutions
|
||||
|
||||
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
|
||||
return await self._inflate_outputs(self.network.claim_search(**kwargs))
|
||||
|
||||
@@ -9,5 +9,8 @@ class Network(BaseNetwork):
|
||||
def resolve(self, urls):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
||||
|
||||
def get_name_proofs(self, block_hash, *names):
|
||||
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
|
||||
|
||||
def claim_search(self, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
||||
@@ -79,6 +79,10 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
raise
|
||||
finally:
|
||||
self.sql.commit()
|
||||
if self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
|
||||
@@ -11,3 +11,210 @@ STREAM_TYPES = {
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = [
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
]
|
||||
|
||||
COMMON_TAGS = {
|
||||
"gaming": "gaming",
|
||||
"people & blogs": "people_and_blogs",
|
||||
"pop culture": "pop_culture",
|
||||
"entertainment": "entertainment",
|
||||
"technology": "technology",
|
||||
"music": "music",
|
||||
"funny": "funny",
|
||||
"education": "education",
|
||||
"learning": "learning",
|
||||
"news": "news",
|
||||
"gameplay": "gameplay",
|
||||
"science & technology": "science_and_technology",
|
||||
"playstation 4": "playstation_4",
|
||||
"beliefs": "beliefs",
|
||||
"nature": "nature",
|
||||
"news & politics": "news_and_politics",
|
||||
"comedy": "comedy",
|
||||
"games": "games",
|
||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
||||
"film & animation": "film_and_animation",
|
||||
"game": "game",
|
||||
"howto & style": "howto_and_style",
|
||||
"weapons": "weapons",
|
||||
"blockchain": "blockchain",
|
||||
"video game": "video_game",
|
||||
"sports": "sports",
|
||||
"walkthrough": "walkthrough",
|
||||
"ps4live": "ps4live",
|
||||
"art": "art",
|
||||
"pc": "pc",
|
||||
"economics": "economics",
|
||||
"automotive": "automotive",
|
||||
"minecraft": "minecraft",
|
||||
"playthrough": "playthrough",
|
||||
"ps4share": "ps4share",
|
||||
"tutorial": "tutorial",
|
||||
"play": "play",
|
||||
"twitch": "twitch",
|
||||
"how to": "how_to",
|
||||
"ps4": "ps4",
|
||||
"bitcoin": "bitcoin",
|
||||
"fortnite": "fortnite",
|
||||
"commentary": "commentary",
|
||||
"lets play": "lets_play",
|
||||
"fun": "fun",
|
||||
"politics": "politics",
|
||||
"xbox": "xbox",
|
||||
"autos & vehicles": "autos_and_vehicles",
|
||||
"travel & events": "travel_and_events",
|
||||
"food": "food",
|
||||
"science": "science",
|
||||
"mature": "mature",
|
||||
"xbox one": "xbox_one",
|
||||
"liberal": "liberal",
|
||||
"democrat": "democrat",
|
||||
"progressive": "progressive",
|
||||
"survival": "survival",
|
||||
"nonprofits & activism": "nonprofits_and_activism",
|
||||
"cryptocurrency": "cryptocurrency",
|
||||
"playstation": "playstation",
|
||||
"nintendo": "nintendo",
|
||||
"government": "government",
|
||||
"steam": "steam",
|
||||
"podcast": "podcast",
|
||||
"horror": "horror",
|
||||
"conservative": "conservative",
|
||||
"reaction": "reaction",
|
||||
"trailer": "trailer",
|
||||
"love": "love",
|
||||
"cnn": "cnn",
|
||||
"republican": "republican",
|
||||
"gamer": "gamer",
|
||||
"political": "political",
|
||||
"hangoutsonair": "hangoutsonair",
|
||||
"hoa": "hoa",
|
||||
"msnbc": "msnbc",
|
||||
"cbs": "cbs",
|
||||
"donald trump": "donald_trump",
|
||||
"fiction": "fiction",
|
||||
"fox news": "fox_news",
|
||||
"anime": "anime",
|
||||
"crypto": "crypto",
|
||||
"ethereum": "ethereum",
|
||||
"call of duty": "call_of_duty",
|
||||
"multiplayer": "multiplayer",
|
||||
"android": "android",
|
||||
"epic": "epic",
|
||||
"rpg": "rpg",
|
||||
"adventure": "adventure",
|
||||
"secular talk": "secular_talk",
|
||||
"btc": "btc",
|
||||
"atheist": "atheist",
|
||||
"atheism": "atheism",
|
||||
"ps3": "ps3",
|
||||
"video games": "video_games",
|
||||
"cod": "cod",
|
||||
"agnostic": "agnostic",
|
||||
"movie": "movie",
|
||||
"online": "online",
|
||||
"fps": "fps",
|
||||
"mod": "mod",
|
||||
"reviews": "reviews",
|
||||
"sharefactory": "sharefactory",
|
||||
"world": "world",
|
||||
"space": "space",
|
||||
"hilarious": "hilarious",
|
||||
"stream": "stream",
|
||||
"lol": "lol",
|
||||
"sony": "sony",
|
||||
"god": "god",
|
||||
"lets": "lets",
|
||||
"dance": "dance",
|
||||
"pvp": "pvp",
|
||||
"tech": "tech",
|
||||
"zombies": "zombies",
|
||||
"pokemon": "pokemon",
|
||||
"fail": "fail",
|
||||
"xbox 360": "xbox_360",
|
||||
"film": "film",
|
||||
"unboxing": "unboxing",
|
||||
"animation": "animation",
|
||||
"travel": "travel",
|
||||
"money": "money",
|
||||
"wwe": "wwe",
|
||||
"how": "how",
|
||||
"mods": "mods",
|
||||
"pubg": "pubg",
|
||||
"indie": "indie",
|
||||
"strategy": "strategy",
|
||||
"history": "history",
|
||||
"rap": "rap",
|
||||
"ios": "ios",
|
||||
"sony computer entertainment": "sony_computer_entertainment",
|
||||
"mobile": "mobile",
|
||||
"trump": "trump",
|
||||
"flat earth": "flat_earth",
|
||||
"hack": "hack",
|
||||
"trap": "trap",
|
||||
"fox": "fox",
|
||||
"vlogging": "vlogging",
|
||||
"news radio": "news_radio",
|
||||
"humor": "humor",
|
||||
"facebook": "facebook",
|
||||
"edm": "edm",
|
||||
"fitness": "fitness",
|
||||
"vaping": "vaping",
|
||||
"hip hop": "hip_hop",
|
||||
"secular": "secular",
|
||||
"jesus": "jesus",
|
||||
"vape": "vape",
|
||||
"song": "song",
|
||||
"remix": "remix",
|
||||
"guitar": "guitar",
|
||||
"daily": "daily",
|
||||
"mining": "mining",
|
||||
"diy": "diy",
|
||||
"videogame": "videogame",
|
||||
"pets & animals": "pets_and_animals",
|
||||
"funny moments": "funny_moments",
|
||||
"religion": "religion",
|
||||
"death": "death",
|
||||
"media": "media",
|
||||
"nbc": "nbc",
|
||||
"war": "war",
|
||||
"freedom": "freedom",
|
||||
"viral": "viral",
|
||||
"meme": "meme",
|
||||
"family": "family",
|
||||
"gold": "gold",
|
||||
"photography": "photography",
|
||||
"chill": "chill",
|
||||
"zombie": "zombie",
|
||||
"computer": "computer",
|
||||
"sniper": "sniper",
|
||||
"bible": "bible",
|
||||
"linux": "linux",
|
||||
"overwatch": "overwatch",
|
||||
"pro": "pro",
|
||||
"dragon": "dragon",
|
||||
"litecoin": "litecoin",
|
||||
"gta": "gta",
|
||||
"iphone": "iphone",
|
||||
"house": "house",
|
||||
"bass": "bass",
|
||||
"bitcoin news": "bitcoin_news",
|
||||
"wii": "wii",
|
||||
"crash": "crash",
|
||||
"league of legends": "league_of_legends",
|
||||
"grand theft auto v": "grand_theft_auto_v",
|
||||
"mario": "mario",
|
||||
"mmorpg": "mmorpg",
|
||||
"satire": "satire",
|
||||
"fire": "fire",
|
||||
"racing": "racing",
|
||||
"apple": "apple",
|
||||
"health": "health",
|
||||
"instrumental": "instrumental",
|
||||
"destiny": "destiny",
|
||||
"truth": "truth",
|
||||
"race": "race"
|
||||
}
|
||||
@@ -9,14 +9,20 @@ from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from dataclasses import dataclass
|
||||
|
||||
from torba.client.basedatabase import query
|
||||
from torba.client.basedatabase import query, interpolate
|
||||
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
class SQLiteOperationalError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query errored')
|
||||
self.metrics = metrics
|
||||
|
||||
|
||||
class SQLiteInterruptedError(sqlite3.OperationalError):
|
||||
@@ -50,7 +56,7 @@ SEARCH_PARAMS = {
|
||||
|
||||
|
||||
ORDER_FIELDS = {
|
||||
'name',
|
||||
'name', 'claim_hash'
|
||||
} | INTEGER_PARAMS
|
||||
|
||||
|
||||
@@ -91,7 +97,6 @@ ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
|
||||
|
||||
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
|
||||
|
||||
db = sqlite3.connect(_path, isolation_level=None, uri=True)
|
||||
db.row_factory = sqlite3.Row
|
||||
ctx.set(
|
||||
@@ -114,21 +119,16 @@ def measure(func):
|
||||
state = ctx.get()
|
||||
if not state.is_tracking_metrics:
|
||||
return func(*args, **kwargs)
|
||||
metric = state.metrics.setdefault(func.__name__, {
|
||||
'calls': 0, 'total': 0, 'isolated': 0, 'errors': 0
|
||||
})
|
||||
metric = {}
|
||||
state.metrics.setdefault(func.__name__, []).append(metric)
|
||||
state.stack.append([])
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except:
|
||||
metric['errors'] += 1
|
||||
raise
|
||||
finally:
|
||||
elapsed = int((time.perf_counter()-start)*1000)
|
||||
metric['calls'] += 1
|
||||
metric['total'] += elapsed
|
||||
metric['isolated'] += (elapsed-sum(state.stack.pop()))
|
||||
metric['total'] = elapsed
|
||||
metric['isolated'] = (elapsed-sum(state.stack.pop()))
|
||||
if state.stack:
|
||||
state.stack[-1].append(elapsed)
|
||||
return wrapper
|
||||
@@ -167,16 +167,17 @@ def execute_query(sql, values) -> List:
|
||||
try:
|
||||
return context.db.execute(sql, values).fetchall()
|
||||
except sqlite3.OperationalError as err:
|
||||
plain_sql = interpolate(sql, values)
|
||||
if context.is_tracking_metrics:
|
||||
context.metrics['execute_query'][-1]['sql'] = plain_sql
|
||||
if str(err) == "interrupted":
|
||||
query_str = sql
|
||||
for k in sorted(values.keys(), reverse=True):
|
||||
query_str = query_str.replace(f":{k}", str(values[k]) if not k.startswith("$") else f"'{values[k]}'")
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", query_str)
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
|
||||
raise SQLiteInterruptedError(context.metrics)
|
||||
context.log.exception('failed running query', exc_info=err)
|
||||
raise SQLiteOperationalError(context.metrics)
|
||||
|
||||
|
||||
@measure
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'order_by' in constraints:
|
||||
sql_order_by = []
|
||||
for order_by in constraints['order_by']:
|
||||
@@ -231,14 +232,6 @@ def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['claim.public_key_hash'] = sqlite3.Binary(
|
||||
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = resolve_url(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
if 'channel_hash' in constraints:
|
||||
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
|
||||
if 'channel_ids' in constraints:
|
||||
@@ -307,7 +300,18 @@ def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
""", **constraints
|
||||
)
|
||||
return sql, values
|
||||
|
||||
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = resolve_url(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
sql, values = _get_claims(cols, for_count, **constraints)
|
||||
return execute_query(sql, values)
|
||||
|
||||
|
||||
@@ -330,7 +334,7 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["height", "^name"]
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
@@ -404,7 +408,7 @@ def resolve_url(raw_url):
|
||||
if channel is not None:
|
||||
if set(query) == {'name'}:
|
||||
# temporarily emulate is_controlling for claims in channel
|
||||
query['order_by'] = ['effective_amount']
|
||||
query['order_by'] = ['effective_amount', '^height']
|
||||
else:
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['channel_hash'] = channel['claim_hash']
|
||||
@@ -420,7 +424,6 @@ def resolve_url(raw_url):
|
||||
return channel
|
||||
|
||||
|
||||
@measure
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
@@ -429,26 +432,61 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
all_items = {item for item in all_items if item not in not_items}
|
||||
any_items = {item for item in any_items if item not in not_items}
|
||||
|
||||
any_queries = {}
|
||||
|
||||
if attr == 'tag':
|
||||
common_tags = any_items & COMMON_TAGS.keys()
|
||||
if common_tags:
|
||||
any_items -= common_tags
|
||||
if len(common_tags) < 5:
|
||||
for item in common_tags:
|
||||
index_name = COMMON_TAGS[item]
|
||||
any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag = '{item}'
|
||||
)
|
||||
"""
|
||||
elif len(common_tags) >= 5:
|
||||
constraints.update({
|
||||
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_common_tag{i}' for i in range(len(common_tags))
|
||||
)
|
||||
any_queries[f'#_any_common_tags'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if any_items:
|
||||
|
||||
constraints.update({
|
||||
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if len(any_queries) == 1:
|
||||
constraints.update(any_queries)
|
||||
elif len(any_queries) > 1:
|
||||
constraints[f'ORed_{attr}_queries__any'] = any_queries
|
||||
|
||||
if all_items:
|
||||
constraints[f'$all_{attr}_count'] = len(all_items)
|
||||
constraints.update({
|
||||
|
||||
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
|
||||
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
|
||||
)
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
@@ -76,30 +76,12 @@ class SQLDB:
|
||||
trending_global integer not null default 0
|
||||
);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
|
||||
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized);
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_height_idx on claim (height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -110,7 +92,6 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
amount integer not null
|
||||
);
|
||||
create index if not exists support_txo_hash_idx on support (txo_hash);
|
||||
create index if not exists support_claim_hash_idx on support (claim_hash, height);
|
||||
"""
|
||||
|
||||
@@ -120,7 +101,6 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
height integer not null
|
||||
);
|
||||
create index if not exists tag_tag_idx on tag (tag);
|
||||
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
|
||||
"""
|
||||
|
||||
@@ -133,6 +113,38 @@ class SQLDB:
|
||||
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
|
||||
"""
|
||||
|
||||
SEARCH_INDEXES = """
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
TAG_INDEXES = '\n'.join(
|
||||
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
|
||||
for tag_value, tag_key in COMMON_TAGS.items()
|
||||
)
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_CLAIM_TABLE +
|
||||
@@ -257,7 +269,7 @@ class SQLDB:
|
||||
|
||||
if tags:
|
||||
self.db.executemany(
|
||||
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
)
|
||||
|
||||
return claims
|
||||
@@ -534,10 +546,15 @@ class SQLDB:
|
||||
""", claim_hashes)
|
||||
|
||||
def _update_effective_amount(self, height, claim_hashes=None):
|
||||
self.execute(
|
||||
f"UPDATE claim SET effective_amount = amount + support_amount "
|
||||
f"WHERE activation_height = {height}"
|
||||
)
|
||||
if claim_hashes:
|
||||
self.execute(
|
||||
f"UPDATE claim SET effective_amount = amount + support_amount "
|
||||
f"WHERE claim_hash IN ({','.join('?' for _ in claim_hashes)})",
|
||||
f"WHERE activation_height < {height} "
|
||||
f" AND claim_hash IN ({','.join('?' for _ in claim_hashes)})",
|
||||
claim_hashes
|
||||
)
|
||||
|
||||
@@ -565,13 +582,12 @@ class SQLDB:
|
||||
claimtrie.claim_hash AS current_winner,
|
||||
MAX(winner.effective_amount)
|
||||
FROM (
|
||||
SELECT normalized, claim_hash, effective_amount, activation_height FROM claim
|
||||
SELECT normalized, claim_hash, effective_amount FROM claim
|
||||
WHERE normalized IN (
|
||||
SELECT normalized FROM claim WHERE activation_height={height} {claim_hashes_sql}
|
||||
) {deleted_names_sql}
|
||||
ORDER BY effective_amount DESC, height ASC, tx_position ASC
|
||||
) AS winner LEFT JOIN claimtrie USING (normalized)
|
||||
WHERE winner.activation_height<={height}
|
||||
GROUP BY winner.normalized
|
||||
HAVING current_winner IS NULL OR current_winner <> winner.claim_hash
|
||||
""", changed_claim_hashes+deleted_names)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import time
|
||||
import math
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
def calculate_elapsed(start) -> int:
|
||||
return int((time.perf_counter() - start) * 1000)
|
||||
|
||||
|
||||
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
|
||||
if not data:
|
||||
return 0, 0, 0, 0, 0, 0, 0, 0
|
||||
data.sort()
|
||||
size = len(data)
|
||||
return (
|
||||
int(sum(data) / size),
|
||||
data[0],
|
||||
data[math.ceil(size * .05) - 1],
|
||||
data[math.ceil(size * .25) - 1],
|
||||
data[math.ceil(size * .50) - 1],
|
||||
data[math.ceil(size * .75) - 1],
|
||||
data[math.ceil(size * .95) - 1],
|
||||
data[-1]
|
||||
)
|
||||
|
||||
|
||||
def remove_select_list(sql) -> str:
|
||||
return sql[sql.index('FROM'):]
|
||||
|
||||
|
||||
class APICallMetrics:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
# total requests received
|
||||
self.receive_count = 0
|
||||
self.cache_response_count = 0
|
||||
|
||||
# millisecond timings for query based responses
|
||||
self.query_response_times = []
|
||||
self.query_intrp_times = []
|
||||
self.query_error_times = []
|
||||
|
||||
self.query_python_times = []
|
||||
self.query_wait_times = []
|
||||
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
|
||||
|
||||
self.individual_sql_times = [] # every SQL query run on server
|
||||
|
||||
# actual queries
|
||||
self.errored_queries = set()
|
||||
self.interrupted_queries = set()
|
||||
|
||||
def to_json_and_reset(self):
|
||||
return {
|
||||
# total requests received
|
||||
"receive_count": self.receive_count,
|
||||
# sum of these is total responses made
|
||||
"cache_response_count": self.cache_response_count,
|
||||
"query_response_count": len(self.query_response_times),
|
||||
"intrp_response_count": len(self.query_intrp_times),
|
||||
"error_response_count": len(self.query_error_times),
|
||||
# millisecond timings for non-cache responses
|
||||
"response": calculate_avg_percentiles(self.query_response_times),
|
||||
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
|
||||
"error": calculate_avg_percentiles(self.query_error_times),
|
||||
# response, interrupt and error each also report the python, wait and sql stats:
|
||||
"python": calculate_avg_percentiles(self.query_python_times),
|
||||
"wait": calculate_avg_percentiles(self.query_wait_times),
|
||||
"sql": calculate_avg_percentiles(self.query_sql_times),
|
||||
# extended timings for individual sql executions
|
||||
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
|
||||
"individual_sql_count": len(self.individual_sql_times),
|
||||
# actual queries
|
||||
"errored_queries": list(self.errored_queries),
|
||||
"interrupted_queries": list(self.interrupted_queries),
|
||||
}
|
||||
|
||||
def start(self):
|
||||
self.receive_count += 1
|
||||
|
||||
def cache_response(self):
|
||||
self.cache_response_count += 1
|
||||
|
||||
def _add_query_timings(self, request_total_time, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
sub_process_total = metrics[self.name][0]['total']
|
||||
individual_query_times = [f['total'] for f in metrics['execute_query']]
|
||||
aggregated_query_time = sum(individual_query_times)
|
||||
self.individual_sql_times.extend(individual_query_times)
|
||||
self.query_sql_times.append(aggregated_query_time)
|
||||
self.query_python_times.append(sub_process_total - aggregated_query_time)
|
||||
self.query_wait_times.append(request_total_time - sub_process_total)
|
||||
|
||||
@staticmethod
|
||||
def _add_queries(query_set, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
for execute_query in metrics['execute_query']:
|
||||
if 'sql' in execute_query:
|
||||
query_set.add(remove_select_list(execute_query['sql']))
|
||||
|
||||
def query_response(self, start, metrics):
|
||||
self.query_response_times.append(calculate_elapsed(start))
|
||||
self._add_query_timings(self.query_response_times[-1], metrics)
|
||||
|
||||
def query_interrupt(self, start, metrics):
|
||||
self.query_intrp_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.interrupted_queries, metrics)
|
||||
self._add_query_timings(self.query_intrp_times[-1], metrics)
|
||||
|
||||
def query_error(self, start, metrics):
|
||||
self.query_error_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.errored_queries, metrics)
|
||||
self._add_query_timings(self.query_error_times[-1], metrics)
|
||||
|
||||
|
||||
class ServerLoadData:
|
||||
|
||||
def __init__(self):
|
||||
self._apis = {}
|
||||
|
||||
def for_api(self, name) -> APICallMetrics:
|
||||
if name not in self._apis:
|
||||
self._apis[name] = APICallMetrics(name)
|
||||
return self._apis[name]
|
||||
|
||||
def to_json_and_reset(self, status):
|
||||
try:
|
||||
return {
|
||||
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
|
||||
'status': status
|
||||
}
|
||||
finally:
|
||||
self._apis = {}
|
||||
@@ -4,13 +4,9 @@ import time
|
||||
import base64
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from weakref import WeakSet
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
|
||||
from aiohttp.http_websocket import WSMsgType, WSCloseCode
|
||||
|
||||
from torba.rpc.jsonrpc import RPCError, JSONRPC
|
||||
from torba.server.session import ElectrumX, SessionManager
|
||||
from torba.server import util
|
||||
@@ -18,56 +14,8 @@ from torba.server import util
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYDB
|
||||
from lbry.wallet.server.db import reader
|
||||
|
||||
|
||||
class AdminWebSocket:
|
||||
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
self.app = Application()
|
||||
self.app['websockets'] = WeakSet()
|
||||
self.app.router.add_get('/', self.on_connect)
|
||||
self.app.on_shutdown.append(self.on_shutdown)
|
||||
self.runner = AppRunner(self.app)
|
||||
|
||||
async def on_status(self, _):
|
||||
if not self.app['websockets']:
|
||||
return
|
||||
self.send_message({
|
||||
'type': 'status',
|
||||
'height': self.manager.daemon.cached_height(),
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
for web_socket in self.app['websockets']:
|
||||
asyncio.create_task(web_socket.send_json(msg))
|
||||
|
||||
async def start(self):
|
||||
await self.runner.setup()
|
||||
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def on_connect(self, request):
|
||||
web_socket = WebSocketResponse()
|
||||
await web_socket.prepare(request)
|
||||
self.app['websockets'].add(web_socket)
|
||||
try:
|
||||
async for msg in web_socket:
|
||||
if msg.type == WSMsgType.TEXT:
|
||||
await self.on_status(None)
|
||||
elif msg.type == WSMsgType.ERROR:
|
||||
print('web socket connection closed with exception %s' %
|
||||
web_socket.exception())
|
||||
finally:
|
||||
self.app['websockets'].discard(web_socket)
|
||||
return web_socket
|
||||
|
||||
@staticmethod
|
||||
async def on_shutdown(app):
|
||||
for web_socket in set(app['websockets']):
|
||||
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
|
||||
|
||||
class ResultCacheItem:
|
||||
@@ -95,9 +43,8 @@ class LBRYSessionManager(SessionManager):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.query_executor = None
|
||||
self.websocket = None
|
||||
self.metrics_processor = None
|
||||
self.command_metrics = {}
|
||||
self.reader_metrics = {}
|
||||
self.metrics = ServerLoadData()
|
||||
self.metrics_loop = None
|
||||
self.running = False
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
@@ -105,69 +52,11 @@ class LBRYSessionManager(SessionManager):
|
||||
self.search_cache['search'] = lrucache(10000)
|
||||
self.search_cache['resolve'] = lrucache(10000)
|
||||
|
||||
def get_command_tracking_info(self, command):
|
||||
if command not in self.command_metrics:
|
||||
self.command_metrics[command] = {
|
||||
'cache_hit': 0,
|
||||
'started': 0,
|
||||
'finished': 0,
|
||||
'total_time': 0,
|
||||
'execution_time': 0,
|
||||
'query_time': 0,
|
||||
'query_count': 0,
|
||||
'interrupted': 0,
|
||||
'interrupted_query_values': [],
|
||||
}
|
||||
return self.command_metrics[command]
|
||||
|
||||
def cache_hit(self, command_name):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['cache_hit'] += 1
|
||||
|
||||
def start_command_tracking(self, command_name):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['started'] += 1
|
||||
|
||||
def finish_command_tracking(self, command_name, elapsed, metrics):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['finished'] += 1
|
||||
command['total_time'] += elapsed
|
||||
if 'execute_query' in metrics:
|
||||
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
|
||||
command['query_time'] += metrics['execute_query']['total']
|
||||
command['query_count'] += metrics['execute_query']['calls']
|
||||
for func_name, func_metrics in metrics.items():
|
||||
reader = self.reader_metrics.setdefault(func_name, {})
|
||||
for key in func_metrics:
|
||||
if key not in reader:
|
||||
reader[key] = func_metrics[key]
|
||||
else:
|
||||
reader[key] += func_metrics[key]
|
||||
|
||||
def interrupted_command_error(self, command_name, elapsed, metrics, kwargs):
|
||||
if self.env.track_metrics:
|
||||
command = self.get_command_tracking_info(command_name)
|
||||
command['finished'] += 1
|
||||
command['interrupted'] += 1
|
||||
command['total_time'] += elapsed
|
||||
command['execution_time'] += (metrics[command_name]['total'] - metrics['execute_query']['total'])
|
||||
command['query_time'] += metrics['execute_query']['total']
|
||||
command['query_count'] += metrics['execute_query']['calls']
|
||||
if len(command['interrupted_query_values']) < 100:
|
||||
command['interrupted_query_values'].append(kwargs)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
commands, self.command_metrics = self.command_metrics, {}
|
||||
reader, self.reader_metrics = self.reader_metrics, {}
|
||||
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
|
||||
if self.websocket is not None:
|
||||
self.websocket.send_message({
|
||||
'commands': commands,
|
||||
'reader': reader
|
||||
})
|
||||
self.websocket.send_message(data)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def start_other(self):
|
||||
@@ -186,12 +75,12 @@ class LBRYSessionManager(SessionManager):
|
||||
if self.websocket is not None:
|
||||
await self.websocket.start()
|
||||
if self.env.track_metrics:
|
||||
self.metrics_processor = asyncio.create_task(self.process_metrics())
|
||||
self.metrics_loop = asyncio.create_task(self.process_metrics())
|
||||
|
||||
async def stop_other(self):
|
||||
self.running = False
|
||||
if self.env.track_metrics:
|
||||
self.metrics_processor.cancel()
|
||||
self.metrics_loop.cancel()
|
||||
if self.websocket is not None:
|
||||
await self.websocket.stop()
|
||||
self.query_executor.shutdown()
|
||||
@@ -215,50 +104,56 @@ class LBRYElectrumX(ElectrumX):
|
||||
'blockchain.transaction.get_height': self.transaction_get_height,
|
||||
'blockchain.claimtrie.search': self.claimtrie_search,
|
||||
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
|
||||
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
|
||||
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
|
||||
'blockchain.block.get_server_height': self.get_server_height,
|
||||
}
|
||||
self.request_handlers.update(handlers)
|
||||
|
||||
async def run_in_executor(self, name, func, kwargs):
|
||||
start = None
|
||||
if self.env.track_metrics:
|
||||
self.session_mgr.start_command_tracking(name)
|
||||
start = time.perf_counter()
|
||||
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
self.session_mgr.interrupted_command_error(
|
||||
name, int((time.perf_counter() - start) * 1000), error.metrics, kwargs
|
||||
)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics.query_error(start, error.metrics)
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except:
|
||||
metrics.query_error(start, {})
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
|
||||
if self.env.track_metrics:
|
||||
elapsed = int((time.perf_counter() - start) * 1000)
|
||||
(result, metrics) = result
|
||||
self.session_mgr.finish_command_tracking(name, elapsed, metrics)
|
||||
(result, metrics_data) = result
|
||||
metrics.query_response(start, metrics_data)
|
||||
|
||||
return base64.b64encode(result).decode()
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
if self.env.track_metrics:
|
||||
metrics = self.session_mgr.metrics.for_api(query_name)
|
||||
else:
|
||||
metrics = APICallMetrics(query_name)
|
||||
metrics.start()
|
||||
cache = self.session_mgr.search_cache[query_name]
|
||||
cache_key = str(kwargs)
|
||||
cache_item = cache.get(cache_key)
|
||||
if cache_item is None:
|
||||
cache_item = cache[cache_key] = ResultCacheItem()
|
||||
elif cache_item.result is not None:
|
||||
self.session_mgr.cache_hit(query_name)
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
async with cache_item.lock:
|
||||
result = cache_item.result
|
||||
if result is None:
|
||||
result = cache_item.result = await self.run_in_executor(
|
||||
query_name, function, kwargs
|
||||
if cache_item.result is None:
|
||||
cache_item.result = await self.run_in_executor(
|
||||
metrics, function, kwargs
|
||||
)
|
||||
else:
|
||||
self.session_mgr.cache_hit(query_name)
|
||||
return result
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
@@ -271,6 +166,9 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
|
||||
def claimtrie_getnameproofs(self, block_hash, *names):
|
||||
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
|
||||
|
||||
async def transaction_get_height(self, tx_hash):
|
||||
self.assert_tx_hash(tx_hash)
|
||||
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import asyncio
|
||||
from weakref import WeakSet
|
||||
|
||||
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
|
||||
from aiohttp.http_websocket import WSMsgType, WSCloseCode
|
||||
|
||||
|
||||
class AdminWebSocket:
|
||||
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
self.app = Application()
|
||||
self.app['websockets'] = WeakSet()
|
||||
self.app.router.add_get('/', self.on_connect)
|
||||
self.app.on_shutdown.append(self.on_shutdown)
|
||||
self.runner = AppRunner(self.app)
|
||||
|
||||
async def on_status(self, _):
|
||||
if not self.app['websockets']:
|
||||
return
|
||||
self.send_message({
|
||||
'type': 'status',
|
||||
'height': self.manager.daemon.cached_height(),
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
for web_socket in self.app['websockets']:
|
||||
asyncio.create_task(web_socket.send_json(msg))
|
||||
|
||||
async def start(self):
|
||||
await self.runner.setup()
|
||||
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def on_connect(self, request):
|
||||
web_socket = WebSocketResponse()
|
||||
await web_socket.prepare(request)
|
||||
self.app['websockets'].add(web_socket)
|
||||
try:
|
||||
async for msg in web_socket:
|
||||
if msg.type == WSMsgType.TEXT:
|
||||
await self.on_status(None)
|
||||
elif msg.type == WSMsgType.ERROR:
|
||||
print('web socket connection closed with exception %s' %
|
||||
web_socket.exception())
|
||||
finally:
|
||||
self.app['websockets'].discard(web_socket)
|
||||
return web_socket
|
||||
|
||||
@staticmethod
|
||||
async def on_shutdown(app):
|
||||
for web_socket in set(app['websockets']):
|
||||
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
@@ -1,10 +1,10 @@
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db.reader import search_to_bytes, initializer
|
||||
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
|
||||
from lbry.wallet.ledger import MainNetLedger
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -24,6 +24,25 @@ DEFAULT_ANY_TAGS = [
|
||||
'nature'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE = [
|
||||
'gaming',
|
||||
'ufos'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE2 = [
|
||||
'city fix',
|
||||
'gaming'
|
||||
]
|
||||
|
||||
RARE_ANY_TAGS = [
|
||||
'city fix',
|
||||
'ufos',
|
||||
]
|
||||
|
||||
CITY_FIX = [
|
||||
'city fix'
|
||||
]
|
||||
|
||||
MATURE_TAGS = [
|
||||
'porn',
|
||||
'nsfw',
|
||||
@@ -47,10 +66,10 @@ ORDER_BY = [
|
||||
|
||||
def get_args(limit=20):
|
||||
args = []
|
||||
any_tags_combinations = [DEFAULT_ANY_TAGS, []]
|
||||
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
|
||||
not_tags_combinations = [MATURE_TAGS, []]
|
||||
for no_totals in [True]:
|
||||
for offset in [0, 20, 40, 60, 80, 100, 1000, 2000, 3000]:
|
||||
for offset in [0, 100]:
|
||||
for any_tags in any_tags_combinations:
|
||||
for not_tags in not_tags_combinations:
|
||||
for order_by in ORDER_BY:
|
||||
@@ -71,13 +90,12 @@ def get_args(limit=20):
|
||||
|
||||
def _search(kwargs):
|
||||
start = time.time()
|
||||
msg = f"offset={kwargs['offset']}, limit={kwargs['limit']}, no_totals={kwargs['no_totals']}, not_tags={kwargs.get('not_tags')}, any_tags={kwargs.get('any_tags')}, order_by={kwargs['order_by']}"
|
||||
try:
|
||||
search_to_bytes(kwargs)
|
||||
t = time.time() - start
|
||||
return t, f"{t} - {msg}"
|
||||
return t, kwargs
|
||||
except Exception as err:
|
||||
return -1, f"failed: error={str(type(err))}({str(err)}) - {msg}"
|
||||
return -1, f"failed: error={str(type(err))}({str(err)})"
|
||||
|
||||
|
||||
async def search(executor, kwargs):
|
||||
@@ -97,21 +115,32 @@ async def main(db_path, max_query_time):
|
||||
tasks = [search(query_executor, constraints) for constraints in get_args()]
|
||||
try:
|
||||
results = await asyncio.gather(*tasks)
|
||||
times = {msg: ts for ts, msg in results}
|
||||
log.info("\n".join(sorted(filter(lambda msg: times[msg] > max_query_time, times.keys()), key=lambda msg: times[msg])))
|
||||
for ts, constraints in results:
|
||||
if ts >= max_query_time:
|
||||
sql = interpolate(*_get_claims("""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
""", **constraints))
|
||||
print(f"Query took {int(ts * 1000)}ms\n{sql}")
|
||||
finally:
|
||||
query_executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
if len(args) >= 1:
|
||||
db_path = args[0]
|
||||
else:
|
||||
db_path = os.path.expanduser('~/claims.db')
|
||||
if len(args) >= 2:
|
||||
max_query_time = float(args[1])
|
||||
else:
|
||||
max_query_time = -3
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
|
||||
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
|
||||
args = parser.parse_args()
|
||||
db_path = args.db_path
|
||||
max_query_time = args.max_time
|
||||
asyncio.run(main(db_path, max_query_time))
|
||||
@@ -0,0 +1,107 @@
|
||||
import time
|
||||
import asyncio
|
||||
import random
|
||||
from argparse import ArgumentParser
|
||||
from torba.client.basenetwork import ClientSession
|
||||
|
||||
|
||||
class AgentSmith(ClientSession):
|
||||
|
||||
async def do_nefarious_things(self):
|
||||
await self.send_request('blockchain.claimtrie.search', {
|
||||
'no_totals': True,
|
||||
'offset': random.choice(range(0, 300, 20)),
|
||||
'limit': 20,
|
||||
'any_tags': (
|
||||
random.choice([[
|
||||
random.choice(['gaming', 'games', 'game']) +
|
||||
random.choice(['entertainment', 'playthrough', 'funny']) +
|
||||
random.choice(['xbox', 'xbox one', 'xbox news'])
|
||||
], [
|
||||
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
|
||||
random.choice(['news', 'sighting', 'sightings'])
|
||||
], [
|
||||
random.choice(['art', 'automotive']),
|
||||
random.choice(['blockchain', 'economics', 'food']),
|
||||
random.choice(['funny', 'learnings', 'nature']),
|
||||
random.choice(['news', 'science', 'technology'])
|
||||
]
|
||||
])
|
||||
),
|
||||
'not_tags': random.choice([[], [
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
]]),
|
||||
'order_by': random.choice([
|
||||
['release_time'],
|
||||
['trending_global', 'trending_mixed'],
|
||||
['effective_amount']
|
||||
])
|
||||
})
|
||||
|
||||
|
||||
class AgentSmithProgram:
|
||||
|
||||
def __init__(self, host, port):
|
||||
self.host, self.port = host, port
|
||||
self.agent_smiths = []
|
||||
|
||||
async def make_one_more_of_them(self):
|
||||
smith = AgentSmith(network=None, server=(self.host, self.port))
|
||||
await smith.create_connection()
|
||||
self.agent_smiths.append(smith)
|
||||
|
||||
async def coordinate_nefarious_activity(self):
|
||||
start = time.perf_counter()
|
||||
await asyncio.gather(
|
||||
*(s.do_nefarious_things() for s in self.agent_smiths),
|
||||
return_exceptions=True
|
||||
)
|
||||
return time.perf_counter() - start
|
||||
|
||||
def __len__(self):
|
||||
return len(self.agent_smiths)
|
||||
|
||||
async def delete_one_smith(self):
|
||||
if self.agent_smiths:
|
||||
await self.agent_smiths.pop().close()
|
||||
|
||||
async def delete_program(self):
|
||||
await asyncio.gather(*(
|
||||
s.close() for s in self.agent_smiths
|
||||
))
|
||||
|
||||
|
||||
async def main(host, port):
|
||||
smiths = AgentSmithProgram(host, port)
|
||||
await smiths.make_one_more_of_them()
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
ease_off = 0
|
||||
for i in range(1000):
|
||||
await asyncio.sleep(1)
|
||||
if activity.done() and activity.result() < .9:
|
||||
print('more, more, more...')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
|
||||
))
|
||||
else:
|
||||
print('!!!!!!!!!!!!!!')
|
||||
print('IS NEO LOSING?')
|
||||
print('!!!!!!!!!!!!!!')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
|
||||
))
|
||||
print(f'coordinate all {len(smiths)} smiths to action')
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
print('finishing up any remaining actions')
|
||||
await activity
|
||||
print('neo has won, deleting agents...')
|
||||
await smiths.delete_program()
|
||||
print('done.')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('--host', dest='host', default='localhost', type=str)
|
||||
parser.add_argument('--port', dest='port', default=50001, type=int)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.host, args.port))
|
||||
@@ -1,9 +1,10 @@
|
||||
import uvloop, asyncio, time, sys, logging
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.metrics import calculate_avg_percentiles
|
||||
|
||||
|
||||
db_path = '/tmp/wallet-server/claims.db'
|
||||
db_path = '../../../lbryconf/wallet-server/claims.db'
|
||||
default_query_timout = 0.25
|
||||
log = logging.getLogger(__name__)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
@@ -16,26 +17,26 @@ async def run_times(executor, iterations, show=True):
|
||||
'no_totals': True,
|
||||
'offset': 0,
|
||||
'limit': 20,
|
||||
'fee_amount': '<1',
|
||||
#'all_tags': ['funny'],
|
||||
'any_tags': [
|
||||
'crypto',
|
||||
'outdoors',
|
||||
'cars',
|
||||
'automotive'
|
||||
'ufos', 'city fix'
|
||||
],
|
||||
'not_tags': [
|
||||
'nsfw', 'xxx', 'mature'
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
],
|
||||
'order_by': [
|
||||
'release_time'
|
||||
]
|
||||
}
|
||||
) for _ in range(iterations)))
|
||||
timings = [r[1]['execute_query']['total'] for r in timings]
|
||||
timings = [r[1]['execute_query'][0]['total'] for r in timings]
|
||||
total = int((time.perf_counter() - start) * 100)
|
||||
if show:
|
||||
avg = sum(timings)/len(timings)
|
||||
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
|
||||
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
|
||||
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
|
||||
stats = calculate_avg_percentiles(timings)
|
||||
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
|
||||
sys.stdout.write(' sample:')
|
||||
for i, t in zip(range(10), timings[::-1]):
|
||||
sys.stdout.write(f' {t}ms')
|
||||
@@ -44,16 +45,16 @@ async def run_times(executor, iterations, show=True):
|
||||
|
||||
async def main():
|
||||
executor = ProcessPoolExecutor(
|
||||
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', default_query_timout, True)
|
||||
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
|
||||
)
|
||||
await run_times(executor, 4, show=False)
|
||||
await run_times(executor, 1)
|
||||
#await run_times(executor, 4, show=False)
|
||||
#await run_times(executor, 1)
|
||||
await run_times(executor, 2**3)
|
||||
await run_times(executor, 2**5)
|
||||
await run_times(executor, 2**7)
|
||||
await run_times(executor, 2**9)
|
||||
await run_times(executor, 2**11)
|
||||
await run_times(executor, 2**13)
|
||||
#await run_times(executor, 2**9)
|
||||
#await run_times(executor, 2**11)
|
||||
#await run_times(executor, 2**13)
|
||||
executor.shutdown(True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -89,6 +89,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([claim], **kwargs)
|
||||
|
||||
async def assertFindsClaims(self, claims, **kwargs):
|
||||
kwargs.setdefault('order_by', ['height', '^name'])
|
||||
results = await self.claim_search(**kwargs)
|
||||
self.assertEqual(len(claims), len(results))
|
||||
for claim, result in zip(claims, results):
|
||||
@@ -165,15 +166,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.create_channel()
|
||||
await self.create_lots_of_streams()
|
||||
|
||||
page = await self.claim_search(page_size=20, channel='@abc')
|
||||
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams)
|
||||
|
||||
page = await self.claim_search(page_size=6, channel='@abc')
|
||||
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[:6])
|
||||
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc')
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[6:])
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ class TestSessionBloat(IntegrationTestCase):
|
||||
|
||||
async def test_session_bloat_from_socket_timeout(self):
|
||||
await self.conductor.stop_spv()
|
||||
await self.ledger.stop()
|
||||
self.conductor.spv_node.session_timeout = 1
|
||||
await self.conductor.start_spv()
|
||||
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
|
||||
|
||||
@@ -11,7 +11,7 @@ class TestTagNormalization(unittest.TestCase):
|
||||
def test_normalize_tag(self):
|
||||
tag = self.assertNormalizedTag
|
||||
tag('', ' \t #!~')
|
||||
tag('tag', 'Tag')
|
||||
tag('tag', 'T\'ag')
|
||||
tag('t ag', '\tT \nAG ')
|
||||
tag('tag hash', '#tag~#hash!')
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import time
|
||||
import unittest
|
||||
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
|
||||
|
||||
|
||||
class TestPercentileCalculation(unittest.TestCase):
|
||||
|
||||
def test_calculate_percentiles(self):
|
||||
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
|
||||
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
|
||||
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
|
||||
self.assertEqual(calculate_avg_percentiles(
|
||||
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
|
||||
|
||||
|
||||
class TestCollectingMetrics(unittest.TestCase):
|
||||
|
||||
def test_happy_path(self):
|
||||
self.maxDiff = None
|
||||
load = ServerLoadData()
|
||||
search = load.for_api('search')
|
||||
self.assertEqual(search.name, 'search')
|
||||
search.start()
|
||||
search.cache_response()
|
||||
search.cache_response()
|
||||
metrics = {
|
||||
'search': [{'total': 40}],
|
||||
'execute_query': [
|
||||
{'total': 20},
|
||||
{'total': 10}
|
||||
]
|
||||
}
|
||||
for x in range(5):
|
||||
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
|
||||
metrics['execute_query'][0]['total'] = 10
|
||||
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
|
||||
search.query_interrupt(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.052, {})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
|
||||
"receive_count": 1,
|
||||
"cache_response_count": 2,
|
||||
"query_response_count": 5,
|
||||
"intrp_response_count": 1,
|
||||
"error_response_count": 2,
|
||||
"response": (53, 51, 51, 52, 53, 54, 55, 55),
|
||||
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
|
||||
"error": (51, 50, 50, 50, 50, 52, 52, 52),
|
||||
"python": (12, 10, 10, 10, 10, 20, 20, 20),
|
||||
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
|
||||
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
|
||||
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
|
||||
"individual_sql_count": 14,
|
||||
"errored_queries": ['FROM claim where something=1'],
|
||||
"interrupted_queries": ['FROM claim where something=1'],
|
||||
}}})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
|
||||
@@ -167,35 +167,35 @@ class TestClaimtrie(TestSQLDB):
|
||||
state(
|
||||
controlling=('Claim A', 10*COIN, 10*COIN, 13),
|
||||
active=[],
|
||||
accepted=[('Claim B', 20*COIN, 20*COIN, 1031)]
|
||||
accepted=[('Claim B', 20*COIN, 0, 1031)]
|
||||
)
|
||||
advance(1010, [self.get_support(stream, 14*COIN)])
|
||||
state(
|
||||
controlling=('Claim A', 10*COIN, 24*COIN, 13),
|
||||
active=[],
|
||||
accepted=[('Claim B', 20*COIN, 20*COIN, 1031)]
|
||||
accepted=[('Claim B', 20*COIN, 0, 1031)]
|
||||
)
|
||||
advance(1020, [self.get_stream('Claim C', 50*COIN)])
|
||||
state(
|
||||
controlling=('Claim A', 10*COIN, 24*COIN, 13),
|
||||
active=[],
|
||||
accepted=[
|
||||
('Claim B', 20*COIN, 20*COIN, 1031),
|
||||
('Claim C', 50*COIN, 50*COIN, 1051)]
|
||||
('Claim B', 20*COIN, 0, 1031),
|
||||
('Claim C', 50*COIN, 0, 1051)]
|
||||
)
|
||||
advance(1031, [])
|
||||
state(
|
||||
controlling=('Claim A', 10*COIN, 24*COIN, 13),
|
||||
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
|
||||
accepted=[('Claim C', 50*COIN, 50*COIN, 1051)]
|
||||
accepted=[('Claim C', 50*COIN, 0, 1051)]
|
||||
)
|
||||
advance(1040, [self.get_stream('Claim D', 300*COIN)])
|
||||
state(
|
||||
controlling=('Claim A', 10*COIN, 24*COIN, 13),
|
||||
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
|
||||
accepted=[
|
||||
('Claim C', 50*COIN, 50*COIN, 1051),
|
||||
('Claim D', 300*COIN, 300*COIN, 1072)]
|
||||
('Claim C', 50*COIN, 0, 1051),
|
||||
('Claim D', 300*COIN, 0, 1072)]
|
||||
)
|
||||
advance(1051, [])
|
||||
state(
|
||||
|
||||
@@ -53,35 +53,45 @@ class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
async def _make_fake_server(self, latency=1.0, port=1337):
|
||||
# local fake server with artificial latency
|
||||
proto = RPCSession()
|
||||
async def __handler(_):
|
||||
await asyncio.sleep(latency)
|
||||
return {'height': 1}
|
||||
|
||||
proto.handle_request = __handler
|
||||
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
# local fake server with artificial latency
|
||||
class FakeSession(RPCSession):
|
||||
async def handle_request(self, request):
|
||||
await asyncio.sleep(latency)
|
||||
return {"height": 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
async def echo(reader, writer):
|
||||
while True:
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
await self._make_fake_server(latency=1.5, port=1340),
|
||||
await self._make_fake_server(latency=0.1, port=1337),
|
||||
await self._make_fake_server(latency=1.0, port=1339),
|
||||
await self._make_fake_server(latency=0.5, port=1338),
|
||||
# fast but unhealthy, should be discarded
|
||||
await self._make_bad_server(),
|
||||
('localhost', 1),
|
||||
('example.that.doesnt.resolve', 9000),
|
||||
await self._make_fake_server(latency=1.2, port=1340),
|
||||
await self._make_fake_server(latency=0.5, port=1337),
|
||||
await self._make_fake_server(latency=0.7, port=1339),
|
||||
],
|
||||
'connect_timeout': 30
|
||||
'connect_timeout': 3
|
||||
})
|
||||
|
||||
network = BaseNetwork(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=3)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
# ensure we are connected to all of them
|
||||
self.assertEqual(len(network.session_pool.sessions), 4)
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
|
||||
self.assertEqual(len(network.session_pool.sessions), 3)
|
||||
@@ -1,11 +1,10 @@
|
||||
import unittest
|
||||
import sqlite3
|
||||
from functools import wraps
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
|
||||
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
|
||||
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
|
||||
from torba.client.hash import sha256
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
@@ -167,6 +166,26 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
("select * from foo LIMIT 20 OFFSET 10", {})
|
||||
)
|
||||
|
||||
def test_query_interpolation(self):
|
||||
self.maxDiff = None
|
||||
# tests that interpolation replaces longer keys first
|
||||
self.assertEqual(
|
||||
interpolate(*query(
|
||||
"select * from foo",
|
||||
a__not='b', b__in='select * from blah where c=:$c',
|
||||
d__any={'one__like': 'o', 'two': 2},
|
||||
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
|
||||
ahash=memoryview(sha256(b'hello world')),
|
||||
limit=10, order_by='b', **{'$c': 3})
|
||||
),
|
||||
"select * from foo WHERE a != 'b' AND "
|
||||
"b IN (select * from blah where c=3) AND "
|
||||
"(one LIKE 'o' OR two = 2) AND "
|
||||
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
|
||||
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
|
||||
"ORDER BY b LIMIT 10",
|
||||
)
|
||||
|
||||
|
||||
class TestQueries(AsyncioTestCase):
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from asyncio import wrap_future
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
|
||||
@@ -191,6 +192,19 @@ def query(select, **constraints):
|
||||
return ' '.join(sql), values
|
||||
|
||||
|
||||
def interpolate(sql, values):
|
||||
for k in sorted(values.keys(), reverse=True):
|
||||
value = values[k]
|
||||
if isinstance(value, memoryview):
|
||||
value = f"X'{hexlify(bytes(value)).decode()}'"
|
||||
elif isinstance(value, str):
|
||||
value = f"'{value}'"
|
||||
else:
|
||||
value = str(value)
|
||||
sql = sql.replace(f":{k}", value)
|
||||
return sql
|
||||
|
||||
|
||||
def rows_to_dict(rows, fields):
|
||||
if rows:
|
||||
return [dict(zip(fields, r)) for r in rows]
|
||||
|
||||
@@ -40,7 +40,11 @@ class ClientSession(BaseClientSession):
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
while not self.is_closing():
|
||||
if (time() - self.last_send) > self.max_seconds_idle:
|
||||
await self.send_request('server.banner')
|
||||
try:
|
||||
await self.send_request('server.banner')
|
||||
except:
|
||||
self.abort()
|
||||
raise
|
||||
await asyncio.sleep(self.max_seconds_idle//3)
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
@@ -223,10 +227,14 @@ class SessionPool:
|
||||
self._dead_servers = []
|
||||
|
||||
async def ensure_connection(self, session):
|
||||
if not session.is_closing():
|
||||
return
|
||||
self._dead_servers.append(session)
|
||||
self.sessions.remove(session)
|
||||
try:
|
||||
return await session.create_connection(self.timeout)
|
||||
if session.is_closing():
|
||||
await session.create_connection(self.timeout)
|
||||
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
|
||||
self.sessions.append(session)
|
||||
self._dead_servers.remove(session)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timeout connecting to %s:%d", *session.server)
|
||||
except asyncio.CancelledError: # pylint: disable=try-except-raise
|
||||
@@ -238,11 +246,9 @@ class SessionPool:
|
||||
log.warning("Could not connect to %s:%d", *session.server)
|
||||
else:
|
||||
log.exception("Connecting to %s:%d raised an exception:", *session.server)
|
||||
self._dead_servers.append(session)
|
||||
self.sessions.remove(session)
|
||||
|
||||
async def get_online_sessions(self):
|
||||
self._lost_master.set()
|
||||
while not self.online:
|
||||
await asyncio.sleep(0.1)
|
||||
self._lost_master.set()
|
||||
await asyncio.sleep(0.5)
|
||||
return self.sessions
|
||||
@@ -206,7 +206,8 @@ class SPVNode:
|
||||
'HOST': self.hostname,
|
||||
'TCP_PORT': str(self.port),
|
||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||
'MAX_QUERY_WORKERS': '0'
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
}
|
||||
# TODO: don't use os.environ
|
||||
os.environ.update(conf)
|
||||
@@ -265,6 +266,7 @@ class BlockchainNode:
|
||||
self.data_path = None
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.blockchain_address = None
|
||||
self._block_expected = 0
|
||||
self.hostname = 'localhost'
|
||||
self.peerport = 9246 + 2 # avoid conflict with default peer port
|
||||
@@ -328,7 +330,7 @@ class BlockchainNode:
|
||||
self.daemon_bin,
|
||||
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
|
||||
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
|
||||
f'-port={self.peerport}'
|
||||
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
|
||||
)
|
||||
self.log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
@@ -363,9 +365,10 @@ class BlockchainNode:
|
||||
self.log.info(out.decode().strip())
|
||||
return out.decode().strip()
|
||||
|
||||
def generate(self, blocks):
|
||||
async def generate(self, blocks, address=None):
|
||||
self._block_expected += blocks
|
||||
return self._cli_cmnd('generate', str(blocks))
|
||||
address = await self.get_raw_change_address()
|
||||
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
|
||||
|
||||
def invalidate_block(self, blockhash):
|
||||
return self._cli_cmnd('invalidateblock', blockhash)
|
||||
@@ -374,7 +377,7 @@ class BlockchainNode:
|
||||
return self._cli_cmnd('getblockhash', str(block))
|
||||
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
return self._cli_cmnd('getrawchangeaddress', 'legacy')
|
||||
|
||||
async def get_balance(self):
|
||||
return float(await self._cli_cmnd('getbalance'))
|
||||
|
||||
@@ -38,6 +38,7 @@ class Env:
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
|
||||
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
|
||||
|
||||
Reference in new issue
Block a user