Compare commits

...
78 Commits
Author SHA1 Message Date
Victor Shyba 7885f34221 upgrading latest lbrycrd wip 2019-07-23 12:42:05 -03:00
Alex Grin ff05eaae48 v0.38.3 2019-07-23 11:14:20 -04:00
Alex Grintsvayg b48b67ed10 fix aws key 2019-07-23 11:14:00 -04:00
Alex Grintsvayg 3ead6f63b6 add lbry-ci bot auth token 2019-07-23 11:04:10 -04:00
Alex Grin 77235f0ad8 v0.38.2 2019-07-22 17:45:43 -04:00
Alex Grintsvayg f6a80ba929 Revert "v0.38.2"
This reverts commit 295ceb90d5.
2019-07-22 17:45:12 -04:00
Alex Grin 295ceb90d5 v0.38.2 2019-07-22 13:31:20 -04:00
Victor Shyba df41a82b23 fix integration test 2019-07-22 11:31:45 -04:00
Victor Shyba 34500e07c6 change picking logic to probe the session before considering it a good session 2019-07-22 11:31:45 -04:00
Alex Grintsvayg 1e6097e3b2 more spv dns entries 2019-07-22 11:24:12 -04:00
Lex Berezhny 521facedc4 script to find max server load 2019-07-22 00:15:05 -04:00
Lex Berezhny de80a2ba97 fixed 0 results when only filtering by lots of common tags 2019-07-21 19:44:10 -04:00
Lex Berezhny 3afd615447 dont run common tag optimization strategy if there are 5 or more common tags provided 2019-07-20 15:45:15 -04:00
Lex Berezhny 87e52bec0a successfuly run indexes after full wallet server sync 2019-07-20 15:45:15 -04:00
Lex Berezhny b89f04b1f1 fix claim integration tests 2019-07-20 15:45:15 -04:00
Lex Berezhny e4f802e5f4 when resolving inside channel break a tie of claims with same effective_amount by using height 2019-07-20 15:45:15 -04:00
Lex Berezhny 16fc2c456f moved executing create index queries outside of a tx 2019-07-20 15:45:15 -04:00
Lex Berezhny ef7656ae98 removed some dead code 2019-07-20 15:45:15 -04:00
Lex Berezhny ad2df4871e popular tags are indexed and use different search strategy than less popular tags 2019-07-20 15:45:15 -04:00
Jack Robison 8d5f50b695 improve claim search test script to show sql for nearly interrupted queries 2019-07-19 14:17:21 -04:00
Lex Berezhny bbc056eef0 added better stats to sqlite_perf_test.py 2019-07-19 11:26:50 -04:00
Jack Robison 82fd3c632e another claim search performance test case 2019-07-19 10:50:37 -04:00
Jack Robison dbae939808 claim search performance test 2019-07-19 10:42:38 -04:00
Jack Robison c2dcacb0de more test cases 2019-07-19 10:37:15 -04:00
Lex Berezhny 8f45618cd0 added improved effective_amount index covering release_time 2019-07-18 23:24:00 -04:00
Lex Berezhny 29a87c850c fix sqlite_perf_test 2019-07-18 19:54:38 -04:00
Lex Berezhny 395518ff76 ver updates to new metrics 2019-07-18 19:54:38 -04:00
Lex Berezhny 08da763327 more metrics cleanup 2019-07-18 19:54:38 -04:00
Lex Berezhny d7ec3c182d tuples 2019-07-18 19:54:38 -04:00
Lex Berezhny ffe6418392 fix metrics json response test 2019-07-18 19:54:38 -04:00
Lex Berezhny 88adf3ee27 oops 2019-07-18 19:54:38 -04:00
Lex Berezhny f406db85dc pylint again 2019-07-18 19:54:38 -04:00
Lex Berezhny d5f8af471a pylint 2019-07-18 19:54:38 -04:00
Lex Berezhny 046459a72d refactored websockets and metrics 2019-07-18 19:54:38 -04:00
Lex Berezhny 49b0f59790 added metrics module 2019-07-18 19:54:38 -04:00
Lex Berezhny 71551058ce query interpolation 2019-07-18 19:54:38 -04:00
Jack Robison 2a9e6473eb Merge pull request #2310 from lbryio/2293
set file status to completed only when all bytes were written
2019-07-18 13:21:41 -04:00
Victor Shyba 576ef9c4ed remove unused variable 2019-07-17 18:54:21 -03:00
Victor Shyba 4d64fca9de fix memory blob showing as completed/verified 2019-07-17 18:54:21 -03:00
Victor Shyba 8b1307e1ca file status completed set only if all bytes were written 2019-07-17 18:54:21 -03:00
Jack Robison eae4ed7b27 Merge pull request #2317 from lbryio/new_2282
torba client networking: improve network timeout handling and fix spurious logs on shutdown
2019-07-17 17:49:59 -04:00
Victor Shyba 6997ea608d fix reconnect and timeouts 2019-07-16 23:59:28 -03:00
Alex Grintsvayg d97d738880 remove RCs from release tool 2019-07-16 21:35:21 -04:00
Lex Berezhny f72b07d0f4 fixup 2019-07-16 21:28:36 -04:00
Lex Berezhny b89916a3d7 guarded dict access 2019-07-16 21:15:39 -04:00
Lex Berezhny e4aeb66257 claim_list doesnt blow up 2019-07-16 21:08:51 -04:00
Jack Robison 578f8f4062 Merge pull request #2320 from lbryio/interrupt-slow-queries
Interrupt and log slow wallet server queries
2019-07-16 20:50:37 -04:00
Jack Robison 8fbf82f4ea filter mutually exclusive tags 2019-07-16 20:05:49 -04:00
Jack Robison 478ae7e657 add claim search performance script 2019-07-16 20:05:49 -04:00
Jack Robison d319761483 interrupt slow queries and log them for debugging
- add QUERY_TIMEOUT_MS env variable

- return sqlite timeout errors, jsonrpc error code -32000
2019-07-16 20:05:45 -04:00
Lex Berezhny 16fb716872 thicker chart line 2019-07-16 17:10:02 -04:00
Lex Berezhny b8d9d097d8 graph taller 2019-07-16 15:23:45 -04:00
Lex Berezhny ec79a1d0e8 improved graph ui 2019-07-16 15:21:41 -04:00
Lex Berezhny 31284e0c5e dont run empty queries 2019-07-16 13:57:22 -04:00
Lex Berezhny 8e64b1840d added errors key to metrics tracking 2019-07-16 13:39:17 -04:00
Lex Berezhny dd9bf04e35 default to HOST for WEBSOCKET_HOST if not set 2019-07-16 12:05:43 -04:00
Thomas Zarebczan 76a27fdef1 str > list for a few fields 2019-07-16 00:59:06 -04:00
Victor Shyba f96443515a optimize header check 2019-07-16 00:58:36 -04:00
Lex Berezhny 0f1bde2274 pylint 2019-07-16 00:57:53 -04:00
Lex Berezhny 6fb94e445f ver gets cache hit info 2019-07-16 00:57:53 -04:00
Lex Berezhny e02a4e44c0 working resolve/search caching 2019-07-16 00:57:53 -04:00
Lex Berezhny 71dc882d5d indexes update 2019-07-16 00:57:53 -04:00
Lex Berezhny ae4a29d41c prevent duplicate tags 2019-07-16 00:57:53 -04:00
Lex Berezhny d27a3b6421 support for UN M.49 region codes in language 2019-07-16 00:57:53 -04:00
Lex Berezhny 03a58cf50a added lrucache to search/resolve 2019-07-16 00:57:53 -04:00
Lex Berezhny f43e370e89 refactor 2019-07-16 00:57:53 -04:00
Lex Berezhny 45bdb9023a ui to add servers 2019-07-16 00:57:53 -04:00
Lex Berezhny 6d7aa93157 sqlite_perf_test uses lbry.wallet.server.db.reader 2019-07-16 00:57:53 -04:00
Lex Berezhny d083910255 refactored metrics gathering for wallet server 2019-07-16 00:57:53 -04:00
Lex Berezhny db3f9af174 refactored metrics gathering for wallet server 2019-07-16 00:57:53 -04:00
Lex Berezhny 7d2ad5e4ed added TRACK_METRICS env variable to torba wallet server 2019-07-16 00:57:53 -04:00
Lex Berezhny 84d3771984 more detailed stress testing metrics and graphs 2019-07-16 00:57:53 -04:00
Lex Berezhny 1eebe006ae pylint 2019-07-16 00:57:53 -04:00
Lex Berezhny 0327ce53a6 uncomment imports 2019-07-16 00:57:53 -04:00
Lex Berezhny 85442c1d07 websocket admin connection to get query metric data 2019-07-16 00:57:53 -04:00
Lex Berezhny a6ed51b7a7 websocket admin connection to get query metric data 2019-07-16 00:57:53 -04:00
Lex Berezhny dc8b4b5252 instrument wallet.server.db.reader with timer metrics 2019-07-16 00:57:53 -04:00
Lex Berezhny 8ca74d3de2 env variables to set websocket host and port 2019-07-16 00:57:53 -04:00
58 changed files with 3675 additions and 713 deletions

No files matched your search

+2 -1
View File
@@ -78,7 +78,8 @@ jobs:
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
file: lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
+2 -1
View File
@@ -1,5 +1,6 @@
import 'package:lbry/src/load_generator.dart' as load;
//import 'package:lbry/src/server_connection.dart' as load;
main(List<String> arguments) {
load.cli();
load.generate_load();
}
+2 -1
View File
@@ -1 +1,2 @@
export 'src/load_generator.dart' show LoadGenerator, LoadDataPoint;
export 'src/load_generator.dart';
export 'src/server_connection.dart';
+43 -23
View File
@@ -9,6 +9,7 @@ class LoadRequest {
final Map payload;
Completer completer;
Stopwatch timer;
Socket socket;
bool get isDone => completer.isCompleted;
int get elapsed => timer.elapsedMilliseconds;
@@ -21,6 +22,7 @@ class LoadRequest {
completer.future.whenComplete(() => timer.stop());
try {
Socket.connect(this.host, this.port).then((socket) {
this.socket = socket;
utf8.decoder.bind(socket).listen((r) {
if (r.contains('"jsonrpc": "2.0", "result": ')) {
socket.close();
@@ -44,24 +46,33 @@ class LoadRequest {
}
}
typedef bool LoadTestCallback(LoadGenerator load_generator, LoadDataPoint stats);
typedef bool LoadTickCallback(ClientLoadGenerator load_generator, ClientLoadDataPoint stats);
class LoadGenerator {
int load = 1;
class ClientLoadGenerator {
Timer _timer;
String host;
int port;
final String host;
final int port;
final LoadTickCallback tickCallback;
int load = 1;
Map query;
List<LoadRequest> backlog = [];
LoadTestCallback cb;
LoadGenerator(this.host, this.port, this.query, this.cb);
ClientLoadGenerator(this.host, this.port, {this.tickCallback, this.query}) {
if (query == null) {
query = {
'id': 1,
'method': 'blockchain.claimtrie.resolve',
'params': ['one', 'two', 'three']
};
}
}
start() {
var previous = spawn_requests();
var backlog = <LoadRequest>[];
backlog = <LoadRequest>[];
_timer = Timer.periodic(Duration(seconds: 1), (t) {
var stat = LoadDataPoint();
var stat = ClientLoadDataPoint(t.tick);
backlog.removeWhere((r) {
if (r.isDone) stat.addCatchup(r);
return r.isDone;
@@ -75,16 +86,20 @@ class LoadGenerator {
}
stat.backlog = backlog.length;
stat.load = load;
if (cb(this, stat)) {
stat.close();
if (tickCallback(this, stat)) {
previous = spawn_requests();
} else {
t.cancel();
stop();
}
});
}
stop() {
_timer.cancel();
for (var remaining in backlog) {
remaining?.socket?.close();
}
}
List<LoadRequest> spawn_requests() {
@@ -97,18 +112,20 @@ class LoadGenerator {
}
class LoadDataPoint {
final DateTime time = new DateTime.now();
class ClientLoadDataPoint {
final int tick;
int success = 0;
int errored = 0;
int backlog = 0;
int catchup = 0;
int _success_total = 0;
int avg_success = 0;
int _catchup_total = 0;
int avg_catchup = 0;
int load = 0;
int get avg_success => _success_total > 0 ? (_success_total/success).round() : 0;
int get avg_catchup => _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
ClientLoadDataPoint(this.tick);
ClientLoadDataPoint.empty(): this(0);
addSuccess(LoadRequest r) {
success++; _success_total += r.elapsed;
@@ -117,16 +134,19 @@ class LoadDataPoint {
addCatchup(LoadRequest r) {
catchup++; _catchup_total += r.elapsed;
}
close() {
avg_success = _success_total > 0 ? (_success_total/success).round() : 0;
avg_catchup = _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
}
}
cli() {
generate_load() {
var runs = 1;
LoadGenerator('localhost', 50001, {
'id': 1,
'method': 'blockchain.claimtrie.resolve',
'params': ['one', 'two', 'three']
}, (t, stats) {
print("run ${runs}: ${stats}");
ClientLoadGenerator('localhost', 50001, tickCallback: (t, stats) {
print("run ${runs}: ${stats.load} load, ${stats.success} success, ${stats.backlog} backlog");
t.load = (runs < 4 ? t.load*2 : t.load/2).round();
return runs++ < 10;
}).start();
@@ -0,0 +1,159 @@
import 'dart:async';
import 'dart:convert';
import 'package:web_socket_channel/io.dart';
import 'package:web_socket_channel/status.dart' as status;
class ServerConnection {
String url = "";
IOWebSocketChannel channel;
bool get isConnected => channel != null && channel.closeCode == null;
final StreamController<ServerLoadDataPoint> _loadDataController = StreamController.broadcast();
Stream<ServerLoadDataPoint> get load_data => _loadDataController.stream;
ServerConnection({this.url});
open() {
if (isConnected) return Future.value('already open');
channel = IOWebSocketChannel.connect(this.url);
int tick = 1;
channel.stream.listen((message) {
var data = json.decode(message);
print(data);
_loadDataController.add(
ServerLoadDataPoint.from_map(
tick, data
)
);
tick++;
});
}
close() {
if (isConnected) {
return channel.sink.close(status.goingAway);
}
return Future.value('already closed');
}
subscribe_to_server_load_data() {
if (isConnected) channel.sink.add('subscribe');
}
unsubscribe_from_server_load_data() {
if (isConnected) channel.sink.add('unsubscribe');
}
}
class TimeStats {
final int avg, min, max;
// percentiles
final int five;
final int twenty_five;
final int fifty;
final int seventy_five;
final int ninety_five;
TimeStats(
this.avg, this.min, this.five, this.twenty_five,
this.fifty, this.seventy_five, this.ninety_five,
this.max
);
TimeStats.from_list(List l): this(
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
);
TimeStats.from_zeros(): this(
0, 0, 0, 0, 0, 0, 0, 0
);
factory TimeStats.from_list_or_zeros(List l) =>
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
}
class APICallMetrics {
// total requests received
final int receive_count;
// sum of these is total responses made
final int cache_response_count;
final int query_response_count;
final int intrp_response_count;
final int error_response_count;
// stacked values for chart
final int cache_response_stack;
final int query_response_stack;
final int intrp_response_stack;
final int error_response_stack;
// millisecond timings for non-cache responses
final TimeStats response;
final TimeStats interrupt;
final TimeStats error;
// response, interrupt and error each also report the python, wait and sql stats:
final TimeStats python;
final TimeStats wait;
final TimeStats sql;
// extended timings for individual sql executions
final TimeStats individual_sql;
final int individual_sql_count;
// actual queries
final List<String> errored_queries;
final List<String> interrupted_queries;
APICallMetrics(
this.receive_count,
this.cache_response_count, this.query_response_count,
this.intrp_response_count, this.error_response_count,
this.response, this.interrupt, this.error,
this.python, this.wait, this.sql,
this.individual_sql, this.individual_sql_count,
this.errored_queries, this.interrupted_queries
):
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
query_response_stack=query_response_count+intrp_response_count+error_response_count,
intrp_response_stack=intrp_response_count+error_response_count,
error_response_stack=error_response_count;
APICallMetrics.from_map(Map data): this(
data["receive_count"] ?? 0,
data["cache_response_count"] ?? 0,
data["query_response_count"] ?? 0,
data["intrp_response_count"] ?? 0,
data["error_response_count"] ?? 0,
TimeStats.from_list_or_zeros(data["response"]),
TimeStats.from_list_or_zeros(data["interrupt"]),
TimeStats.from_list_or_zeros(data["error"]),
TimeStats.from_list_or_zeros(data["python"]),
TimeStats.from_list_or_zeros(data["wait"]),
TimeStats.from_list_or_zeros(data["sql"]),
TimeStats.from_list_or_zeros(data["individual_sql"]),
data["individual_sql_count"] ?? 0,
List<String>.from(data["errored_queries"] ?? const []),
List<String>.from(data["interrupted_queries"] ?? const []),
);
}
class ServerLoadDataPoint {
final int tick;
final int sessions;
final APICallMetrics search;
final APICallMetrics resolve;
const ServerLoadDataPoint(
this.tick, this.sessions, this.search, this.resolve
);
ServerLoadDataPoint.from_map(int tick, Map data): this(
tick, (data['status'] ?? const {})['sessions'] ?? 0,
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
);
ServerLoadDataPoint.empty(): this(
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
);
}
connect_and_listen_for_load_data() {
ServerConnection(url: 'ws://localhost:8181/')..open()..load_data.listen((m) {
print(m.search);
});
}
+22 -22
View File
@@ -7,7 +7,7 @@ packages:
name: analyzer
url: "https://pub.dartlang.org"
source: hosted
version: "0.36.4"
version: "0.37.0"
args:
dependency: transitive
description:
@@ -63,14 +63,14 @@ packages:
name: csslib
url: "https://pub.dartlang.org"
source: hosted
version: "0.16.0"
version: "0.16.1"
dart_style:
dependency: transitive
description:
name: dart_style
url: "https://pub.dartlang.org"
source: hosted
version: "1.2.8"
version: "1.2.9"
fixnum:
dependency: transitive
description:
@@ -84,7 +84,7 @@ packages:
name: front_end
url: "https://pub.dartlang.org"
source: hosted
version: "0.1.19"
version: "0.1.20"
glob:
dependency: transitive
description:
@@ -134,20 +134,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "0.6.1+1"
json_rpc_2:
dependency: transitive
description:
name: json_rpc_2
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.0"
kernel:
dependency: transitive
description:
name: kernel
url: "https://pub.dartlang.org"
source: hosted
version: "0.3.19"
version: "0.3.20"
matcher:
dependency: transitive
description:
@@ -210,7 +203,7 @@ packages:
name: pedantic
url: "https://pub.dartlang.org"
source: hosted
version: "1.7.0"
version: "1.8.0+1"
pool:
dependency: transitive
description:
@@ -224,14 +217,14 @@ packages:
name: protobuf
url: "https://pub.dartlang.org"
source: hosted
version: "0.13.12"
version: "0.13.15"
protoc_plugin:
dependency: "direct dev"
description:
name: protoc_plugin
url: "https://pub.dartlang.org"
source: hosted
version: "17.0.2"
version: "17.0.4"
pub_semver:
dependency: transitive
description:
@@ -302,6 +295,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.0"
stream_transform:
dependency: transitive
description:
name: stream_transform
url: "https://pub.dartlang.org"
source: hosted
version: "0.0.19"
string_scanner:
dependency: transitive
description:
@@ -322,7 +322,7 @@ packages:
name: test
url: "https://pub.dartlang.org"
source: hosted
version: "1.6.4"
version: "1.6.5"
test_api:
dependency: transitive
description:
@@ -336,7 +336,7 @@ packages:
name: test_core
url: "https://pub.dartlang.org"
source: hosted
version: "0.2.6"
version: "0.2.7"
typed_data:
dependency: transitive
description:
@@ -344,13 +344,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "1.1.6"
vm_service_client:
vm_service_lib:
dependency: transitive
description:
name: vm_service_client
name: vm_service_lib
url: "https://pub.dartlang.org"
source: hosted
version: "0.2.6+2"
version: "3.22.0"
watcher:
dependency: transitive
description:
@@ -359,12 +359,12 @@ packages:
source: hosted
version: "0.9.7+10"
web_socket_channel:
dependency: transitive
dependency: "direct main"
description:
name: web_socket_channel
url: "https://pub.dartlang.org"
source: hosted
version: "1.0.13"
version: "1.0.14"
yaml:
dependency: transitive
description:
+1
View File
@@ -11,6 +11,7 @@ dependencies:
http: ^0.12.0+2
path: ^1.4.1
protobuf: ^0.13.12
web_socket_channel: ^1.0.4
dev_dependencies:
pedantic: ^1.0.0
+69 -35
View File
@@ -1,49 +1,83 @@
import 'package:flutter/material.dart';
import 'package:flutter/foundation.dart' show debugDefaultTargetPlatformOverride;
import 'package:provider/provider.dart';
import 'package:ver/src/servers.dart';
import 'package:ver/src/models/server.dart';
import 'package:ver/utils.dart';
import 'package:ver/time_series_chart.dart';
class VerApp extends StatelessWidget {
@override
Widget build(BuildContext context) {
return MaterialApp(
title: 'Ver',
theme: ThemeData(
brightness: Brightness.light,
primarySwatch: Colors.lightBlue,
fontFamily: 'Roboto',
),
home: VerHomePage(title: 'Wallet Server'),
);
}
class UnderConstructionPage extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(title: Text('Under Construction')),
body: SizedBox.expand(
child: Center(child: Text('Under Construction')),
),
);
}
}
class VerHomePage extends StatefulWidget {
VerHomePage({Key key, this.title}) : super(key: key);
final String title;
@override
_VerHomePageState createState() => _VerHomePageState();
class MainPage extends StatefulWidget {
@override
_MainPageState createState() => _MainPageState();
}
class _VerHomePageState extends State<VerHomePage> {
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(
title: Text(widget.title),
),
body: new Padding(
padding: const EdgeInsets.all(8.0),
child: SimpleTimeSeriesChart()
),
);
}
}
class _MainPageState extends State<MainPage> {
int _currentIndex = 3;
@override
Widget build(BuildContext context) {
return Scaffold(
body: IndexedStack(
index: _currentIndex,
children: [
UnderConstructionPage(),
UnderConstructionPage(),
UnderConstructionPage(),
ServersSectionNavigation(),
UnderConstructionPage(),
],
),
bottomNavigationBar: BottomNavigationBar(
items: const <BottomNavigationBarItem>[
BottomNavigationBarItem(icon: Icon(Icons.home), title: Text('Home')),
BottomNavigationBarItem(icon: Icon(Icons.trending_up), title: Text('Trending')),
BottomNavigationBarItem(icon: Icon(Icons.subscriptions), title: Text('Subscriptions')),
BottomNavigationBarItem(icon: Icon(Icons.router), title: Text('Servers')),
BottomNavigationBarItem(icon: Icon(Icons.folder), title: Text('Library')),
],
currentIndex: _currentIndex,
onTap: (int index) {
setState(() {
_currentIndex = index;
});
},
selectedItemColor: Colors.amber[800],
),
);
}
}
void main() {
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
runApp(new VerApp());
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
runApp(
MaterialApp(
title: 'Ver',
theme: ThemeData(
brightness: Brightness.dark,
//primaryColor: Colors.lightBlue[800],
//accentColor: Colors.cyan[600],
fontFamily: 'Roboto',
),
home: MultiProvider(
providers: [
ChangeNotifierProvider<ServerManager>(builder: (context) => ServerManager())
],
child: MainPage()
)
)
);
}
@@ -0,0 +1,190 @@
import 'dart:async';
import 'dart:collection';
import 'package:flutter/foundation.dart';
import 'package:lbry/lbry.dart';
class ServerManager extends ChangeNotifier {
final List<Server> _servers = [];
UnmodifiableListView<Server> get items => UnmodifiableListView(_servers);
add(Server server) {
_servers.add(server);
notifyListeners();
}
remove(Server server) {
server.dispose();
_servers.remove(server);
notifyListeners();
}
@override
void dispose() {
super.dispose();
for (var server in _servers) {
server.dispose();
}
}
}
class Server extends ChangeNotifier {
String _label = "";
String get label => _label;
String get labelOrHost => _label.length > 0 ? _label : host;
String _host = "";
String get host => _host;
int _port = 8181;
int get port => _port;
bool _ssl = false;
bool get ssl => _ssl;
String get url => _connection.url;
_setURL(String host, int port, bool ssl) {
_host = host;
_port = port;
_connection.url = "ws${ssl?'s':''}://$host:$port";
}
DateTime _added = new DateTime.now();
String get added => _added.toIso8601String();
bool _isDefault = false;
bool get isDefault => _isDefault;
bool _isEnabled = false;
bool get isEnabled => _isEnabled;
bool _isTrackingServerLoad = false;
bool get isTrackingServerLoad => _isTrackingServerLoad;
_setIsTrackingServerLoad(bool toggle_tracking) {
if (_isTrackingServerLoad && !toggle_tracking) {
_connection.unsubscribe_from_server_load_data();
} else if (!_isTrackingServerLoad && toggle_tracking) {
_connection.subscribe_to_server_load_data();
}
_isTrackingServerLoad = toggle_tracking;
}
ClientLoadManager clientLoadManager;
final ServerConnection _connection = ServerConnection();
bool get isConnected => _connection.isConnected;
Stream<ServerLoadDataPoint> get serverLoadStream => _connection.load_data;
final List<ServerLoadDataPoint> serverLoadData = [ServerLoadDataPoint.empty()];
Server() {
clientLoadManager = ClientLoadManager(this);
serverLoadStream.listen(serverLoadData.add);
}
update({String host, int port, bool ssl, String label,
bool isDefault, bool isEnabled, bool isTrackingServerLoad}) {
if (host != null && port != null && ssl != null) {
_setURL(host, port, ssl);
}
if (isTrackingServerLoad != null) {
_setIsTrackingServerLoad(isTrackingServerLoad);
}
_label = label ?? _label;
_isDefault = isDefault ?? _isDefault;
_isEnabled = isEnabled ?? _isEnabled;
notifyListeners();
}
connect() async {
await _connection.open();
if (_isTrackingServerLoad) {
_connection.subscribe_to_server_load_data();
}
notifyListeners();
}
disconnect() {
clientLoadManager.stop();
_connection.close();
notifyListeners();
}
@override
void dispose() {
disconnect();
super.dispose();
}
}
class ClientLoadManager extends ChangeNotifier {
final Server _server;
ClientLoadGenerator clientLoadGenerator;
final StreamController<ClientLoadDataPoint> _loadDataController = StreamController.broadcast();
Stream<ClientLoadDataPoint> get clientLoadStream => _loadDataController.stream;
final List<ClientLoadDataPoint> clientLoadData = [ClientLoadDataPoint.empty()];
int _load = 1;
int get load => _load;
int _offset = 0;
int get offset => _offset;
bool _noTotals = false;
bool get noTotals => _noTotals;
ClientLoadManager(this._server) {
clientLoadStream.listen(clientLoadData.add);
}
update({int load, int offset, bool noTotals}) {
_load = load ?? _load;
_offset = offset ?? _offset;
_noTotals = noTotals ?? _noTotals;
notifyListeners();
}
start() {
clientLoadData.clear();
clientLoadData.add(ClientLoadDataPoint.empty());
clientLoadGenerator = ClientLoadGenerator(
_server.host, _server.port,
query: {
'id': 1,
'method': 'blockchain.claimtrie.search',
'params': {
'no_totals': _noTotals,
'offset': _offset,
'limit': 20,
'fee_amount': '<1',
//'all_tags': ['funny'],
'any_tags': [
'crypto',
'outdoors',
'cars',
'automotive'
]
}
}, tickCallback: (t, stats) {
_loadDataController.add(stats);
//increase = max(1, min(100, increase+2)-stats.backlog);
//increase += 1;
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
t.load = _load;//rand.nextInt(10)+5;
return true;
})..start();
}
stop() {
if (clientLoadGenerator != null) {
clientLoadGenerator.stop();
}
}
@override
void dispose() {
super.dispose();
stop();
}
}
+73
View File
@@ -0,0 +1,73 @@
import 'package:flutter/material.dart';
import 'package:ver/src/widgets/server.dart';
import 'package:provider/provider.dart';
import 'models/server.dart';
class ServerListPage extends StatelessWidget {
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text('Servers')),
body: ServerList(),
floatingActionButton: FloatingActionButton(
child: Icon(Icons.add),
onPressed: () => Navigator.of(context).pushNamed('/edit', arguments: true),
)
);
}
class ServerFormPage extends StatelessWidget {
final bool creating;
ServerFormPage(this.creating);
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text(creating ? 'Add Server' : 'Modify Server')),
body: ServerForm(creating),
);
}
class ServerViewPage extends StatelessWidget {
final Server server;
ServerViewPage(this.server);
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text(server.labelOrHost)),
body: ChangeNotifierProvider<Server>.value(
value: server,
child: ServerView()
)
);
}
class ServersSectionNavigation extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Navigator(
onGenerateRoute: (RouteSettings settings) {
return MaterialPageRoute(
settings: settings,
builder: (BuildContext context) {
switch (settings.name) {
case '/':
return ServerListPage();
case '/edit':
return ServerFormPage(settings.arguments);
case '/view':
return ServerViewPage(settings.arguments);
}
return ServerListPage();
},
);
},
);
}
}
@@ -0,0 +1,172 @@
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:provider/provider.dart';
import '../models/server.dart';
import 'time_series_chart.dart';
class ServerList extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Consumer<ServerManager>(
builder: (_, servers, __) =>
ListView.builder(
itemCount: servers.items.length,
itemBuilder: (context, index) =>
ChangeNotifierProvider<Server>.value(
value: servers.items[index],
child: ServerListItem()
)
)
);
}
}
class ServerListItem extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Card(
child: Consumer<Server>(
builder: (_, server, __) =>
ListTile(
leading: FlutterLogo(size: 72.0),
title: Text(server.labelOrHost),
subtitle: Text("${server.url}\nadded ${server.added}"),
trailing: Icon(Icons.more_vert),
isThreeLine: true,
onTap: ()=> Navigator.of(context).pushNamed('/view', arguments: server),
)
)
);
}
}
class ServerView extends StatelessWidget {
@override
Widget build(BuildContext context) => ServerCharts();
}
class ServerForm extends StatefulWidget {
final bool creating;
ServerForm(this.creating);
@override
_ServerFormState createState() => _ServerFormState();
}
class _ServerFormState extends State<ServerForm> {
final _formKey = GlobalKey<FormState>();
final _formData = {
'label': '',
'host': 'localhost',
'port': 8181,
'ssl': false,
'isDefault': true,
'isEnabled': false,
'isTrackingServerLoad': false
};
@override
Widget build(BuildContext context) {
return Form(
key: _formKey,
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: <Widget>[
ListTile(title: TextFormField(
initialValue: _formData['label'],
decoration: const InputDecoration(
labelText: 'Label',
hintText: 'Optional text to display in server list.'
),
onSaved: (value) => _formData['label'] = value,
)),
ListTile(title: TextFormField(
initialValue: _formData['host'],
keyboardType: TextInputType.url,
inputFormatters: [
WhitelistingTextInputFormatter(RegExp(r'[\w\-\.]+'))
],
decoration: const InputDecoration(
labelText: 'Host Name',
hintText: 'Enter the host name of the server.'
),
validator: (value) => value.isEmpty ? 'A host name is required.' : null,
onSaved: (value) => _formData['host'] = value,
)),
ListTile(title: TextFormField(
initialValue: _formData['port'].toString(),
keyboardType: TextInputType.number,
inputFormatters: [
WhitelistingTextInputFormatter.digitsOnly
],
decoration: const InputDecoration(
labelText: 'Port',
hintText: 'Enter the port of the server.'
),
validator: (value) => value.isEmpty ? 'A port is required.' : null,
onSaved: (value) => setState(() => _formData['port'] = int.parse(value)),
)),
SwitchListTile(
title: Text('Requires SSL.'),
value: _formData['ssl'],
onChanged: (value) => setState(() => _formData['ssl'] = value),
),
SwitchListTile(
title: Text('Use this as your primary and default server.'),
value: _formData['isDefault'],
onChanged: (value) => setState(() => _formData['isDefault'] = value),
),
SwitchListTile(
title: Text('Always stay connected.'),
value: _formData['isEnabled'],
onChanged: (value) => setState(() => _formData['isEnabled'] = value),
),
SwitchListTile(
title: Text('Track server load.'),
value: _formData['isTrackingServerLoad'],
onChanged: (value) =>
setState(() => _formData['isTrackingServerLoad'] = value),
),
ListTile(title: RaisedButton(
onPressed: () {
if (_formKey.currentState.validate()) {
_formKey.currentState.save();
var manager = Provider.of<ServerManager>(
context, listen: false
);
var server = Server();
server.update(
label: _formData['label'],
host: _formData['host'],
port: _formData['port'],
ssl: _formData['ssl'],
isDefault: _formData['isDefault'],
isEnabled: _formData['isEnabled'],
isTrackingServerLoad: _formData['isTrackingServerLoad'],
);
manager.add(server);
if (server.isEnabled) {
server.connect();
}
Navigator.of(context).pop();
}
},
child: Text(widget.creating ? 'Add Server' : 'Update Server'),
),
),
],
),
);
}
}
@@ -0,0 +1,354 @@
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:charts_flutter/flutter.dart' as charts;
import 'package:charts_flutter/src/base_chart_state.dart' as state;
import 'package:charts_common/common.dart' as common;
import 'package:provider/provider.dart';
import 'package:lbry/lbry.dart';
import '../models/server.dart';
class ServerCharts extends StatelessWidget {
@override
Widget build(BuildContext context) {
var server = Provider.of<Server>(context, listen: false);
return ListView(children: <Widget>[
SizedBox(height: 300.0, child: APILoadChart(
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
)),
//SizedBox(height: 300.0, child: APILoadChart(
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
//)),
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
]);
}
}
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
class APILoadChart extends StatefulWidget {
final Server server;
final String name;
final APIGetter getter;
APILoadChart(this.server, this.name, this.getter);
@override
State<StatefulWidget> createState() => APILoadChartState();
}
class APILoadChartState extends State<APILoadChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
/*
charts.Series<ServerLoadDataPoint, int>(
id: 'Received',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
strokeWidthPxFn: (_, __) => 4.0,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
data: widget.server.serverLoadData,
),*/
charts.Series<ServerLoadDataPoint, int>(
id: 'Cache',
colorFn: (_, __) =>
charts.MaterialPalette.green.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Query',
colorFn: (_, __) =>
charts.MaterialPalette.blue.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Interrupts',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Errors',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
data: widget.server.serverLoadData,
),
/*
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Interrupted',
colorFn: (_, __) =>
charts.MaterialPalette.pink.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Errored',
colorFn: (_, __) =>
charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} From Cache',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
data: widget.server.serverLoadData,
),
*/
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData,
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
)
);
}
}
class ServerPerformanceChart extends StatefulWidget {
final Server server;
ServerPerformanceChart(this.server);
@override
State<StatefulWidget> createState() => ServerPerformanceChartState();
}
class ServerPerformanceChartState extends State<ServerPerformanceChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ServerLoadDataPoint, int>(
id: 'Waiting 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Waiting',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Executing',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'SQLite 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. SQLite',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
data: widget.server.serverLoadData,
),
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class ClientLoadChart extends StatefulWidget {
final ClientLoadManager client;
ClientLoadChart(this.client);
@override
State<StatefulWidget> createState() => ClientLoadChartState();
}
class ClientLoadChartState extends State<ClientLoadChart> {
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ClientLoadDataPoint, int>(
id: 'Load',
colorFn: (_, __) => charts.MaterialPalette.black.darker,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.load,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Success',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.success,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Backlog',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.backlog,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Catch-up',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.catchup,
data: widget.client.clientLoadData,
)
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ClientLoadDataPoint>(
stream: widget.client.clientLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class ClientPerformanceChart extends StatefulWidget {
final ClientLoadManager client;
ClientPerformanceChart(this.client);
@override
State<StatefulWidget> createState() => ClientPerformanceChartState();
}
class ClientPerformanceChartState extends State<ClientPerformanceChart> {
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ClientLoadDataPoint, int>(
id: 'Avg. Success Time',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.avg_success,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Avg. Catch-up Time',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.avg_catchup,
data: widget.client.clientLoadData,
),
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ClientLoadDataPoint>(
stream: widget.client.clientLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class BetterLineChart extends charts.LineChart {
final int itemCount;
final Object lastItem;
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(
seriesList,
animate: false,
defaultRenderer: renderer,
behaviors: [charts.SeriesLegend()],
domainAxis: charts.NumericAxisSpec(
viewport: new charts.NumericExtents(
max(0, seriesList[0].data.last.tick - 60), seriesList[0].data.last.tick
),
renderSpec: new charts.SmallTickRendererSpec(
labelStyle: new charts.TextStyleSpec(
color: charts.MaterialPalette.gray.shade50
),
lineStyle: new charts.LineStyleSpec(
color: charts.MaterialPalette.black
)
),
),
primaryMeasureAxis: new charts.NumericAxisSpec(
renderSpec: new charts.GridlineRendererSpec(
labelStyle: new charts.TextStyleSpec(
color: charts.MaterialPalette.white
),
lineStyle: new charts.LineStyleSpec(
color: charts.MaterialPalette.gray.shade100
),
),
tickProviderSpec: new charts.BasicNumericTickProviderSpec(
dataIsInWholeNumbers: true,
desiredTickCount: 5
)
),
);
@override
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
state.BaseChartState chartState) {
super.updateCommonChart(baseChart, oldWidget, chartState);
final prev = oldWidget as BetterLineChart;
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
chartState.markChartDirty();
}
}
}
@@ -1,149 +0,0 @@
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:charts_flutter/flutter.dart' as charts;
import 'package:charts_flutter/src/base_chart_state.dart' as state;
import 'package:charts_common/common.dart' as common;
import 'package:lbry/lbry.dart';
class SimpleTimeSeriesChart extends StatefulWidget {
SimpleTimeSeriesChart({Key key}) : super(key: key);
@override
_SimpleTimeSeriesChartState createState() => _SimpleTimeSeriesChartState();
}
class _SimpleTimeSeriesChartState extends State<SimpleTimeSeriesChart> {
final List<LoadDataPoint> loadData = [];
final List<charts.Series<LoadDataPoint, DateTime>> loadSeries = [];
final List<charts.Series<LoadDataPoint, DateTime>> timeSeries = [];
final Random rand = Random();
LoadGenerator loadGenerator;
@override
void initState() {
super.initState();
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Load',
colorFn: (_, __) => charts.MaterialPalette.black.darker,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.load,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Success',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.success,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Backlog',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.backlog,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Catch-up',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.catchup,
data: loadData,
)
);
timeSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Avg. Success Time',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.avg_success,
data: loadData,
)
);
timeSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Avg. Catch-up Time',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.avg_catchup,
data: loadData,
)
);
var increase = 1;
loadData.add(LoadDataPoint());
loadGenerator = LoadGenerator('spv2.lbry.com', 50001, {
'id': 1,
'method': 'blockchain.claimtrie.search',
'params': {
'offset': 0,
'limit': 20,
'fee_amount': '<1',
'all_tags': ['funny'],
'any_tags': [
'crypto',
'outdoors',
'cars',
'automotive'
]
}
}, (t, stats) {
setState(() {
//if (loadData.length > 60) loadData.removeAt(0);
loadData.add(stats);
});
//increase = max(1, min(30, (increase*1.1).ceil())-stats.backlog);
increase += 1;
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
t.load = increase;//rand.nextInt(10)+5;
return true;
})..start();
}
@override
void dispose() {
loadGenerator.stop();
super.dispose();
}
@override
Widget build(BuildContext context) {
return Column(children: <Widget>[
SizedBox(height: 250.0, child: BetterTimeSeriesChart(loadSeries)),
SizedBox(height: 250.0, child: BetterTimeSeriesChart(timeSeries)),
]);
}
}
class BetterTimeSeriesChart extends charts.TimeSeriesChart {
final int itemCount;
final Object lastItem;
BetterTimeSeriesChart(
List<charts.Series<dynamic, DateTime>> seriesList):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(seriesList, behaviors: [charts.SeriesLegend()]);
@override
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
state.BaseChartState chartState) {
super.updateCommonChart(baseChart, oldWidget, chartState);
final prev = oldWidget as BetterTimeSeriesChart;
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
chartState.markChartDirty();
}
}
}
@@ -60,7 +60,7 @@ int main(int argc, char **argv) {
flutter::FlutterWindowController flutter_controller(icu_data_path);
// Start the engine.
if (!flutter_controller.CreateWindow(800, 600, "ver - lbry workbench",
if (!flutter_controller.CreateWindow(800, 1200, "ver - lbry workbench",
assets_path, arguments)) {
return EXIT_FAILURE;
}
+28
View File
@@ -43,6 +43,20 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "1.14.11"
convert:
dependency: transitive
description:
name: convert
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.1"
crypto:
dependency: transitive
description:
name: crypto
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.6"
cupertino_icons:
dependency: "direct main"
description:
@@ -137,6 +151,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "0.13.15"
provider:
dependency: "direct main"
description:
name: provider
url: "https://pub.dartlang.org"
source: hosted
version: "3.0.0+1"
quiver:
dependency: transitive
description:
@@ -205,5 +226,12 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.8"
web_socket_channel:
dependency: transitive
description:
name: web_socket_channel
url: "https://pub.dartlang.org"
source: hosted
version: "1.0.14"
sdks:
dart: ">=2.2.2 <3.0.0"
+1
View File
@@ -12,6 +12,7 @@ dependencies:
sdk: flutter
cupertino_icons: ^0.1.2
charts_flutter: ^0.6.0
provider: ^3.0.0
lbry:
path: ../lbry
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.1"
__version__ = "0.38.3"
version = tuple(__version__.split('.'))
+2 -2
View File
@@ -69,10 +69,10 @@ class BlobManager:
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
+3
View File
@@ -551,6 +551,9 @@ class Config(CLIConfig):
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
+23 -18
View File
@@ -1742,12 +1742,12 @@ class Daemon(metaclass=JSONRPCServerType):
multiple channels at the same time,
includes claims with invalid signatures,
use in conjunction with --valid_channel_signature
--channel_ids=<channel_ids> : (str) claims signed by any of these channels
--channel_ids=<channel_ids> : (list) claims signed by any of these channels
(arguments must be claim ids of the channels),
includes claims with invalid signatures,
implies --has_channel_signature,
use in conjunction with --valid_channel_signature
--not_channel_ids=<not_channel_ids>: (str) exclude claims signed by any of these channels
--not_channel_ids=<not_channel_ids>: (list) exclude claims signed by any of these channels
(arguments must be claim ids of the channels)
--has_channel_signature : (bool) claims with a channel signature (valid or invalid)
--valid_channel_signature : (bool) claims with a valid channel signature or no signature,
@@ -1815,7 +1815,7 @@ class Daemon(metaclass=JSONRPCServerType):
--not_locations=<not_locations> : (list) find claims not containing any of these locations
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--order_by=<order_by> : (str) field to order by, default is descending order, to do an
--order_by=<order_by> : (list) field to order by, default is descending order, to do an
ascending order prepend ^ to the field name, eg. '^amount'
available fields: 'name', 'height', 'release_time',
'publish_time', 'amount', 'effective_amount',
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
]
}
"""
cmnt_list: dict = await jsonrpc_post(
result = await jsonrpc_post(
self.conf.comment_server,
"get_claim_comments",
claim_id=claim_id,
@@ -3425,20 +3425,25 @@ class Daemon(metaclass=JSONRPCServerType):
page_size=page_size,
top_level=not include_replies
)
if 'items' in cmnt_list:
items = cmnt_list['items']
chnl_uris = {cmnt['channel_url'] for cmnt in items if 'channel_url' in cmnt}
claims = await self.resolve(tuple(chnl_uris))
for cmnt in cmnt_list['items']:
if 'channel_url' in cmnt:
if cmnt['channel_url'] in claims:
channel = claims[cmnt['channel_url']]
cmnt['is_channel_signature_valid'] = is_comment_signed_by_channel(cmnt, channel)
continue
cmnt['is_channel_signature_valid'] = False
if is_channel_signature_valid:
cmnt_list['items'] = [c for c in cmnt_list['items'] if c['is_channel_signature_valid']]
return cmnt_list
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
comment['is_channel_signature_valid'] = False
if is_channel_signature_valid:
result['items'] = [
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
]
return result
async def jsonrpc_comment_create(self, claim_id, comment, parent_id=None, channel_account_id=None,
channel_name=None, channel_id=None) -> dict:
+7 -2
View File
@@ -379,7 +379,9 @@ class Language(Metadata):
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2:
if parts and len(parts[0]) == 2 and parts[0].isalpha():
self.region = parts.pop(0)
if parts and len(parts[0]) == 3 and parts[0].isdigit():
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@@ -404,10 +406,13 @@ class Language(Metadata):
@property
def region(self) -> str:
if self.message.region:
return LocationMessage.Country.Name(self.message.region)
r = LocationMessage.Country.Name(self.message.region)
return r[1:] if r.startswith('R') else r
@region.setter
def region(self, region: str):
if len(region) == 3:
region = 'R'+region
self.message.region = LocationMessage.Country.Value(region)
+2 -2
View File
@@ -6,8 +6,8 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
def clean_tags(tags: List[str]):
return [tag for tag in (normalize_tag(tag) for tag in tags) if tag]
return [tag for tag in set(normalize_tag(tag) for tag in tags) if tag]
File diff suppressed because it is too large. Load diff
+7 -16
View File
@@ -56,7 +56,6 @@ class ManagedStream:
'stream_claim_info',
'download_id',
'rowid',
'written_bytes',
'content_fee',
'downloader',
'analytics_manager',
@@ -89,7 +88,6 @@ class ManagedStream:
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.written_bytes = 0
self.content_fee = content_fee
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
@@ -121,6 +119,10 @@ class ManagedStream:
def status(self) -> str:
return self._status
@property
def written_bytes(self) -> int:
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
@@ -205,21 +207,13 @@ class ManagedStream:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if self.full_path and self.output_file_exists:
if self.written_bytes:
written_bytes = self.written_bytes
else:
written_bytes = os.stat(self.full_path).st_size
else:
if not self.output_file_exists:
full_path = None
file_name = None
download_directory = None
written_bytes = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'completed': self.written_bytes >= self.descriptor.lower_bound_decrypted_length(),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
@@ -233,7 +227,7 @@ class ManagedStream:
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
'written_bytes': written_bytes,
'written_bytes': self.written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
@@ -371,7 +365,6 @@ class ManagedStream:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
@@ -387,7 +380,6 @@ class ManagedStream:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
self.written_bytes = 0
if isinstance(err, asyncio.TimeoutError):
self.downloader.stop()
await self.blob_manager.storage.change_file_download_dir_and_file_name(
@@ -425,7 +417,6 @@ class ManagedStream:
self.stream_hash, self.download_directory, self.file_name
)
await self.update_status(ManagedStream.STATUS_RUNNING)
self.written_bytes = 0
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
await self.started_writing.wait()
+2
View File
@@ -214,7 +214,9 @@ class CommandTestCase(IntegrationTestCase):
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
height = self.ledger.headers.height
await self.generate(1)
await self.on_header(height + 1)
await self.on_transaction_dict(claim)
return claim
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+46 -2
View File
@@ -3,10 +3,11 @@ import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL
from lbry.schema.url import URL, normalize_name
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
@@ -72,7 +73,50 @@ class MainNetLedger(BaseLedger):
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return result
return await self.validate_resolutions(result)
async def validate_resolutions(self, resolutions: dict):
provables = {}
for url, txo in resolutions.items():
if isinstance(txo, dict) and 'error' in txo:
continue
parsed_url = URL.parse(url)
# cases we check: names without sequence, claim id or any modifier
# except: @channel/name as the signature is enough, instead we check '@channel'
if parsed_url.has_channel:
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
if not provables:
return resolutions
root_hash = self.headers.claim_trie_root.decode()
names = list(provables.keys())
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
for proof, name in zip(proofs, names):
all_failed = True
for url, txid in provables[name]:
if not proof.get('txhash'):
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
continue
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
if txid != expected_txid:
resolutions[url] = {
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
}
else:
all_failed = False
if not all_failed:
try:
verify_proof(proof, root_hash, name)
continue
except InvalidProofError as error:
for url, _ in provables[name]:
resolution = resolutions[url]
if isinstance(resolution, dict) and 'error' in resolution:
continue
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
return resolutions
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
+3
View File
@@ -9,5 +9,8 @@ class Network(BaseNetwork):
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def get_name_proofs(self, block_hash, *names):
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
@@ -68,6 +68,7 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.search_cache = {}
def advance_blocks(self, blocks):
self.sql.begin()
@@ -78,6 +79,12 @@ class LBRYBlockProcessor(BlockProcessor):
raise
finally:
self.sql.commit()
if self.db.first_sync and self.height == self.daemon.cached_height():
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
+207
View File
@@ -11,3 +11,210 @@ STREAM_TYPES = {
'binary': 5,
'model': 6
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
+208 -53
View File
@@ -1,17 +1,34 @@
import sqlite3
import time
import struct
from typing import Tuple, List
import sqlite3
import logging
from typing import Tuple, List, Dict, Union, Type, Optional
from binascii import unhexlify
from decimal import Decimal
from contextvars import ContextVar
from functools import wraps
from dataclasses import dataclass
from torba.client.basedatabase import query
from torba.client.basedatabase import query, interpolate
from lbry.schema.url import URL, normalize_name
from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
class SQLiteOperationalError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query errored')
self.metrics = metrics
class SQLiteInterruptedError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query interrupted')
self.metrics = metrics
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -39,7 +56,7 @@ SEARCH_PARAMS = {
ORDER_FIELDS = {
'name',
'name', 'claim_hash'
} | INTEGER_PARAMS
@@ -48,25 +65,119 @@ PRAGMAS = """
"""
db = ContextVar('db')
ledger = ContextVar('ledger')
@dataclass
class ReaderState:
db: sqlite3.Connection
stack: List[List]
metrics: Dict
is_tracking_metrics: bool
ledger: Type[BaseLedger]
query_timeout: float
log: logging.Logger
def close(self):
self.db.close()
def reset_metrics(self):
self.stack = []
self.metrics = {}
def set_query_timeout(self):
stop_at = time.perf_counter() + self.query_timeout
def interruptor():
if time.perf_counter() >= stop_at:
self.db.interrupt()
return
self.db.set_progress_handler(interruptor, 100)
def initializer(_path, _ledger_name):
_db = sqlite3.connect(_path, isolation_level=None, uri=True)
_db.row_factory = sqlite3.Row
db.set(_db)
from lbry.wallet.ledger import MainNetLedger, RegTestLedger
ledger.set(MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
db = sqlite3.connect(_path, isolation_level=None, uri=True)
db.row_factory = sqlite3.Row
ctx.set(
ReaderState(
db=db, stack=[], metrics={}, is_tracking_metrics=_measure,
ledger=MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger,
query_timeout=query_timeout, log=log
)
)
def cleanup():
db.get().close()
db.set(None)
ledger.set(None)
ctx.get().close()
ctx.set(None)
def get_claims(cols, for_count=False, **constraints):
def measure(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
metric = {}
state.metrics.setdefault(func.__name__, []).append(metric)
state.stack.append([])
start = time.perf_counter()
try:
return func(*args, **kwargs)
finally:
elapsed = int((time.perf_counter()-start)*1000)
metric['total'] = elapsed
metric['isolated'] = (elapsed-sum(state.stack.pop()))
if state.stack:
state.stack[-1].append(elapsed)
return wrapper
def reports_metrics(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
state.reset_metrics()
r = func(*args, **kwargs)
return r, state.metrics
return wrapper
@reports_metrics
def search_to_bytes(constraints) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(search(constraints))
@reports_metrics
def resolve_to_bytes(urls) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(resolve(urls))
def encode_result(result):
return Outputs.to_bytes(*result)
@measure
def execute_query(sql, values) -> List:
context = ctx.get()
context.set_query_timeout()
try:
return context.db.execute(sql, values).fetchall()
except sqlite3.OperationalError as err:
plain_sql = interpolate(sql, values)
if context.is_tracking_metrics:
context.metrics['execute_query'][-1]['sql'] = plain_sql
if str(err) == "interrupted":
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
raise SQLiteInterruptedError(context.metrics)
context.log.exception('failed running query', exc_info=err)
raise SQLiteOperationalError(context.metrics)
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'order_by' in constraints:
sql_order_by = []
for order_by in constraints['order_by']:
@@ -120,15 +231,7 @@ def get_claims(cols, for_count=False, **constraints):
if 'public_key_id' in constraints:
constraints['claim.public_key_hash'] = sqlite3.Binary(
ledger.get().address_to_hash160(constraints.pop('public_key_id')))
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = _resolve_one(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
if 'channel_hash' in constraints:
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_ids' in constraints:
@@ -197,10 +300,23 @@ def get_claims(cols, for_count=False, **constraints):
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)
return db.get().execute(sql, values).fetchall()
return sql, values
def get_claims_count(**constraints):
def get_claims(cols, for_count=False, **constraints) -> List:
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
sql, values = _get_claims(cols, for_count, **constraints)
return execute_query(sql, values)
@measure
def get_claims_count(**constraints) -> int:
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -208,6 +324,7 @@ def get_claims_count(**constraints):
return count[0][0]
@measure
def search(constraints) -> Tuple[List, List, int, int]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
@@ -217,19 +334,17 @@ def search(constraints) -> Tuple[List, List, int, int]:
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["height", "^name"]
constraints['order_by'] = ["claim_hash"]
txo_rows = _search(**constraints)
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
return txo_rows, extra_txo_rows, constraints['offset'], total
def search_to_bytes(constraints) -> bytes:
return Outputs.to_bytes(*search(constraints))
def _search(**constraints):
return get_claims(
"""
@@ -249,25 +364,26 @@ def _search(**constraints):
)
@measure
def resolve(urls) -> Tuple[List, List]:
result = []
channel_hashes = set()
for raw_url in urls:
match = _resolve_one(raw_url)
match = resolve_url(raw_url)
result.append(match)
if isinstance(match, sqlite3.Row) and match['channel_hash']:
channel_hashes.add(match['channel_hash'])
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
return result, extra_txo_rows
def resolve_to_bytes(urls) -> bytes:
return Outputs.to_bytes(*resolve(urls))
@measure
def resolve_url(raw_url):
def _resolve_one(raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
@@ -292,7 +408,7 @@ def _resolve_one(raw_url):
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount']
query['order_by'] = ['effective_amount', '^height']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
@@ -309,28 +425,68 @@ def _resolve_one(raw_url):
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
not_items = set(cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
if attr == 'tag':
common_tags = any_items & COMMON_TAGS.keys()
if common_tags:
any_items -= common_tags
if len(common_tags) < 5:
for item in common_tags:
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
elif len(common_tags) >= 5:
constraints.update({
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
})
values = ', '.join(
f':$any_common_tag{i}' for i in range(len(common_tags))
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count:
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
constraints[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
all_items = cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
@@ -353,7 +509,6 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
)
"""
not_items = cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
if not_items:
constraints.update({
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
+40 -32
View File
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
)
from .common import CLAIM_TYPES, STREAM_TYPES
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -76,33 +76,12 @@ class SQLDB:
trending_global integer not null default 0
);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_release_time_idx on claim (release_time);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_height_idx on claim (height);
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
create index if not exists claim_effective_amount_idx on claim (effective_amount);
create index if not exists claim_trending_group_idx on claim (trending_group);
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
create index if not exists claim_trending_local_idx on claim (trending_local);
create index if not exists claim_trending_global_idx on claim (trending_global);
"""
CREATE_SUPPORT_TABLE = """
@@ -113,7 +92,6 @@ class SQLDB:
claim_hash bytes not null,
amount integer not null
);
create index if not exists support_txo_hash_idx on support (txo_hash);
create index if not exists support_claim_hash_idx on support (claim_hash, height);
"""
@@ -123,9 +101,7 @@ class SQLDB:
claim_hash bytes not null,
height integer not null
);
create index if not exists tag_tag_idx on tag (tag);
create index if not exists tag_claim_hash_idx on tag (claim_hash);
create index if not exists tag_height_idx on tag (height);
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_CLAIMTRIE_TABLE = """
@@ -137,6 +113,38 @@ class SQLDB:
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
TAG_INDEXES = '\n'.join(
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
for tag_value, tag_key in COMMON_TAGS.items()
)
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_CLAIM_TABLE +
@@ -200,7 +208,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags = [], [], []
claim_hashes, claims, tags = [], [], {}
for txo in txos:
tx = txo.tx_ref.tx
@@ -254,14 +262,14 @@ class SQLDB:
claim_record['claim_type'] = CLAIM_TYPES['channel']
for tag in clean_tags(claim.message.tags):
tags.append((tag, claim_hash, tx.height))
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
if clear_first:
self._clear_claim_metadata(claim_hashes)
if tags:
self.db.executemany(
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
return claims
+135
View File
@@ -0,0 +1,135 @@
import time
import math
from typing import Tuple
def calculate_elapsed(start) -> int:
return int((time.perf_counter() - start) * 1000)
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
if not data:
return 0, 0, 0, 0, 0, 0, 0, 0
data.sort()
size = len(data)
return (
int(sum(data) / size),
data[0],
data[math.ceil(size * .05) - 1],
data[math.ceil(size * .25) - 1],
data[math.ceil(size * .50) - 1],
data[math.ceil(size * .75) - 1],
data[math.ceil(size * .95) - 1],
data[-1]
)
def remove_select_list(sql) -> str:
return sql[sql.index('FROM'):]
class APICallMetrics:
def __init__(self, name):
self.name = name
# total requests received
self.receive_count = 0
self.cache_response_count = 0
# millisecond timings for query based responses
self.query_response_times = []
self.query_intrp_times = []
self.query_error_times = []
self.query_python_times = []
self.query_wait_times = []
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
self.individual_sql_times = [] # every SQL query run on server
# actual queries
self.errored_queries = set()
self.interrupted_queries = set()
def to_json_and_reset(self):
return {
# total requests received
"receive_count": self.receive_count,
# sum of these is total responses made
"cache_response_count": self.cache_response_count,
"query_response_count": len(self.query_response_times),
"intrp_response_count": len(self.query_intrp_times),
"error_response_count": len(self.query_error_times),
# millisecond timings for non-cache responses
"response": calculate_avg_percentiles(self.query_response_times),
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
"error": calculate_avg_percentiles(self.query_error_times),
# response, interrupt and error each also report the python, wait and sql stats:
"python": calculate_avg_percentiles(self.query_python_times),
"wait": calculate_avg_percentiles(self.query_wait_times),
"sql": calculate_avg_percentiles(self.query_sql_times),
# extended timings for individual sql executions
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
"individual_sql_count": len(self.individual_sql_times),
# actual queries
"errored_queries": list(self.errored_queries),
"interrupted_queries": list(self.interrupted_queries),
}
def start(self):
self.receive_count += 1
def cache_response(self):
self.cache_response_count += 1
def _add_query_timings(self, request_total_time, metrics):
if metrics and 'execute_query' in metrics:
sub_process_total = metrics[self.name][0]['total']
individual_query_times = [f['total'] for f in metrics['execute_query']]
aggregated_query_time = sum(individual_query_times)
self.individual_sql_times.extend(individual_query_times)
self.query_sql_times.append(aggregated_query_time)
self.query_python_times.append(sub_process_total - aggregated_query_time)
self.query_wait_times.append(request_total_time - sub_process_total)
@staticmethod
def _add_queries(query_set, metrics):
if metrics and 'execute_query' in metrics:
for execute_query in metrics['execute_query']:
if 'sql' in execute_query:
query_set.add(remove_select_list(execute_query['sql']))
def query_response(self, start, metrics):
self.query_response_times.append(calculate_elapsed(start))
self._add_query_timings(self.query_response_times[-1], metrics)
def query_interrupt(self, start, metrics):
self.query_intrp_times.append(calculate_elapsed(start))
self._add_queries(self.interrupted_queries, metrics)
self._add_query_timings(self.query_intrp_times[-1], metrics)
def query_error(self, start, metrics):
self.query_error_times.append(calculate_elapsed(start))
self._add_queries(self.errored_queries, metrics)
self._add_query_timings(self.query_error_times[-1], metrics)
class ServerLoadData:
def __init__(self):
self._apis = {}
def for_api(self, name) -> APICallMetrics:
if name not in self._apis:
self._apis[name] = APICallMetrics(name)
return self._apis[name]
def to_json_and_reset(self, status):
try:
return {
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
'status': status
}
finally:
self._apis = {}
+110 -11
View File
@@ -1,17 +1,40 @@
import os
import math
import time
import base64
import asyncio
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from torba.rpc.jsonrpc import RPCError
from torba.rpc.jsonrpc import RPCError, JSONRPC
from torba.server.session import ElectrumX, SessionManager
from torba.server import util
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYDB
from lbry.wallet.server.db import reader
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
class ResultCacheItem:
__slots__ = '_result', 'lock', 'has_result'
def __init__(self):
self.has_result = asyncio.Event()
self.lock = asyncio.Lock()
self._result = None
@property
def result(self) -> str:
return self._result
@result.setter
def result(self, result: str):
self._result = result
if result is not None:
self.has_result.set()
class LBRYSessionManager(SessionManager):
@@ -19,23 +42,54 @@ class LBRYSessionManager(SessionManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.query_executor = None
self.websocket = None
self.metrics = ServerLoadData()
self.metrics_loop = None
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
async def process_metrics(self):
while self.running:
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
if self.websocket is not None:
self.websocket.send_message(data)
await asyncio.sleep(1)
async def start_other(self):
args = dict(initializer=reader.initializer, initargs=('claims.db', self.env.coin.NET))
self.running = True
args = dict(
initializer=reader.initializer,
initargs=(self.logger, 'claims.db', self.env.coin.NET, self.env.database_query_timeout,
self.env.track_metrics)
)
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
self.query_executor = ThreadPoolExecutor(max_workers=1, **args)
else:
self.query_executor = ProcessPoolExecutor(
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args
)
if self.websocket is not None:
await self.websocket.start()
if self.env.track_metrics:
self.metrics_loop = asyncio.create_task(self.process_metrics())
async def stop_other(self):
self.running = False
if self.env.track_metrics:
self.metrics_loop.cancel()
if self.websocket is not None:
await self.websocket.stop()
self.query_executor.shutdown()
class LBRYElectrumX(ElectrumX):
PROTOCOL_MIN = (0, 0) # temporary, for supporting 0.10 protocol
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -50,26 +104,71 @@ class LBRYElectrumX(ElectrumX):
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': self.get_server_height,
}
self.request_handlers.update(handlers)
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
start = time.perf_counter()
try:
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
except reader.SQLiteInterruptedError as error:
metrics.query_interrupt(start, error.metrics)
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics.query_error(start, error.metrics)
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except:
metrics.query_error(start, {})
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
if self.env.track_metrics:
(result, metrics_data) = result
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
async def run_and_cache_query(self, query_name, function, kwargs):
if self.env.track_metrics:
metrics = self.session_mgr.metrics.for_api(query_name)
else:
metrics = APICallMetrics(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
return cache_item.result
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
metrics, function, kwargs
)
else:
metrics.cache_response()
return cache_item.result
async def claimtrie_search(self, **kwargs):
if 'claim_id' in kwargs:
self.assert_claim_id(kwargs['claim_id'])
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, reader.search_to_bytes, kwargs
)).decode()
if kwargs:
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
async def claimtrie_resolve(self, *urls):
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, reader.resolve_to_bytes, urls
)).decode()
if urls:
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
async def get_server_height(self):
return self.bp.height
def claimtrie_getnameproofs(self, block_hash, *names):
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
async def transaction_get_height(self, tx_hash):
self.assert_tx_hash(tx_hash)
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
+55
View File
@@ -0,0 +1,55 @@
import asyncio
from weakref import WeakSet
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
+146
View File
@@ -0,0 +1,146 @@
import os
import time
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
DEFAULT_ANY_TAGS = [
'blockchain',
'news',
'learning',
'technology',
'automotive',
'economics',
'food',
'science',
'art',
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
'mature',
'xxx'
]
ORDER_BY = [
[
"trending_global",
"trending_mixed",
],
[
"release_time"
],
[
"effective_amount"
]
]
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
args.append(kw)
print(len(args), "argument combinations")
return args
def _search(kwargs):
start = time.time()
try:
search_to_bytes(kwargs)
t = time.time() - start
return t, kwargs
except Exception as err:
return -1, f"failed: error={str(type(err))}({str(err)})"
async def search(executor, kwargs):
try:
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
except Exception as err:
return f"failed (err={str(type(err))}({err}))- {kwargs}"
async def main(db_path, max_query_time):
args = dict(initializer=initializer, initargs=(log, db_path, MainNetLedger, 0.25))
workers = max(os.cpu_count(), 4)
log.info(f"using {workers} reader processes")
query_executor = ProcessPoolExecutor(workers, **args)
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
for ts, constraints in results:
if ts >= max_query_time:
sql = interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints))
print(f"Query took {int(ts * 1000)}ms\n{sql}")
finally:
query_executor.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
asyncio.run(main(db_path, max_query_time))
+107
View File
@@ -0,0 +1,107 @@
import time
import asyncio
import random
from argparse import ArgumentParser
from torba.client.basenetwork import ClientSession
class AgentSmith(ClientSession):
async def do_nefarious_things(self):
await self.send_request('blockchain.claimtrie.search', {
'no_totals': True,
'offset': random.choice(range(0, 300, 20)),
'limit': 20,
'any_tags': (
random.choice([[
random.choice(['gaming', 'games', 'game']) +
random.choice(['entertainment', 'playthrough', 'funny']) +
random.choice(['xbox', 'xbox one', 'xbox news'])
], [
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
random.choice(['news', 'sighting', 'sightings'])
], [
random.choice(['art', 'automotive']),
random.choice(['blockchain', 'economics', 'food']),
random.choice(['funny', 'learnings', 'nature']),
random.choice(['news', 'science', 'technology'])
]
])
),
'not_tags': random.choice([[], [
'porn', 'mature', 'xxx', 'nsfw'
]]),
'order_by': random.choice([
['release_time'],
['trending_global', 'trending_mixed'],
['effective_amount']
])
})
class AgentSmithProgram:
def __init__(self, host, port):
self.host, self.port = host, port
self.agent_smiths = []
async def make_one_more_of_them(self):
smith = AgentSmith(network=None, server=(self.host, self.port))
await smith.create_connection()
self.agent_smiths.append(smith)
async def coordinate_nefarious_activity(self):
start = time.perf_counter()
await asyncio.gather(
*(s.do_nefarious_things() for s in self.agent_smiths),
return_exceptions=True
)
return time.perf_counter() - start
def __len__(self):
return len(self.agent_smiths)
async def delete_one_smith(self):
if self.agent_smiths:
await self.agent_smiths.pop().close()
async def delete_program(self):
await asyncio.gather(*(
s.close() for s in self.agent_smiths
))
async def main(host, port):
smiths = AgentSmithProgram(host, port)
await smiths.make_one_more_of_them()
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
ease_off = 0
for i in range(1000):
await asyncio.sleep(1)
if activity.done() and activity.result() < .9:
print('more, more, more...')
await asyncio.gather(*(
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
))
else:
print('!!!!!!!!!!!!!!')
print('IS NEO LOSING?')
print('!!!!!!!!!!!!!!')
await asyncio.gather(*(
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
))
print(f'coordinate all {len(smiths)} smiths to action')
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
print('finishing up any remaining actions')
await activity
print('neo has won, deleting agents...')
await smiths.delete_program()
print('done.')
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--host', dest='host', default='localhost', type=str)
parser.add_argument('--port', dest='port', default=50001, type=int)
args = parser.parse_args()
asyncio.run(main(args.host, args.port))
+32 -61
View File
@@ -74,28 +74,19 @@ def get_release_text(desc: str):
yield line[len(RELEASE_TEXT):]
def get_previous_final(repo, current_release):
assert current_release.rc is not None, "Need an rc to find the previous final release."
previous = None
for tag in repo.tags(current_release.rc+1):
previous = tag
return previous
class Version:
def __init__(self, major=0, minor=0, micro=0, rc=None):
def __init__(self, major=0, minor=0, micro=0):
self.major = int(major)
self.minor = int(minor)
self.micro = int(micro)
self.rc = rc if rc is None else int(rc)
@classmethod
def from_string(cls, version_string):
(major, minor, micro), rc = version_string.split('.'), None
if 'rc' in micro:
micro, rc = micro.split('rc')
return cls(major, minor, micro, rc)
return cls(major, minor, micro)
@classmethod
def from_content(cls, content):
@@ -106,20 +97,12 @@ class Version:
def increment(self, action):
cls = self.__class__
if action == '*-rc':
assert self.rc is not None, f"Can't drop rc designation because {self} is already not an rc."
return cls(self.major, self.minor, self.micro)
elif action == '*+rc':
assert self.rc is not None, "Must already be an rc to increment."
return cls(self.major, self.minor, self.micro, self.rc+1)
assert self.rc is None, f"Can't start a new rc because {self} is already an rc."
if action == 'major+rc':
return cls(self.major+1, rc=1)
elif action == 'minor+rc':
return cls(self.major, self.minor+1, rc=1)
elif action == 'micro+rc':
return cls(self.major, self.minor, self.micro+1, 1)
if action == 'major':
return cls(self.major+1)
elif action == 'minor':
return cls(self.major, self.minor+1)
elif action == 'micro':
return cls(self.major, self.minor, self.micro+1)
raise ValueError(f'unknown action: {action}')
@@ -128,10 +111,7 @@ class Version:
return f'v{self}'
def __str__(self):
version = '.'.join(str(p) for p in [self.major, self.minor, self.micro])
if self.rc is not None:
version += f'rc{self.rc}'
return version
return '.'.join(str(p) for p in [self.major, self.minor, self.micro])
def release(args):
@@ -139,15 +119,15 @@ def release(args):
repo = gh.repository('lbryio', 'lbry-sdk')
version_file = repo.file_contents('lbry/lbry/__init__.py')
if not args.confirm:
print("\nDRY RUN ONLY. RUN WITH --confirm TO DO A REAL RELEASE.\n")
current_version = Version.from_content(version_file)
print(f'Current Version: {current_version}')
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
if args.action == '*-rc':
previous_release = repo.release_from_tag(args.start_tag or get_previous_final(repo, current_version))
else:
previous_release = repo.release_from_tag(current_version.tag)
previous_release = repo.release_from_tag(current_version.tag)
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
print()
@@ -165,7 +145,7 @@ def release(args):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body):
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
if not (args.action == '*-rc' and type_label == 'fixup'):
if type_label != 'fixup':
area = areas.setdefault(area_name, [])
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
else:
@@ -203,7 +183,7 @@ def release(args):
for skipped in unlabeled:
print(skipped)
if not args.dry_run:
if args.confirm:
commit = version_file.update(
new_version.tag,
@@ -223,9 +203,10 @@ def release(args):
name=new_version.tag,
body=body.getvalue(),
draft=True,
prerelease=new_version.rc is not None
)
return 0
class TestReleaseTool(unittest.TestCase):
@@ -235,44 +216,34 @@ class TestReleaseTool(unittest.TestCase):
def test_version_increment(self):
v = Version.from_string('1.2.3')
self.assertTrue(str(v.increment('major+rc')), '2.0.0rc1')
self.assertTrue(str(v.increment('minor+rc')), '1.3.0rc1')
self.assertTrue(str(v.increment('micro+rc')), '1.2.4rc1')
with self.assertRaisesRegex(AssertionError, "Must already be an rc to increment."):
v.increment('*+rc')
with self.assertRaisesRegex(AssertionError, "Can't drop rc designation"):
v.increment('*-rc')
v = Version.from_string('1.2.3rc3')
self.assertTrue(str(v.increment('*+rc')), '1.2.3rc4')
self.assertTrue(str(v.increment('*-rc')), '1.2.3')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('major+rc')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('minor+rc')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('micro+rc')
self.assertTrue(str(v.increment('major')), '2.0.0')
self.assertTrue(str(v.increment('minor')), '1.3.0')
self.assertTrue(str(v.increment('micro')), '1.2.4')
def test():
runner = unittest.TextTestRunner(verbosity=2)
loader = unittest.TestLoader()
suite = loader.loadTestsFromTestCase(TestReleaseTool)
runner.run(suite)
return 0 if runner.run(suite).wasSuccessful() else 1
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--test", default=False, action="store_true", help="run unit tests")
parser.add_argument("--dry-run", default=False, action="store_true", help="show what will be done")
parser.add_argument("--confirm", default=False, action="store_true",
help="without this flag, it will only print what it will do but will not actually do it")
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
parser.add_argument("action", nargs="?", choices=['major+rc', 'minor+rc', 'micro+rc', '*+rc', '*-rc'])
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
args = parser.parse_args()
if args.test:
test()
if args.action == "test":
code = test()
else:
release(args)
code = release(args)
print()
return code
if __name__ == "__main__":
main()
sys.exit(main())
+40 -61
View File
@@ -1,81 +1,60 @@
import uvloop, asyncio, sqlite3, time, sys
import uvloop, asyncio, time, sys, logging
from concurrent.futures import ProcessPoolExecutor
from contextvars import ContextVar
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
db_path = '/tmp/wallet-server/claims.db'
db = ContextVar('db')
def init():
conn = sqlite3.connect(db_path)
db.set(conn)
def reader():
conn = db.get()
start = time.time()
conn.execute("""
SELECT
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
FROM claim
LEFT JOIN claimtrie USING (claim_hash)
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
WHERE
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ('alexandria ocasio-cortez', 'Alien', 'alt news', 'art', 'audio',
'automotive', 'beliefs', 'blockchain', 'dog grooming', 'economics', 'food',
'learning', 'mature', 'nature', 'news', 'physics', 'science', 'technology')
)
AND NOT EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash AND tag IN ('nsfw', 'xxx', 'mature')
)
ORDER BY claim.height DESC, claim.normalized ASC
LIMIT 20 OFFSET 100
""").fetchall()
elapsed = time.time() - start
return elapsed
db_path = '../../../lbryconf/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
async def run_times(executor, iterations, show=True):
start = time.time()
timings = await asyncio.gather(*(
asyncio.get_running_loop().run_in_executor(executor, reader) for _ in range(iterations)
))
total = time.time() - start
start = time.perf_counter()
timings = await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(
executor, reader.search_to_bytes, {
'no_totals': True,
'offset': 0,
'limit': 20,
'any_tags': [
'ufos', 'city fix'
],
'not_tags': [
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query'][0]['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total:.5f}ms total concurrent, {len(timings)*avg:.5f}ms total sequential (avg*runs)")
print(f" {total/len(timings):.5f}ms/query concurrent (total/runs)")
print(f" {avg:.5f}ms/query actual average (sum(queries)/runs)")
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t:.5f}ms')
sys.stdout.write(f' {t}ms')
print(' ...\n' if len(timings) > 10 else '\n')
async def main():
executor = ProcessPoolExecutor(4, initializer=init)
await run_times(executor, 4, show=False)
await run_times(executor, 1)
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
await run_times(executor, 2**9)
await run_times(executor, 2**11)
await run_times(executor, 2**13)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
@@ -89,6 +89,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim], **kwargs)
async def assertFindsClaims(self, claims, **kwargs):
kwargs.setdefault('order_by', ['height', '^name'])
results = await self.claim_search(**kwargs)
self.assertEqual(len(claims), len(results))
for claim, result in zip(claims, results):
@@ -165,15 +166,15 @@ class ClaimSearchCommand(ClaimTestCase):
await self.create_channel()
await self.create_lots_of_streams()
page = await self.claim_search(page_size=20, channel='@abc')
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams)
page = await self.claim_search(page_size=6, channel='@abc')
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[:6])
page = await self.claim_search(page=2, page_size=6, channel='@abc')
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[6:])
@@ -1,132 +1,31 @@
import asyncio
import socket
import time
import logging
from unittest.mock import Mock
from torba.testcase import IntegrationTestCase, Conductor
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import lbry.wallet
from lbry.schema.claim import Claim
from lbry.wallet.transaction import Transaction, Output
from lbry.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
log = logging.getLogger(__name__)
def wrap_callback_event(fn, callback):
def inner(*a, **kw):
callback()
return fn(*a, **kw)
return inner
class TestSessionBloat(IntegrationTestCase):
"""
ERROR:asyncio:Fatal read error on socket transport
protocol: <lbrynet.wallet.server.session.LBRYElectrumX object at 0x7f7e3bfcaf60>
transport: <_SelectorSocketTransport fd=3236 read=polling write=<idle, bufsize=0>>
Traceback (most recent call last):
File "/usr/lib/python3.7/asyncio/selector_events.py", line 801, in _read_ready__data_received
data = self._sock.recv(self.max_size)
TimeoutError: [Errno 110] Connection timed out
Tests that server cleans up stale connections after session timeout and client times out too.
"""
LEDGER = lbry.wallet
async def asyncSetUp(self):
self.conductor = Conductor(
ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY
)
await self.conductor.start_blockchain()
self.addCleanup(self.conductor.stop_blockchain)
await self.conductor.start_spv()
self.session_manager = self.conductor.spv_node.server.session_mgr
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64)
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 64)
self.addCleanup(self.conductor.stop_spv)
await self.conductor.start_wallet()
self.addCleanup(self.conductor.stop_wallet)
self.client_session = list(self.session_manager.sessions)[0]
self.client_session.transport.set_write_buffer_limits(0, 0)
self.paused_session = asyncio.Event(loop=self.loop)
self.resumed_session = asyncio.Event(loop=self.loop)
def paused():
self.resumed_session.clear()
self.paused_session.set()
def delayed_resume():
self.paused_session.clear()
time.sleep(1)
self.resumed_session.set()
self.client_session.pause_writing = wrap_callback_event(self.client_session.pause_writing, paused)
self.client_session.resume_writing = wrap_callback_event(self.client_session.resume_writing, delayed_resume)
self.blockchain = self.conductor.blockchain_node
self.wallet_node = self.conductor.wallet_node
self.manager = self.wallet_node.manager
self.ledger = self.wallet_node.ledger
self.wallet = self.wallet_node.wallet
self.account = self.wallet_node.wallet.default_account
async def test_session_bloat_from_socket_timeout(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
await self.blockchain.generate(1)
await asyncio.wait([
self.on_transaction_id(sendtxid1),
self.on_transaction_id(sendtxid2)
])
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.description = "0" * 8000
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
self.paused_session.clear()
self.resumed_session.clear()
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await asyncio.wait_for(self.paused_session.wait(), 2)
self.assertEqual(1, len(self.session_manager.sessions))
real_sock = self.client_session.transport._extra.pop('socket')
mock_sock = Mock(spec=socket.socket)
for attr in dir(real_sock):
if not attr.startswith('__'):
setattr(mock_sock, attr, getattr(real_sock, attr))
def recv(*a, **kw):
raise TimeoutError("[Errno 110] Connection timed out")
mock_sock.recv = recv
self.client_session.transport._sock = mock_sock
self.client_session.transport._extra['socket'] = mock_sock
self.assertFalse(self.resumed_session.is_set())
self.assertFalse(self.session_manager.session_event.is_set())
await self.session_manager.session_event.wait()
self.assertEqual(0, len(self.session_manager.sessions))
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
await self.conductor.start_spv()
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
await session.create_connection()
session.ping_task.cancel()
await session.send_request('server.banner', ())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 1)
self.assertFalse(session.is_closing())
await asyncio.sleep(1.1)
with self.assertRaises(asyncio.TimeoutError):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
+12
View File
@@ -17,6 +17,18 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
async def test_sync_blob_file_manager_on_startup(self):
await self.setup_blob_manager(save_blobs=True)
+7
View File
@@ -72,6 +72,13 @@ class TestLanguages(TestCase):
self.assertEqual(stream.languages[2].region, 'US')
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US'])
stream.languages.append('es-419')
self.assertEqual(stream.languages[3].langtag, 'es-419')
self.assertEqual(stream.languages[3].language, 'es')
self.assertEqual(stream.languages[3].script, None)
self.assertEqual(stream.languages[3].region, '419')
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US', 'es-419'])
stream = Stream()
stream.languages.extend(['en-Latn-US', 'es-ES', 'de-DE'])
self.assertEqual(stream.languages[0].language, 'en')
+5 -1
View File
@@ -11,9 +11,13 @@ class TestTagNormalization(unittest.TestCase):
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'Tag')
tag('tag', 'T\'ag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
def test_clean_tags(self):
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))
cleaned = clean_tags(['fOo', '!taG', 'FoO'])
self.assertIn('tag', cleaned)
self.assertIn('foo', cleaned)
self.assertEqual(len(cleaned), 2)
+10 -1
View File
@@ -40,6 +40,15 @@ class TestManagedStream(BlobExchangeTestBase):
self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir
)
async def test_status_file_completed(self):
await self._test_transfer_stream(10)
self.assertTrue(self.stream.output_file_exists)
self.assertTrue(self.stream.as_dict()['completed'])
with open(self.stream.full_path, 'w+b') as outfile:
outfile.truncate(1)
self.assertTrue(self.stream.output_file_exists)
self.assertFalse(self.stream.as_dict()['completed'])
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
await self.setup_stream(blob_count)
mock_node = mock.Mock(spec=Node)
@@ -52,7 +61,7 @@ class TestManagedStream(BlobExchangeTestBase):
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
await self.stream.finished_writing.wait()
await self.stream.finished_write_attempt.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
if stop_when_done:
await self.stream.stop()
@@ -0,0 +1,60 @@
import time
import unittest
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
class TestPercentileCalculation(unittest.TestCase):
def test_calculate_percentiles(self):
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
self.assertEqual(calculate_avg_percentiles(
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
class TestCollectingMetrics(unittest.TestCase):
def test_happy_path(self):
self.maxDiff = None
load = ServerLoadData()
search = load.for_api('search')
self.assertEqual(search.name, 'search')
search.start()
search.cache_response()
search.cache_response()
metrics = {
'search': [{'total': 40}],
'execute_query': [
{'total': 20},
{'total': 10}
]
}
for x in range(5):
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
metrics['execute_query'][0]['total'] = 10
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
search.query_interrupt(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.052, {})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
"receive_count": 1,
"cache_response_count": 2,
"query_response_count": 5,
"intrp_response_count": 1,
"error_response_count": 2,
"response": (53, 51, 51, 52, 53, 54, 55, 55),
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
"error": (51, 50, 50, 50, 50, 52, 52, 52),
"python": (12, 10, 10, 10, 10, 20, 20, 20),
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
"individual_sql_count": 14,
"errored_queries": ['FROM claim where something=1'],
"interrupted_queries": ['FROM claim where something=1'],
}}})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
+3 -1
View File
@@ -1,6 +1,7 @@
import unittest
import ecdsa
import hashlib
import logging
from binascii import hexlify
from torba.client.constants import COIN, NULL_HASH32
@@ -36,6 +37,7 @@ class OldWalletServerTransaction:
class TestSQLDB(unittest.TestCase):
query_timeout = 0.25
def setUp(self):
self.first_sync = False
@@ -44,7 +46,7 @@ class TestSQLDB(unittest.TestCase):
db_url = 'file:test_sqldb?mode=memory&cache=shared'
self.sql = writer.SQLDB(self, db_url)
self.addCleanup(self.sql.close)
reader.initializer(db_url, 'regtest')
reader.initializer(logging.getLogger(__name__), db_url, 'regtest', self.query_timeout)
self.addCleanup(reader.cleanup)
self.timer = Timer('BlockProcessor')
self.sql.open()
@@ -53,35 +53,45 @@ class ReconnectTests(IntegrationTestCase):
class ServerPickingTestCase(AsyncioTestCase):
async def _make_fake_server(self, latency=1.0, port=1337):
# local fake server with artificial latency
proto = RPCSession()
async def __handler(_):
await asyncio.sleep(latency)
return {'height': 1}
proto.handle_request = __handler
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
async def _make_fake_server(self, latency=1.0, port=1):
# local fake server with artificial latency
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
return {"height": 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def _make_bad_server(self, port=42420):
async def echo(reader, writer):
while True:
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
await self._make_fake_server(latency=1.5, port=1340),
await self._make_fake_server(latency=0.1, port=1337),
await self._make_fake_server(latency=1.0, port=1339),
await self._make_fake_server(latency=0.5, port=1338),
# fast but unhealthy, should be discarded
await self._make_bad_server(),
('localhost', 1),
('example.that.doesnt.resolve', 9000),
await self._make_fake_server(latency=1.2, port=1340),
await self._make_fake_server(latency=0.5, port=1337),
await self._make_fake_server(latency=0.7, port=1339),
],
'connect_timeout': 30
'connect_timeout': 3
})
network = BaseNetwork(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
await asyncio.wait_for(network.on_connected.first, timeout=3)
self.assertTrue(network.is_connected)
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
# ensure we are connected to all of them
self.assertEqual(len(network.session_pool.sessions), 4)
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
self.assertEqual(len(network.session_pool.sessions), 3)
+21 -2
View File
@@ -1,11 +1,10 @@
import unittest
import sqlite3
from functools import wraps
from torba.client.wallet import Wallet
from torba.client.constants import COIN
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
from torba.client.hash import sha256
from torba.testcase import AsyncioTestCase
@@ -167,6 +166,26 @@ class TestQueryBuilder(unittest.TestCase):
("select * from foo LIMIT 20 OFFSET 10", {})
)
def test_query_interpolation(self):
self.maxDiff = None
# tests that interpolation replaces longer keys first
self.assertEqual(
interpolate(*query(
"select * from foo",
a__not='b', b__in='select * from blah where c=:$c',
d__any={'one__like': 'o', 'two': 2},
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
ahash=memoryview(sha256(b'hello world')),
limit=10, order_by='b', **{'$c': 3})
),
"select * from foo WHERE a != 'b' AND "
"b IN (select * from blah where c=3) AND "
"(one LIKE 'o' OR two = 2) AND "
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
"ORDER BY b LIMIT 10",
)
class TestQueries(AsyncioTestCase):
+14
View File
@@ -1,6 +1,7 @@
import logging
import asyncio
from asyncio import wrap_future
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
@@ -191,6 +192,19 @@ def query(select, **constraints):
return ' '.join(sql), values
def interpolate(sql, values):
for k in sorted(values.keys(), reverse=True):
value = values[k]
if isinstance(value, memoryview):
value = f"X'{hexlify(bytes(value)).decode()}'"
elif isinstance(value, str):
value = f"'{value}'"
else:
value = str(value)
sql = sql.replace(f":{k}", value)
return sql
def rows_to_dict(rows, fields):
if rows:
return [dict(zip(fields, r)) for r in rows]
+20 -17
View File
@@ -170,23 +170,26 @@ class BaseHeaders:
async def repair(self):
previous_header_hash = fail = None
for height in range(self.height):
raw = self.get_raw_header(height)
header = self.deserialize(height, raw)
if height:
if header['prev_block_hash'] != previous_header_hash:
fail = True
else:
if self.hash_header(raw) != self.genesis_hash:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
return
previous_header_hash = self.hash_header(raw)
self.io.seek(0)
batch_size = 10000
for start_height in range(0, self.height, batch_size):
headers = self.io.read(self.header_size*batch_size)
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
if header['prev_block_hash'] != previous_header_hash:
fail = True
else:
if header_hash != self.genesis_hash:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
return
previous_header_hash = header_hash
@staticmethod
def get_proof_of_work(header_hash: bytes) -> ArithUint256:
+25 -12
View File
@@ -15,28 +15,36 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, **kwargs):
def __init__(self, *args, network, server, timeout=30, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
self.max_seconds_idle = 60
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.ping_task = None
async def send_request(self, method, args=()):
try:
return await super().send_request(method, args)
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
except RPCError as e:
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
raise e
except asyncio.TimeoutError:
self.abort()
raise
async def ping_forever(self):
# TODO: change to 'ping' on newer protocol (above 1.2)
while not self.is_closing():
if (time() - self.last_send) > self.max_seconds_idle:
await self.send_request('server.banner')
try:
await self.send_request('server.banner')
except:
self.abort()
raise
await asyncio.sleep(self.max_seconds_idle//3)
async def create_connection(self, timeout=6):
@@ -209,17 +217,24 @@ class SessionPool:
self.ensure_connection(session)
for session in self.sessions
], return_exceptions=True)
await asyncio.wait([asyncio.sleep(3), self._lost_master.wait()], return_when='FIRST_COMPLETED')
try:
await asyncio.wait_for(self._lost_master.wait(), timeout=3)
except asyncio.TimeoutError:
pass
self._lost_master.clear()
if not self.sessions:
self.sessions.extend(self._dead_servers)
self._dead_servers = []
async def ensure_connection(self, session):
if not session.is_closing():
return
self._dead_servers.append(session)
self.sessions.remove(session)
try:
return await session.create_connection(self.timeout)
if session.is_closing():
await session.create_connection(self.timeout)
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
self.sessions.append(session)
self._dead_servers.remove(session)
except asyncio.TimeoutError:
log.warning("Timeout connecting to %s:%d", *session.server)
except asyncio.CancelledError: # pylint: disable=try-except-raise
@@ -231,11 +246,9 @@ class SessionPool:
log.warning("Could not connect to %s:%d", *session.server)
else:
log.exception("Connecting to %s:%d raised an exception:", *session.server)
self._dead_servers.append(session)
self.sessions.remove(session)
async def get_online_sessions(self):
self._lost_master.set()
while not self.online:
await asyncio.sleep(0.1)
self._lost_master.set()
await asyncio.sleep(0.5)
return self.sessions
+10 -5
View File
@@ -195,6 +195,7 @@ class SPVNode:
self.server = None
self.hostname = 'localhost'
self.port = 50001 + 1 # avoid conflict with default daemon
self.session_timeout = 600
async def start(self, blockchain_node: 'BlockchainNode'):
self.data_path = tempfile.mkdtemp()
@@ -204,7 +205,9 @@ class SPVNode:
'REORG_LIMIT': '100',
'HOST': self.hostname,
'TCP_PORT': str(self.port),
'MAX_QUERY_WORKERS': '0'
'SESSION_TIMEOUT': str(self.session_timeout),
'MAX_QUERY_WORKERS': '0',
'INDIVIDUAL_TAG_INDEXES': '',
}
# TODO: don't use os.environ
os.environ.update(conf)
@@ -263,6 +266,7 @@ class BlockchainNode:
self.data_path = None
self.protocol = None
self.transport = None
self.blockchain_address = None
self._block_expected = 0
self.hostname = 'localhost'
self.peerport = 9246 + 2 # avoid conflict with default peer port
@@ -326,7 +330,7 @@ class BlockchainNode:
self.daemon_bin,
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
f'-port={self.peerport}'
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
)
self.log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(
@@ -361,9 +365,10 @@ class BlockchainNode:
self.log.info(out.decode().strip())
return out.decode().strip()
def generate(self, blocks):
async def generate(self, blocks, address=None):
self._block_expected += blocks
return self._cli_cmnd('generate', str(blocks))
address = await self.get_raw_change_address()
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
def invalidate_block(self, blockhash):
return self._cli_cmnd('invalidateblock', blockhash)
@@ -372,7 +377,7 @@ class BlockchainNode:
return self._cli_cmnd('getblockhash', str(block))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress')
return self._cli_cmnd('getrawchangeaddress', 'legacy')
async def get_balance(self):
return float(await self._cli_cmnd('getbalance'))
+2
View File
@@ -165,6 +165,8 @@ class JSONRPC(object):
METHOD_NOT_FOUND = -32601
INVALID_ARGS = -32602
INTERNAL_ERROR = -32603
QUERY_TIMEOUT = -32000
# Codes specific to this library
ERROR_CODE_UNAVAILABLE = -100
+2
View File
@@ -98,6 +98,8 @@ class Daemon:
return aiohttp.ClientSession(connector=self.connector, connector_owner=False)
async def _send_data(self, data):
if not self.connector:
raise asyncio.CancelledError('Tried to send request during shutdown.')
async with self.workqueue_semaphore:
async with self.client_session() as session:
async with session.post(self.current_url(), data=data) as resp:
+5 -1
View File
@@ -38,6 +38,10 @@ class Env:
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
self.daemon_url = self.required('DAEMON_URL')
if coin is not None:
assert issubclass(coin, Coin)
@@ -47,7 +51,6 @@ class Env:
network = self.default('NET', 'mainnet').strip()
self.coin = Coin.lookup_coin_class(coin_name, network)
self.cache_MB = self.integer('CACHE_MB', 1200)
self.host = self.default('HOST', 'localhost')
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
# Server stuff
self.tcp_port = self.integer('TCP_PORT', None)
@@ -84,6 +87,7 @@ class Env:
self.identities = [identity
for identity in (clearnet_identity, tor_identity)
if identity is not None]
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 250)) / 1000.0
@classmethod
def default(cls, envvar, default):
-33
View File
@@ -5,38 +5,6 @@ from concurrent.futures.thread import ThreadPoolExecutor
import torba
from torba.server.mempool import MemPool, MemPoolAPI
from torba.server.session import SessionManager, SessionBase
CONNECTION_TIMED_OUT = 110
NO_ROUTE_TO_HOST = 113
def handle_socket_errors(socket_errors=(CONNECTION_TIMED_OUT, NO_ROUTE_TO_HOST)):
err_msgs = tuple((f"[Errno {err_code}]" for err_code in socket_errors))
log = logging.getLogger(__name__)
def protocol_exception_handler(loop, context):
exception = context['exception']
if 'protocol' not in context or 'transport' not in context:
raise exception
if not isinstance(context['protocol'], SessionBase):
raise exception
session: SessionBase = context['protocol']
transport: asyncio.Transport = context['transport']
message = context['message']
if message not in ("Fatal read error on socket transport", "Fatal write error on socket transport"):
raise exception
for err_msg in err_msgs:
if str(exception).startswith(err_msg):
log.debug("caught: '%s' for %s", str(exception), session)
transport.abort()
transport.close()
loop.create_task(session.close(force_after=1))
return
raise exception
return protocol_exception_handler
class Notifications:
@@ -121,7 +89,6 @@ class Server:
)
async def start(self):
asyncio.get_event_loop().set_exception_handler(handle_socket_errors())
env = self.env
min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
self.log.info(f'software version: {torba.__version__}')
+4 -3
View File
@@ -255,9 +255,10 @@ class SessionManager:
async def _clear_stale_sessions(self):
"""Cut off sessions that haven't done anything for 10 minutes."""
session_timeout = self.env.session_timeout
while True:
await sleep(60)
stale_cutoff = time.time() - self.env.session_timeout
await sleep(session_timeout // 10)
stale_cutoff = time.time() - session_timeout
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions:
@@ -267,7 +268,7 @@ class SessionManager:
# Give the sockets some time to close gracefully
if stale_sessions:
await asyncio.wait([
session.close() for session in stale_sessions
session.close(force_after=session_timeout // 10) for session in stale_sessions
])
# Consolidate small groups