Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a357cb6b26 | ||
|
|
5ba2190ca7 | ||
|
|
0bd3c3f503 | ||
|
|
489da88e79 |
No files matched your search
+1
-2
@@ -78,8 +78,7 @@ jobs:
|
||||
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key:
|
||||
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
|
||||
@@ -9,10 +9,6 @@ lint:
|
||||
cd torba && pylint --rcfile=setup.cfg torba
|
||||
cd torba && mypy --ignore-missing-imports torba
|
||||
|
||||
test:
|
||||
cd lbry && tox
|
||||
cd torba && tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r lbry/scripts/idea/* .idea
|
||||
@@ -1,6 +1,5 @@
|
||||
import 'package:lbry/src/load_generator.dart' as load;
|
||||
//import 'package:lbry/src/server_connection.dart' as load;
|
||||
|
||||
main(List<String> arguments) {
|
||||
load.generate_load();
|
||||
load.cli();
|
||||
}
|
||||
@@ -1,2 +1 @@
|
||||
export 'src/load_generator.dart';
|
||||
export 'src/server_connection.dart';
|
||||
export 'src/load_generator.dart' show LoadGenerator, LoadDataPoint;
|
||||
@@ -9,7 +9,6 @@ class LoadRequest {
|
||||
final Map payload;
|
||||
Completer completer;
|
||||
Stopwatch timer;
|
||||
Socket socket;
|
||||
|
||||
bool get isDone => completer.isCompleted;
|
||||
int get elapsed => timer.elapsedMilliseconds;
|
||||
@@ -22,7 +21,6 @@ class LoadRequest {
|
||||
completer.future.whenComplete(() => timer.stop());
|
||||
try {
|
||||
Socket.connect(this.host, this.port).then((socket) {
|
||||
this.socket = socket;
|
||||
utf8.decoder.bind(socket).listen((r) {
|
||||
if (r.contains('"jsonrpc": "2.0", "result": ')) {
|
||||
socket.close();
|
||||
@@ -46,33 +44,24 @@ class LoadRequest {
|
||||
}
|
||||
}
|
||||
|
||||
typedef bool LoadTickCallback(ClientLoadGenerator load_generator, ClientLoadDataPoint stats);
|
||||
typedef bool LoadTestCallback(LoadGenerator load_generator, LoadDataPoint stats);
|
||||
|
||||
class ClientLoadGenerator {
|
||||
Timer _timer;
|
||||
|
||||
final String host;
|
||||
final int port;
|
||||
final LoadTickCallback tickCallback;
|
||||
class LoadGenerator {
|
||||
int load = 1;
|
||||
Timer _timer;
|
||||
String host;
|
||||
int port;
|
||||
Map query;
|
||||
List<LoadRequest> backlog = [];
|
||||
|
||||
ClientLoadGenerator(this.host, this.port, {this.tickCallback, this.query}) {
|
||||
if (query == null) {
|
||||
query = {
|
||||
'id': 1,
|
||||
'method': 'blockchain.claimtrie.resolve',
|
||||
'params': ['one', 'two', 'three']
|
||||
};
|
||||
}
|
||||
}
|
||||
LoadTestCallback cb;
|
||||
|
||||
LoadGenerator(this.host, this.port, this.query, this.cb);
|
||||
|
||||
start() {
|
||||
var previous = spawn_requests();
|
||||
backlog = <LoadRequest>[];
|
||||
var backlog = <LoadRequest>[];
|
||||
_timer = Timer.periodic(Duration(seconds: 1), (t) {
|
||||
var stat = ClientLoadDataPoint(t.tick);
|
||||
var stat = LoadDataPoint();
|
||||
backlog.removeWhere((r) {
|
||||
if (r.isDone) stat.addCatchup(r);
|
||||
return r.isDone;
|
||||
@@ -86,20 +75,16 @@ class ClientLoadGenerator {
|
||||
}
|
||||
stat.backlog = backlog.length;
|
||||
stat.load = load;
|
||||
stat.close();
|
||||
if (tickCallback(this, stat)) {
|
||||
if (cb(this, stat)) {
|
||||
previous = spawn_requests();
|
||||
} else {
|
||||
stop();
|
||||
t.cancel();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
_timer.cancel();
|
||||
for (var remaining in backlog) {
|
||||
remaining?.socket?.close();
|
||||
}
|
||||
}
|
||||
|
||||
List<LoadRequest> spawn_requests() {
|
||||
@@ -112,20 +97,18 @@ class ClientLoadGenerator {
|
||||
|
||||
}
|
||||
|
||||
class ClientLoadDataPoint {
|
||||
final int tick;
|
||||
class LoadDataPoint {
|
||||
final DateTime time = new DateTime.now();
|
||||
int success = 0;
|
||||
int errored = 0;
|
||||
int backlog = 0;
|
||||
int catchup = 0;
|
||||
int _success_total = 0;
|
||||
int avg_success = 0;
|
||||
int _catchup_total = 0;
|
||||
int avg_catchup = 0;
|
||||
int load = 0;
|
||||
|
||||
ClientLoadDataPoint(this.tick);
|
||||
ClientLoadDataPoint.empty(): this(0);
|
||||
int get avg_success => _success_total > 0 ? (_success_total/success).round() : 0;
|
||||
int get avg_catchup => _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
|
||||
|
||||
addSuccess(LoadRequest r) {
|
||||
success++; _success_total += r.elapsed;
|
||||
@@ -134,19 +117,16 @@ class ClientLoadDataPoint {
|
||||
addCatchup(LoadRequest r) {
|
||||
catchup++; _catchup_total += r.elapsed;
|
||||
}
|
||||
|
||||
close() {
|
||||
avg_success = _success_total > 0 ? (_success_total/success).round() : 0;
|
||||
avg_catchup = _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
generate_load() {
|
||||
cli() {
|
||||
var runs = 1;
|
||||
ClientLoadGenerator('localhost', 50001, tickCallback: (t, stats) {
|
||||
print("run ${runs}: ${stats.load} load, ${stats.success} success, ${stats.backlog} backlog");
|
||||
LoadGenerator('localhost', 50001, {
|
||||
'id': 1,
|
||||
'method': 'blockchain.claimtrie.resolve',
|
||||
'params': ['one', 'two', 'three']
|
||||
}, (t, stats) {
|
||||
print("run ${runs}: ${stats}");
|
||||
t.load = (runs < 4 ? t.load*2 : t.load/2).round();
|
||||
return runs++ < 10;
|
||||
}).start();
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:web_socket_channel/io.dart';
|
||||
import 'package:web_socket_channel/status.dart' as status;
|
||||
|
||||
|
||||
class ServerConnection {
|
||||
String url = "";
|
||||
IOWebSocketChannel channel;
|
||||
bool get isConnected => channel != null && channel.closeCode == null;
|
||||
|
||||
final StreamController<ServerLoadDataPoint> _loadDataController = StreamController.broadcast();
|
||||
Stream<ServerLoadDataPoint> get load_data => _loadDataController.stream;
|
||||
|
||||
ServerConnection({this.url});
|
||||
|
||||
open() {
|
||||
if (isConnected) return Future.value('already open');
|
||||
channel = IOWebSocketChannel.connect(this.url);
|
||||
int tick = 1;
|
||||
channel.stream.listen((message) {
|
||||
var data = json.decode(message);
|
||||
print(data);
|
||||
_loadDataController.add(
|
||||
ServerLoadDataPoint.from_map(
|
||||
tick, data
|
||||
)
|
||||
);
|
||||
tick++;
|
||||
});
|
||||
}
|
||||
|
||||
close() {
|
||||
if (isConnected) {
|
||||
return channel.sink.close(status.goingAway);
|
||||
}
|
||||
return Future.value('already closed');
|
||||
}
|
||||
|
||||
subscribe_to_server_load_data() {
|
||||
if (isConnected) channel.sink.add('subscribe');
|
||||
}
|
||||
|
||||
unsubscribe_from_server_load_data() {
|
||||
if (isConnected) channel.sink.add('unsubscribe');
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class TimeStats {
|
||||
final int avg, min, max;
|
||||
// percentiles
|
||||
final int five;
|
||||
final int twenty_five;
|
||||
final int fifty;
|
||||
final int seventy_five;
|
||||
final int ninety_five;
|
||||
TimeStats(
|
||||
this.avg, this.min, this.five, this.twenty_five,
|
||||
this.fifty, this.seventy_five, this.ninety_five,
|
||||
this.max
|
||||
);
|
||||
TimeStats.from_list(List l): this(
|
||||
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
|
||||
);
|
||||
TimeStats.from_zeros(): this(
|
||||
0, 0, 0, 0, 0, 0, 0, 0
|
||||
);
|
||||
factory TimeStats.from_list_or_zeros(List l) =>
|
||||
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
|
||||
}
|
||||
|
||||
|
||||
class APICallMetrics {
|
||||
// total requests received
|
||||
final int receive_count;
|
||||
// sum of these is total responses made
|
||||
final int cache_response_count;
|
||||
final int query_response_count;
|
||||
final int intrp_response_count;
|
||||
final int error_response_count;
|
||||
// stacked values for chart
|
||||
final int cache_response_stack;
|
||||
final int query_response_stack;
|
||||
final int intrp_response_stack;
|
||||
final int error_response_stack;
|
||||
// millisecond timings for non-cache responses
|
||||
final TimeStats response;
|
||||
final TimeStats interrupt;
|
||||
final TimeStats error;
|
||||
// response, interrupt and error each also report the python, wait and sql stats:
|
||||
final TimeStats python;
|
||||
final TimeStats wait;
|
||||
final TimeStats sql;
|
||||
// extended timings for individual sql executions
|
||||
final TimeStats individual_sql;
|
||||
final int individual_sql_count;
|
||||
// actual queries
|
||||
final List<String> errored_queries;
|
||||
final List<String> interrupted_queries;
|
||||
APICallMetrics(
|
||||
this.receive_count,
|
||||
this.cache_response_count, this.query_response_count,
|
||||
this.intrp_response_count, this.error_response_count,
|
||||
this.response, this.interrupt, this.error,
|
||||
this.python, this.wait, this.sql,
|
||||
this.individual_sql, this.individual_sql_count,
|
||||
this.errored_queries, this.interrupted_queries
|
||||
):
|
||||
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
|
||||
query_response_stack=query_response_count+intrp_response_count+error_response_count,
|
||||
intrp_response_stack=intrp_response_count+error_response_count,
|
||||
error_response_stack=error_response_count;
|
||||
APICallMetrics.from_map(Map data): this(
|
||||
data["receive_count"] ?? 0,
|
||||
data["cache_response_count"] ?? 0,
|
||||
data["query_response_count"] ?? 0,
|
||||
data["intrp_response_count"] ?? 0,
|
||||
data["error_response_count"] ?? 0,
|
||||
TimeStats.from_list_or_zeros(data["response"]),
|
||||
TimeStats.from_list_or_zeros(data["interrupt"]),
|
||||
TimeStats.from_list_or_zeros(data["error"]),
|
||||
TimeStats.from_list_or_zeros(data["python"]),
|
||||
TimeStats.from_list_or_zeros(data["wait"]),
|
||||
TimeStats.from_list_or_zeros(data["sql"]),
|
||||
TimeStats.from_list_or_zeros(data["individual_sql"]),
|
||||
data["individual_sql_count"] ?? 0,
|
||||
List<String>.from(data["errored_queries"] ?? const []),
|
||||
List<String>.from(data["interrupted_queries"] ?? const []),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServerLoadDataPoint {
|
||||
final int tick;
|
||||
final int sessions;
|
||||
final APICallMetrics search;
|
||||
final APICallMetrics resolve;
|
||||
const ServerLoadDataPoint(
|
||||
this.tick, this.sessions, this.search, this.resolve
|
||||
);
|
||||
ServerLoadDataPoint.from_map(int tick, Map data): this(
|
||||
tick, (data['status'] ?? const {})['sessions'] ?? 0,
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
|
||||
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
|
||||
);
|
||||
ServerLoadDataPoint.empty(): this(
|
||||
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
connect_and_listen_for_load_data() {
|
||||
ServerConnection(url: 'ws://localhost:8181/')..open()..load_data.listen((m) {
|
||||
print(m.search);
|
||||
});
|
||||
}
|
||||
@@ -7,7 +7,7 @@ packages:
|
||||
name: analyzer
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.37.0"
|
||||
version: "0.36.4"
|
||||
args:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -63,14 +63,14 @@ packages:
|
||||
name: csslib
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.16.1"
|
||||
version: "0.16.0"
|
||||
dart_style:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: dart_style
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.2.9"
|
||||
version: "1.2.8"
|
||||
fixnum:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -84,7 +84,7 @@ packages:
|
||||
name: front_end
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.1.20"
|
||||
version: "0.1.19"
|
||||
glob:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -134,13 +134,20 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.6.1+1"
|
||||
json_rpc_2:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: json_rpc_2
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.1.0"
|
||||
kernel:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: kernel
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.3.20"
|
||||
version: "0.3.19"
|
||||
matcher:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -203,7 +210,7 @@ packages:
|
||||
name: pedantic
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.8.0+1"
|
||||
version: "1.7.0"
|
||||
pool:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -217,14 +224,14 @@ packages:
|
||||
name: protobuf
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.13.15"
|
||||
version: "0.13.12"
|
||||
protoc_plugin:
|
||||
dependency: "direct dev"
|
||||
description:
|
||||
name: protoc_plugin
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "17.0.4"
|
||||
version: "17.0.2"
|
||||
pub_semver:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -295,13 +302,6 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.0.0"
|
||||
stream_transform:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: stream_transform
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.0.19"
|
||||
string_scanner:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -322,7 +322,7 @@ packages:
|
||||
name: test
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.6.5"
|
||||
version: "1.6.4"
|
||||
test_api:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -336,7 +336,7 @@ packages:
|
||||
name: test_core
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.2.7"
|
||||
version: "0.2.6"
|
||||
typed_data:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -344,13 +344,13 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.1.6"
|
||||
vm_service_lib:
|
||||
vm_service_client:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: vm_service_lib
|
||||
name: vm_service_client
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "3.22.0"
|
||||
version: "0.2.6+2"
|
||||
watcher:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -359,12 +359,12 @@ packages:
|
||||
source: hosted
|
||||
version: "0.9.7+10"
|
||||
web_socket_channel:
|
||||
dependency: "direct main"
|
||||
dependency: transitive
|
||||
description:
|
||||
name: web_socket_channel
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.0.14"
|
||||
version: "1.0.13"
|
||||
yaml:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
||||
@@ -11,7 +11,6 @@ dependencies:
|
||||
http: ^0.12.0+2
|
||||
path: ^1.4.1
|
||||
protobuf: ^0.13.12
|
||||
web_socket_channel: ^1.0.4
|
||||
|
||||
dev_dependencies:
|
||||
pedantic: ^1.0.0
|
||||
|
||||
@@ -1,83 +1,49 @@
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter/foundation.dart' show debugDefaultTargetPlatformOverride;
|
||||
import 'package:provider/provider.dart';
|
||||
import 'package:ver/src/servers.dart';
|
||||
import 'package:ver/src/models/server.dart';
|
||||
import 'package:ver/utils.dart';
|
||||
import 'package:ver/time_series_chart.dart';
|
||||
|
||||
|
||||
class UnderConstructionPage extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Scaffold(
|
||||
appBar: AppBar(title: Text('Under Construction')),
|
||||
body: SizedBox.expand(
|
||||
child: Center(child: Text('Under Construction')),
|
||||
),
|
||||
);
|
||||
}
|
||||
class VerApp extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return MaterialApp(
|
||||
title: 'Ver',
|
||||
theme: ThemeData(
|
||||
brightness: Brightness.light,
|
||||
primarySwatch: Colors.lightBlue,
|
||||
fontFamily: 'Roboto',
|
||||
),
|
||||
home: VerHomePage(title: 'Wallet Server'),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class MainPage extends StatefulWidget {
|
||||
@override
|
||||
_MainPageState createState() => _MainPageState();
|
||||
class VerHomePage extends StatefulWidget {
|
||||
VerHomePage({Key key, this.title}) : super(key: key);
|
||||
final String title;
|
||||
@override
|
||||
_VerHomePageState createState() => _VerHomePageState();
|
||||
}
|
||||
|
||||
|
||||
class _MainPageState extends State<MainPage> {
|
||||
int _currentIndex = 3;
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Scaffold(
|
||||
body: IndexedStack(
|
||||
index: _currentIndex,
|
||||
children: [
|
||||
UnderConstructionPage(),
|
||||
UnderConstructionPage(),
|
||||
UnderConstructionPage(),
|
||||
ServersSectionNavigation(),
|
||||
UnderConstructionPage(),
|
||||
],
|
||||
),
|
||||
bottomNavigationBar: BottomNavigationBar(
|
||||
items: const <BottomNavigationBarItem>[
|
||||
BottomNavigationBarItem(icon: Icon(Icons.home), title: Text('Home')),
|
||||
BottomNavigationBarItem(icon: Icon(Icons.trending_up), title: Text('Trending')),
|
||||
BottomNavigationBarItem(icon: Icon(Icons.subscriptions), title: Text('Subscriptions')),
|
||||
BottomNavigationBarItem(icon: Icon(Icons.router), title: Text('Servers')),
|
||||
BottomNavigationBarItem(icon: Icon(Icons.folder), title: Text('Library')),
|
||||
],
|
||||
currentIndex: _currentIndex,
|
||||
onTap: (int index) {
|
||||
setState(() {
|
||||
_currentIndex = index;
|
||||
});
|
||||
},
|
||||
selectedItemColor: Colors.amber[800],
|
||||
),
|
||||
);
|
||||
}
|
||||
class _VerHomePageState extends State<VerHomePage> {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Scaffold(
|
||||
appBar: AppBar(
|
||||
title: Text(widget.title),
|
||||
),
|
||||
body: new Padding(
|
||||
padding: const EdgeInsets.all(8.0),
|
||||
child: SimpleTimeSeriesChart()
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void main() {
|
||||
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
|
||||
runApp(
|
||||
MaterialApp(
|
||||
title: 'Ver',
|
||||
theme: ThemeData(
|
||||
brightness: Brightness.dark,
|
||||
//primaryColor: Colors.lightBlue[800],
|
||||
//accentColor: Colors.cyan[600],
|
||||
fontFamily: 'Roboto',
|
||||
),
|
||||
home: MultiProvider(
|
||||
providers: [
|
||||
ChangeNotifierProvider<ServerManager>(builder: (context) => ServerManager())
|
||||
],
|
||||
child: MainPage()
|
||||
)
|
||||
)
|
||||
);
|
||||
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
|
||||
runApp(new VerApp());
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
import 'dart:async';
|
||||
import 'dart:collection';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:lbry/lbry.dart';
|
||||
|
||||
|
||||
class ServerManager extends ChangeNotifier {
|
||||
final List<Server> _servers = [];
|
||||
UnmodifiableListView<Server> get items => UnmodifiableListView(_servers);
|
||||
|
||||
add(Server server) {
|
||||
_servers.add(server);
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
remove(Server server) {
|
||||
server.dispose();
|
||||
_servers.remove(server);
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
super.dispose();
|
||||
for (var server in _servers) {
|
||||
server.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class Server extends ChangeNotifier {
|
||||
|
||||
String _label = "";
|
||||
String get label => _label;
|
||||
String get labelOrHost => _label.length > 0 ? _label : host;
|
||||
|
||||
String _host = "";
|
||||
String get host => _host;
|
||||
int _port = 8181;
|
||||
int get port => _port;
|
||||
bool _ssl = false;
|
||||
bool get ssl => _ssl;
|
||||
String get url => _connection.url;
|
||||
_setURL(String host, int port, bool ssl) {
|
||||
_host = host;
|
||||
_port = port;
|
||||
_connection.url = "ws${ssl?'s':''}://$host:$port";
|
||||
}
|
||||
|
||||
DateTime _added = new DateTime.now();
|
||||
String get added => _added.toIso8601String();
|
||||
|
||||
bool _isDefault = false;
|
||||
bool get isDefault => _isDefault;
|
||||
|
||||
bool _isEnabled = false;
|
||||
bool get isEnabled => _isEnabled;
|
||||
|
||||
bool _isTrackingServerLoad = false;
|
||||
bool get isTrackingServerLoad => _isTrackingServerLoad;
|
||||
_setIsTrackingServerLoad(bool toggle_tracking) {
|
||||
if (_isTrackingServerLoad && !toggle_tracking) {
|
||||
_connection.unsubscribe_from_server_load_data();
|
||||
} else if (!_isTrackingServerLoad && toggle_tracking) {
|
||||
_connection.subscribe_to_server_load_data();
|
||||
}
|
||||
_isTrackingServerLoad = toggle_tracking;
|
||||
}
|
||||
|
||||
ClientLoadManager clientLoadManager;
|
||||
|
||||
final ServerConnection _connection = ServerConnection();
|
||||
bool get isConnected => _connection.isConnected;
|
||||
Stream<ServerLoadDataPoint> get serverLoadStream => _connection.load_data;
|
||||
final List<ServerLoadDataPoint> serverLoadData = [ServerLoadDataPoint.empty()];
|
||||
|
||||
Server() {
|
||||
clientLoadManager = ClientLoadManager(this);
|
||||
serverLoadStream.listen(serverLoadData.add);
|
||||
}
|
||||
|
||||
update({String host, int port, bool ssl, String label,
|
||||
bool isDefault, bool isEnabled, bool isTrackingServerLoad}) {
|
||||
if (host != null && port != null && ssl != null) {
|
||||
_setURL(host, port, ssl);
|
||||
}
|
||||
if (isTrackingServerLoad != null) {
|
||||
_setIsTrackingServerLoad(isTrackingServerLoad);
|
||||
}
|
||||
_label = label ?? _label;
|
||||
_isDefault = isDefault ?? _isDefault;
|
||||
_isEnabled = isEnabled ?? _isEnabled;
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
connect() async {
|
||||
await _connection.open();
|
||||
if (_isTrackingServerLoad) {
|
||||
_connection.subscribe_to_server_load_data();
|
||||
}
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
clientLoadManager.stop();
|
||||
_connection.close();
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
disconnect();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class ClientLoadManager extends ChangeNotifier {
|
||||
final Server _server;
|
||||
|
||||
ClientLoadGenerator clientLoadGenerator;
|
||||
final StreamController<ClientLoadDataPoint> _loadDataController = StreamController.broadcast();
|
||||
Stream<ClientLoadDataPoint> get clientLoadStream => _loadDataController.stream;
|
||||
final List<ClientLoadDataPoint> clientLoadData = [ClientLoadDataPoint.empty()];
|
||||
|
||||
int _load = 1;
|
||||
int get load => _load;
|
||||
int _offset = 0;
|
||||
int get offset => _offset;
|
||||
bool _noTotals = false;
|
||||
bool get noTotals => _noTotals;
|
||||
|
||||
ClientLoadManager(this._server) {
|
||||
clientLoadStream.listen(clientLoadData.add);
|
||||
}
|
||||
|
||||
update({int load, int offset, bool noTotals}) {
|
||||
_load = load ?? _load;
|
||||
_offset = offset ?? _offset;
|
||||
_noTotals = noTotals ?? _noTotals;
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
start() {
|
||||
clientLoadData.clear();
|
||||
clientLoadData.add(ClientLoadDataPoint.empty());
|
||||
clientLoadGenerator = ClientLoadGenerator(
|
||||
_server.host, _server.port,
|
||||
query: {
|
||||
'id': 1,
|
||||
'method': 'blockchain.claimtrie.search',
|
||||
'params': {
|
||||
'no_totals': _noTotals,
|
||||
'offset': _offset,
|
||||
'limit': 20,
|
||||
'fee_amount': '<1',
|
||||
//'all_tags': ['funny'],
|
||||
'any_tags': [
|
||||
'crypto',
|
||||
'outdoors',
|
||||
'cars',
|
||||
'automotive'
|
||||
]
|
||||
}
|
||||
}, tickCallback: (t, stats) {
|
||||
_loadDataController.add(stats);
|
||||
//increase = max(1, min(100, increase+2)-stats.backlog);
|
||||
//increase += 1;
|
||||
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
|
||||
t.load = _load;//rand.nextInt(10)+5;
|
||||
return true;
|
||||
})..start();
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (clientLoadGenerator != null) {
|
||||
clientLoadGenerator.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
super.dispose();
|
||||
stop();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:ver/src/widgets/server.dart';
|
||||
import 'package:provider/provider.dart';
|
||||
import 'models/server.dart';
|
||||
|
||||
|
||||
class ServerListPage extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) =>
|
||||
Scaffold(
|
||||
appBar: AppBar(title: Text('Servers')),
|
||||
body: ServerList(),
|
||||
floatingActionButton: FloatingActionButton(
|
||||
child: Icon(Icons.add),
|
||||
onPressed: () => Navigator.of(context).pushNamed('/edit', arguments: true),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServerFormPage extends StatelessWidget {
|
||||
|
||||
final bool creating;
|
||||
ServerFormPage(this.creating);
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) =>
|
||||
Scaffold(
|
||||
appBar: AppBar(title: Text(creating ? 'Add Server' : 'Modify Server')),
|
||||
body: ServerForm(creating),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServerViewPage extends StatelessWidget {
|
||||
final Server server;
|
||||
ServerViewPage(this.server);
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) =>
|
||||
Scaffold(
|
||||
appBar: AppBar(title: Text(server.labelOrHost)),
|
||||
body: ChangeNotifierProvider<Server>.value(
|
||||
value: server,
|
||||
child: ServerView()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
class ServersSectionNavigation extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Navigator(
|
||||
onGenerateRoute: (RouteSettings settings) {
|
||||
return MaterialPageRoute(
|
||||
settings: settings,
|
||||
builder: (BuildContext context) {
|
||||
switch (settings.name) {
|
||||
case '/':
|
||||
return ServerListPage();
|
||||
case '/edit':
|
||||
return ServerFormPage(settings.arguments);
|
||||
case '/view':
|
||||
return ServerViewPage(settings.arguments);
|
||||
}
|
||||
return ServerListPage();
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:provider/provider.dart';
|
||||
import '../models/server.dart';
|
||||
import 'time_series_chart.dart';
|
||||
|
||||
|
||||
class ServerList extends StatelessWidget {
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Consumer<ServerManager>(
|
||||
builder: (_, servers, __) =>
|
||||
ListView.builder(
|
||||
itemCount: servers.items.length,
|
||||
itemBuilder: (context, index) =>
|
||||
ChangeNotifierProvider<Server>.value(
|
||||
value: servers.items[index],
|
||||
child: ServerListItem()
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class ServerListItem extends StatelessWidget {
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Card(
|
||||
child: Consumer<Server>(
|
||||
builder: (_, server, __) =>
|
||||
ListTile(
|
||||
leading: FlutterLogo(size: 72.0),
|
||||
title: Text(server.labelOrHost),
|
||||
subtitle: Text("${server.url}\nadded ${server.added}"),
|
||||
trailing: Icon(Icons.more_vert),
|
||||
isThreeLine: true,
|
||||
onTap: ()=> Navigator.of(context).pushNamed('/view', arguments: server),
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ServerView extends StatelessWidget {
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) => ServerCharts();
|
||||
}
|
||||
|
||||
|
||||
class ServerForm extends StatefulWidget {
|
||||
final bool creating;
|
||||
|
||||
ServerForm(this.creating);
|
||||
|
||||
@override
|
||||
_ServerFormState createState() => _ServerFormState();
|
||||
}
|
||||
|
||||
|
||||
class _ServerFormState extends State<ServerForm> {
|
||||
|
||||
final _formKey = GlobalKey<FormState>();
|
||||
final _formData = {
|
||||
'label': '',
|
||||
'host': 'localhost',
|
||||
'port': 8181,
|
||||
'ssl': false,
|
||||
'isDefault': true,
|
||||
'isEnabled': false,
|
||||
'isTrackingServerLoad': false
|
||||
};
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Form(
|
||||
key: _formKey,
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: <Widget>[
|
||||
ListTile(title: TextFormField(
|
||||
initialValue: _formData['label'],
|
||||
decoration: const InputDecoration(
|
||||
labelText: 'Label',
|
||||
hintText: 'Optional text to display in server list.'
|
||||
),
|
||||
onSaved: (value) => _formData['label'] = value,
|
||||
)),
|
||||
ListTile(title: TextFormField(
|
||||
initialValue: _formData['host'],
|
||||
keyboardType: TextInputType.url,
|
||||
inputFormatters: [
|
||||
WhitelistingTextInputFormatter(RegExp(r'[\w\-\.]+'))
|
||||
],
|
||||
decoration: const InputDecoration(
|
||||
labelText: 'Host Name',
|
||||
hintText: 'Enter the host name of the server.'
|
||||
),
|
||||
validator: (value) => value.isEmpty ? 'A host name is required.' : null,
|
||||
onSaved: (value) => _formData['host'] = value,
|
||||
)),
|
||||
ListTile(title: TextFormField(
|
||||
initialValue: _formData['port'].toString(),
|
||||
keyboardType: TextInputType.number,
|
||||
inputFormatters: [
|
||||
WhitelistingTextInputFormatter.digitsOnly
|
||||
],
|
||||
decoration: const InputDecoration(
|
||||
labelText: 'Port',
|
||||
hintText: 'Enter the port of the server.'
|
||||
),
|
||||
validator: (value) => value.isEmpty ? 'A port is required.' : null,
|
||||
onSaved: (value) => setState(() => _formData['port'] = int.parse(value)),
|
||||
)),
|
||||
SwitchListTile(
|
||||
title: Text('Requires SSL.'),
|
||||
value: _formData['ssl'],
|
||||
onChanged: (value) => setState(() => _formData['ssl'] = value),
|
||||
),
|
||||
SwitchListTile(
|
||||
title: Text('Use this as your primary and default server.'),
|
||||
value: _formData['isDefault'],
|
||||
onChanged: (value) => setState(() => _formData['isDefault'] = value),
|
||||
),
|
||||
SwitchListTile(
|
||||
title: Text('Always stay connected.'),
|
||||
value: _formData['isEnabled'],
|
||||
onChanged: (value) => setState(() => _formData['isEnabled'] = value),
|
||||
),
|
||||
SwitchListTile(
|
||||
title: Text('Track server load.'),
|
||||
value: _formData['isTrackingServerLoad'],
|
||||
onChanged: (value) =>
|
||||
setState(() => _formData['isTrackingServerLoad'] = value),
|
||||
),
|
||||
ListTile(title: RaisedButton(
|
||||
onPressed: () {
|
||||
if (_formKey.currentState.validate()) {
|
||||
_formKey.currentState.save();
|
||||
var manager = Provider.of<ServerManager>(
|
||||
context, listen: false
|
||||
);
|
||||
var server = Server();
|
||||
server.update(
|
||||
label: _formData['label'],
|
||||
host: _formData['host'],
|
||||
port: _formData['port'],
|
||||
ssl: _formData['ssl'],
|
||||
isDefault: _formData['isDefault'],
|
||||
isEnabled: _formData['isEnabled'],
|
||||
isTrackingServerLoad: _formData['isTrackingServerLoad'],
|
||||
);
|
||||
manager.add(server);
|
||||
if (server.isEnabled) {
|
||||
server.connect();
|
||||
}
|
||||
Navigator.of(context).pop();
|
||||
}
|
||||
},
|
||||
child: Text(widget.creating ? 'Add Server' : 'Update Server'),
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,354 +0,0 @@
|
||||
import 'dart:math';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:charts_flutter/flutter.dart' as charts;
|
||||
import 'package:charts_flutter/src/base_chart_state.dart' as state;
|
||||
import 'package:charts_common/common.dart' as common;
|
||||
import 'package:provider/provider.dart';
|
||||
import 'package:lbry/lbry.dart';
|
||||
import '../models/server.dart';
|
||||
|
||||
|
||||
class ServerCharts extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
var server = Provider.of<Server>(context, listen: false);
|
||||
return ListView(children: <Widget>[
|
||||
SizedBox(height: 300.0, child: APILoadChart(
|
||||
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
|
||||
)),
|
||||
//SizedBox(height: 300.0, child: APILoadChart(
|
||||
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
|
||||
//)),
|
||||
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
|
||||
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
|
||||
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
|
||||
|
||||
class APILoadChart extends StatefulWidget {
|
||||
final Server server;
|
||||
final String name;
|
||||
final APIGetter getter;
|
||||
APILoadChart(this.server, this.name, this.getter);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => APILoadChartState();
|
||||
}
|
||||
|
||||
|
||||
class APILoadChartState extends State<APILoadChart> {
|
||||
|
||||
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Received',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
strokeWidthPxFn: (_, __) => 4.0,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
|
||||
data: widget.server.serverLoadData,
|
||||
),*/
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Cache',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Query',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.blue.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Interrupts',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Errors',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
/*
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Interrupted',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.pink.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} Errored',
|
||||
colorFn: (_, __) =>
|
||||
charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: '${widget.name} From Cache',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
|
||||
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
*/
|
||||
];
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ServerLoadDataPoint>(
|
||||
stream: widget.server.serverLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData,
|
||||
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ServerPerformanceChart extends StatefulWidget {
|
||||
final Server server;
|
||||
ServerPerformanceChart(this.server);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => ServerPerformanceChartState();
|
||||
}
|
||||
|
||||
|
||||
class ServerPerformanceChartState extends State<ServerPerformanceChart> {
|
||||
|
||||
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Waiting 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Waiting',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. Executing',
|
||||
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'SQLite 95 Percentile',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
charts.Series<ServerLoadDataPoint, int>(
|
||||
id: 'Avg. SQLite',
|
||||
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
|
||||
domainFn: (ServerLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
|
||||
data: widget.server.serverLoadData,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ServerLoadDataPoint>(
|
||||
stream: widget.server.serverLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
class ClientLoadChart extends StatefulWidget {
|
||||
final ClientLoadManager client;
|
||||
ClientLoadChart(this.client);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => ClientLoadChartState();
|
||||
}
|
||||
|
||||
|
||||
class ClientLoadChartState extends State<ClientLoadChart> {
|
||||
|
||||
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Load',
|
||||
colorFn: (_, __) => charts.MaterialPalette.black.darker,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.load,
|
||||
data: widget.client.clientLoadData,
|
||||
),
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Success',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.success,
|
||||
data: widget.client.clientLoadData,
|
||||
),
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Backlog',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.backlog,
|
||||
data: widget.client.clientLoadData,
|
||||
),
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Catch-up',
|
||||
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.catchup,
|
||||
data: widget.client.clientLoadData,
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ClientLoadDataPoint>(
|
||||
stream: widget.client.clientLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ClientPerformanceChart extends StatefulWidget {
|
||||
final ClientLoadManager client;
|
||||
ClientPerformanceChart(this.client);
|
||||
|
||||
@override
|
||||
State<StatefulWidget> createState() => ClientPerformanceChartState();
|
||||
}
|
||||
|
||||
|
||||
class ClientPerformanceChartState extends State<ClientPerformanceChart> {
|
||||
|
||||
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
seriesData = [
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Avg. Success Time',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.avg_success,
|
||||
data: widget.client.clientLoadData,
|
||||
),
|
||||
charts.Series<ClientLoadDataPoint, int>(
|
||||
id: 'Avg. Catch-up Time',
|
||||
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
|
||||
domainFn: (ClientLoadDataPoint load, _) => load.tick,
|
||||
measureFn: (ClientLoadDataPoint load, _) => load.avg_catchup,
|
||||
data: widget.client.clientLoadData,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return StreamBuilder<ClientLoadDataPoint>(
|
||||
stream: widget.client.clientLoadStream,
|
||||
builder: (BuildContext context, _) => BetterLineChart(seriesData)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class BetterLineChart extends charts.LineChart {
|
||||
|
||||
final int itemCount;
|
||||
final Object lastItem;
|
||||
|
||||
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
|
||||
itemCount = seriesList[0].data.length,
|
||||
lastItem = seriesList[0].data.last,
|
||||
super(
|
||||
seriesList,
|
||||
animate: false,
|
||||
defaultRenderer: renderer,
|
||||
behaviors: [charts.SeriesLegend()],
|
||||
domainAxis: charts.NumericAxisSpec(
|
||||
viewport: new charts.NumericExtents(
|
||||
max(0, seriesList[0].data.last.tick - 60), seriesList[0].data.last.tick
|
||||
),
|
||||
renderSpec: new charts.SmallTickRendererSpec(
|
||||
labelStyle: new charts.TextStyleSpec(
|
||||
color: charts.MaterialPalette.gray.shade50
|
||||
),
|
||||
lineStyle: new charts.LineStyleSpec(
|
||||
color: charts.MaterialPalette.black
|
||||
)
|
||||
),
|
||||
),
|
||||
primaryMeasureAxis: new charts.NumericAxisSpec(
|
||||
renderSpec: new charts.GridlineRendererSpec(
|
||||
labelStyle: new charts.TextStyleSpec(
|
||||
color: charts.MaterialPalette.white
|
||||
),
|
||||
lineStyle: new charts.LineStyleSpec(
|
||||
color: charts.MaterialPalette.gray.shade100
|
||||
),
|
||||
),
|
||||
tickProviderSpec: new charts.BasicNumericTickProviderSpec(
|
||||
dataIsInWholeNumbers: true,
|
||||
desiredTickCount: 5
|
||||
)
|
||||
),
|
||||
);
|
||||
|
||||
@override
|
||||
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
|
||||
state.BaseChartState chartState) {
|
||||
super.updateCommonChart(baseChart, oldWidget, chartState);
|
||||
final prev = oldWidget as BetterLineChart;
|
||||
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
|
||||
chartState.markChartDirty();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import 'dart:math';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:charts_flutter/flutter.dart' as charts;
|
||||
import 'package:charts_flutter/src/base_chart_state.dart' as state;
|
||||
import 'package:charts_common/common.dart' as common;
|
||||
import 'package:lbry/lbry.dart';
|
||||
|
||||
|
||||
class SimpleTimeSeriesChart extends StatefulWidget {
|
||||
SimpleTimeSeriesChart({Key key}) : super(key: key);
|
||||
@override
|
||||
_SimpleTimeSeriesChartState createState() => _SimpleTimeSeriesChartState();
|
||||
}
|
||||
|
||||
|
||||
class _SimpleTimeSeriesChartState extends State<SimpleTimeSeriesChart> {
|
||||
final List<LoadDataPoint> loadData = [];
|
||||
final List<charts.Series<LoadDataPoint, DateTime>> loadSeries = [];
|
||||
final List<charts.Series<LoadDataPoint, DateTime>> timeSeries = [];
|
||||
final Random rand = Random();
|
||||
LoadGenerator loadGenerator;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
loadSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Load',
|
||||
colorFn: (_, __) => charts.MaterialPalette.black.darker,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.load,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
loadSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Success',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.success,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
loadSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Backlog',
|
||||
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.backlog,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
loadSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Catch-up',
|
||||
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.catchup,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
timeSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Avg. Success Time',
|
||||
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.avg_success,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
timeSeries.add(
|
||||
charts.Series<LoadDataPoint, DateTime>(
|
||||
id: 'Avg. Catch-up Time',
|
||||
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
|
||||
domainFn: (LoadDataPoint load, _) => load.time,
|
||||
measureFn: (LoadDataPoint load, _) => load.avg_catchup,
|
||||
data: loadData,
|
||||
)
|
||||
);
|
||||
var increase = 1;
|
||||
loadData.add(LoadDataPoint());
|
||||
loadGenerator = LoadGenerator('spv2.lbry.com', 50001, {
|
||||
'id': 1,
|
||||
'method': 'blockchain.claimtrie.search',
|
||||
'params': {
|
||||
'offset': 0,
|
||||
'limit': 20,
|
||||
'fee_amount': '<1',
|
||||
'all_tags': ['funny'],
|
||||
'any_tags': [
|
||||
'crypto',
|
||||
'outdoors',
|
||||
'cars',
|
||||
'automotive'
|
||||
]
|
||||
}
|
||||
}, (t, stats) {
|
||||
setState(() {
|
||||
//if (loadData.length > 60) loadData.removeAt(0);
|
||||
loadData.add(stats);
|
||||
});
|
||||
//increase = max(1, min(30, (increase*1.1).ceil())-stats.backlog);
|
||||
increase += 1;
|
||||
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
|
||||
t.load = increase;//rand.nextInt(10)+5;
|
||||
return true;
|
||||
})..start();
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
loadGenerator.stop();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Column(children: <Widget>[
|
||||
SizedBox(height: 250.0, child: BetterTimeSeriesChart(loadSeries)),
|
||||
SizedBox(height: 250.0, child: BetterTimeSeriesChart(timeSeries)),
|
||||
]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class BetterTimeSeriesChart extends charts.TimeSeriesChart {
|
||||
|
||||
final int itemCount;
|
||||
final Object lastItem;
|
||||
|
||||
BetterTimeSeriesChart(
|
||||
List<charts.Series<dynamic, DateTime>> seriesList):
|
||||
itemCount = seriesList[0].data.length,
|
||||
lastItem = seriesList[0].data.last,
|
||||
super(seriesList, behaviors: [charts.SeriesLegend()]);
|
||||
|
||||
@override
|
||||
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
|
||||
state.BaseChartState chartState) {
|
||||
super.updateCommonChart(baseChart, oldWidget, chartState);
|
||||
final prev = oldWidget as BetterTimeSeriesChart;
|
||||
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
|
||||
chartState.markChartDirty();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ int main(int argc, char **argv) {
|
||||
flutter::FlutterWindowController flutter_controller(icu_data_path);
|
||||
|
||||
// Start the engine.
|
||||
if (!flutter_controller.CreateWindow(800, 1200, "ver - lbry workbench",
|
||||
if (!flutter_controller.CreateWindow(800, 600, "ver - lbry workbench",
|
||||
assets_path, arguments)) {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
@@ -43,20 +43,6 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.14.11"
|
||||
convert:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: convert
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.1.1"
|
||||
crypto:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: crypto
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.0.6"
|
||||
cupertino_icons:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
@@ -151,13 +137,6 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "0.13.15"
|
||||
provider:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: provider
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "3.0.0+1"
|
||||
quiver:
|
||||
dependency: transitive
|
||||
description:
|
||||
@@ -226,12 +205,5 @@ packages:
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.0.8"
|
||||
web_socket_channel:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: web_socket_channel
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.0.14"
|
||||
sdks:
|
||||
dart: ">=2.2.2 <3.0.0"
|
||||
@@ -12,7 +12,6 @@ dependencies:
|
||||
sdk: flutter
|
||||
cupertino_icons: ^0.1.2
|
||||
charts_flutter: ^0.6.0
|
||||
provider: ^3.0.0
|
||||
lbry:
|
||||
path: ../lbry
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.38.3"
|
||||
__version__ = "0.38.0"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -69,10 +69,10 @@ class BlobManager:
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
|
||||
+3
-3
@@ -22,6 +22,9 @@ CURRENCIES = {
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
|
||||
)
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
@@ -551,9 +554,6 @@ class Config(CLIConfig):
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
|
||||
@@ -163,6 +163,3 @@ class ComponentManager:
|
||||
if component.component_name == component_name:
|
||||
return component.component
|
||||
raise NameError(component_name)
|
||||
|
||||
def has_component(self, component_name):
|
||||
return any(component for component in self.components if component_name == component.component_name)
|
||||
@@ -4,6 +4,8 @@ import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
@@ -11,6 +13,7 @@ from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
from lbry import utils
|
||||
from lbry.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
@@ -20,7 +23,8 @@ from lbry.extras.daemon.Component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import LbryWalletManager
|
||||
from lbry.wallet.header import Headers
|
||||
from lbry.wallet import Network
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -108,34 +112,21 @@ class HeadersComponent(Component):
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self.headers = Headers(self.headers_file)
|
||||
self.is_downloading_headers = False
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def _round_progress(self, local_height, remote_height):
|
||||
return min(max(math.ceil(float(local_height) / float(remote_height) * 100), 0), 100)
|
||||
|
||||
async def get_status(self) -> dict:
|
||||
progress = None
|
||||
if self.is_downloading_headers:
|
||||
progress = self._headers_progress_percent
|
||||
elif self.component_manager.has_component(WALLET_COMPONENT):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
if wallet_manager and wallet_manager.ledger.network.remote_height > 0:
|
||||
local_height = wallet_manager.ledger.headers.height
|
||||
remote_height = wallet_manager.ledger.network.remote_height
|
||||
progress = self._round_progress(local_height, remote_height)
|
||||
return {
|
||||
'downloading_headers': True,
|
||||
'download_progress': progress
|
||||
} if progress is not None and progress < 100 else {}
|
||||
async def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.headers.bytes_size
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
@@ -145,68 +136,94 @@ class HeadersComponent(Component):
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
buffer, header_size = b'', self.headers.header_size
|
||||
async for chunk in response.content.iter_any():
|
||||
chunk = buffer + chunk
|
||||
remaining = len(chunk) % header_size
|
||||
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
|
||||
if not chunk:
|
||||
continue
|
||||
if not await self.headers.connect(len(self.headers), chunk):
|
||||
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
|
||||
return
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
self.headers.bytes_size, final_size_after_download
|
||||
)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while True:
|
||||
chunk = await response.content.read(512)
|
||||
if not chunk:
|
||||
break
|
||||
fd.write(chunk)
|
||||
self._headers_progress_percent = math.ceil(
|
||||
float(fd.tell()) / float(final_size_after_download) * 100
|
||||
)
|
||||
log.info("fetched headers from s3, now verifying integrity after download.")
|
||||
self._check_header_file_integrity()
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
|
||||
def local_header_file_size(self):
|
||||
if os.path.isfile(self.headers_file):
|
||||
return os.stat(self.headers_file).st_size
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
try:
|
||||
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return response.content_length // HEADER_SIZE
|
||||
except OSError:
|
||||
log.exception("Failed to download headers using https.")
|
||||
async def get_remote_height(self):
|
||||
ledger = SimpleNamespace()
|
||||
ledger.config = {
|
||||
'default_servers': self.conf.lbryum_servers,
|
||||
'data_path': self.conf.wallet_dir
|
||||
}
|
||||
net = Network(ledger)
|
||||
first_connection = net.on_connected.first
|
||||
asyncio.ensure_future(net.start()) # TODO: SKETCHY! it might be trapping a CancelledError and not raising it
|
||||
await first_connection
|
||||
remote_height = await net.get_server_height()
|
||||
await net.stop()
|
||||
return remote_height
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
async def should_download_headers_from_s3(self):
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.headers.height
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
return True
|
||||
local_height = self.local_header_file_height()
|
||||
remote_height = await self.get_remote_height()
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
with open(self.headers_file, "rb") as headers_file:
|
||||
hashsum.update(headers_file.read(checksum_length_in_bytes))
|
||||
current_checksum = hashsum.hexdigest()
|
||||
if current_checksum != checksum:
|
||||
msg = f"Expected checksum {checksum}, got {current_checksum}"
|
||||
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
|
||||
log.warning("Deleting header file so it can be downloaded again.")
|
||||
os.unlink(self.headers_file)
|
||||
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
|
||||
log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
if await self.should_download_headers_from_s3():
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
await self.headers.close()
|
||||
self._downloading_headers = False
|
||||
|
||||
async def stop(self):
|
||||
pass
|
||||
@@ -225,9 +242,9 @@ class WalletComponent(Component):
|
||||
return self.wallet_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.wallet_manager.ledger.network.remote_height:
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
remote_height = self.wallet_manager.ledger.network.remote_height
|
||||
remote_height = await self.wallet_manager.ledger.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
@@ -399,8 +416,10 @@ class StreamManagerComponent(Component):
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT)\
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
except NameError:
|
||||
node = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
|
||||
@@ -1742,12 +1742,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
multiple channels at the same time,
|
||||
includes claims with invalid signatures,
|
||||
use in conjunction with --valid_channel_signature
|
||||
--channel_ids=<channel_ids> : (list) claims signed by any of these channels
|
||||
--channel_ids=<channel_ids> : (str) claims signed by any of these channels
|
||||
(arguments must be claim ids of the channels),
|
||||
includes claims with invalid signatures,
|
||||
implies --has_channel_signature,
|
||||
use in conjunction with --valid_channel_signature
|
||||
--not_channel_ids=<not_channel_ids>: (list) exclude claims signed by any of these channels
|
||||
--not_channel_ids=<not_channel_ids>: (str) exclude claims signed by any of these channels
|
||||
(arguments must be claim ids of the channels)
|
||||
--has_channel_signature : (bool) claims with a channel signature (valid or invalid)
|
||||
--valid_channel_signature : (bool) claims with a valid channel signature or no signature,
|
||||
@@ -1815,7 +1815,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--not_locations=<not_locations> : (list) find claims not containing any of these locations
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
--order_by=<order_by> : (list) field to order by, default is descending order, to do an
|
||||
--order_by=<order_by> : (str) field to order by, default is descending order, to do an
|
||||
ascending order prepend ^ to the field name, eg. '^amount'
|
||||
available fields: 'name', 'height', 'release_time',
|
||||
'publish_time', 'amount', 'effective_amount',
|
||||
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
]
|
||||
}
|
||||
"""
|
||||
result = await jsonrpc_post(
|
||||
cmnt_list: dict = await jsonrpc_post(
|
||||
self.conf.comment_server,
|
||||
"get_claim_comments",
|
||||
claim_id=claim_id,
|
||||
@@ -3425,25 +3425,20 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
page_size=page_size,
|
||||
top_level=not include_replies
|
||||
)
|
||||
|
||||
for comment in result.get('items', []):
|
||||
channel_url = comment.get('channel_url')
|
||||
if not channel_url:
|
||||
continue
|
||||
resolve_response = await self.resolve([channel_url])
|
||||
if isinstance(resolve_response[channel_url], Output):
|
||||
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
|
||||
comment, resolve_response[channel_url]
|
||||
)
|
||||
else:
|
||||
comment['is_channel_signature_valid'] = False
|
||||
|
||||
if is_channel_signature_valid:
|
||||
result['items'] = [
|
||||
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
|
||||
]
|
||||
|
||||
return result
|
||||
if 'items' in cmnt_list:
|
||||
items = cmnt_list['items']
|
||||
chnl_uris = {cmnt['channel_url'] for cmnt in items if 'channel_url' in cmnt}
|
||||
claims = await self.resolve(tuple(chnl_uris))
|
||||
for cmnt in cmnt_list['items']:
|
||||
if 'channel_url' in cmnt:
|
||||
if cmnt['channel_url'] in claims:
|
||||
channel = claims[cmnt['channel_url']]
|
||||
cmnt['is_channel_signature_valid'] = is_comment_signed_by_channel(cmnt, channel)
|
||||
continue
|
||||
cmnt['is_channel_signature_valid'] = False
|
||||
if is_channel_signature_valid:
|
||||
cmnt_list['items'] = [c for c in cmnt_list['items'] if c['is_channel_signature_valid']]
|
||||
return cmnt_list
|
||||
|
||||
async def jsonrpc_comment_create(self, claim_id, comment, parent_id=None, channel_account_id=None,
|
||||
channel_name=None, channel_id=None) -> dict:
|
||||
|
||||
@@ -379,9 +379,7 @@ class Language(Metadata):
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2 and parts[0].isalpha():
|
||||
self.region = parts.pop(0)
|
||||
if parts and len(parts[0]) == 3 and parts[0].isdigit():
|
||||
if parts and len(parts[0]) == 2:
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@@ -406,13 +404,10 @@ class Language(Metadata):
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
r = LocationMessage.Country.Name(self.message.region)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
return LocationMessage.Country.Name(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
if len(region) == 3:
|
||||
region = 'R'+region
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
|
||||
|
||||
|
||||
def normalize_tag(tag: str):
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
|
||||
|
||||
|
||||
def clean_tags(tags: List[str]):
|
||||
return [tag for tag in set(normalize_tag(tag) for tag in tags) if tag]
|
||||
return [tag for tag in (normalize_tag(tag) for tag in tags) if tag]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -56,6 +56,7 @@ class ManagedStream:
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'written_bytes',
|
||||
'content_fee',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
@@ -88,6 +89,7 @@ class ManagedStream:
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.written_bytes = 0
|
||||
self.content_fee = content_fee
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
@@ -119,10 +121,6 @@ class ManagedStream:
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
|
||||
async def update_status(self, status: str):
|
||||
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
|
||||
self._status = status
|
||||
@@ -207,13 +205,21 @@ class ManagedStream:
|
||||
full_path = self.full_path
|
||||
file_name = self.file_name
|
||||
download_directory = self.download_directory
|
||||
if not self.output_file_exists:
|
||||
if self.full_path and self.output_file_exists:
|
||||
if self.written_bytes:
|
||||
written_bytes = self.written_bytes
|
||||
else:
|
||||
written_bytes = os.stat(self.full_path).st_size
|
||||
else:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
written_bytes = None
|
||||
return {
|
||||
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
|
||||
'completed': self.written_bytes >= self.descriptor.lower_bound_decrypted_length(),
|
||||
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
|
||||
or not self.saving.is_set()) or all(
|
||||
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
@@ -227,7 +233,7 @@ class ManagedStream:
|
||||
'key': self.descriptor.key,
|
||||
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': self.written_bytes,
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
@@ -365,6 +371,7 @@ class ManagedStream:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
self.written_bytes += len(decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
@@ -380,6 +387,7 @@ class ManagedStream:
|
||||
if os.path.isfile(output_path):
|
||||
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
|
||||
os.remove(output_path)
|
||||
self.written_bytes = 0
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
self.downloader.stop()
|
||||
await self.blob_manager.storage.change_file_download_dir_and_file_name(
|
||||
@@ -417,6 +425,7 @@ class ManagedStream:
|
||||
self.stream_hash, self.download_directory, self.file_name
|
||||
)
|
||||
await self.update_status(ManagedStream.STATUS_RUNNING)
|
||||
self.written_bytes = 0
|
||||
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
|
||||
await self.started_writing.wait()
|
||||
|
||||
|
||||
@@ -214,9 +214,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim)
|
||||
height = self.ledger.headers.height
|
||||
await self.generate(1)
|
||||
await self.on_header(height + 1)
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -3,11 +3,10 @@ import logging
|
||||
from binascii import unhexlify
|
||||
from typing import Tuple, List, Dict
|
||||
|
||||
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from torba.client.baseaccount import SingleKey
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.resolve import Resolver
|
||||
from lbry.wallet.account import Account
|
||||
@@ -73,50 +72,7 @@ class MainNetLedger(BaseLedger):
|
||||
result[url] = txo
|
||||
else:
|
||||
result[url] = {'error': f'{url} did not resolve to a claim'}
|
||||
return await self.validate_resolutions(result)
|
||||
|
||||
async def validate_resolutions(self, resolutions: dict):
|
||||
provables = {}
|
||||
for url, txo in resolutions.items():
|
||||
if isinstance(txo, dict) and 'error' in txo:
|
||||
continue
|
||||
parsed_url = URL.parse(url)
|
||||
# cases we check: names without sequence, claim id or any modifier
|
||||
# except: @channel/name as the signature is enough, instead we check '@channel'
|
||||
if parsed_url.has_channel:
|
||||
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
|
||||
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
|
||||
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
|
||||
if not provables:
|
||||
return resolutions
|
||||
root_hash = self.headers.claim_trie_root.decode()
|
||||
names = list(provables.keys())
|
||||
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
|
||||
for proof, name in zip(proofs, names):
|
||||
all_failed = True
|
||||
for url, txid in provables[name]:
|
||||
if not proof.get('txhash'):
|
||||
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
|
||||
continue
|
||||
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
|
||||
if txid != expected_txid:
|
||||
resolutions[url] = {
|
||||
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
|
||||
}
|
||||
else:
|
||||
all_failed = False
|
||||
if not all_failed:
|
||||
try:
|
||||
verify_proof(proof, root_hash, name)
|
||||
continue
|
||||
except InvalidProofError as error:
|
||||
for url, _ in provables[name]:
|
||||
resolution = resolutions[url]
|
||||
if isinstance(resolution, dict) and 'error' in resolution:
|
||||
continue
|
||||
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
|
||||
return resolutions
|
||||
return result
|
||||
|
||||
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
|
||||
return await self._inflate_outputs(self.network.claim_search(**kwargs))
|
||||
|
||||
@@ -184,8 +184,6 @@ class LbryWalletManager(BaseWalletManager):
|
||||
len(set(receiving_addresses).difference(set(migrated_receiving))))
|
||||
|
||||
def get_best_blockhash(self):
|
||||
if len(self.ledger.headers) <= 0:
|
||||
return self.ledger.genesis_hash
|
||||
return self.ledger.headers.hash(self.ledger.headers.height).decode()
|
||||
|
||||
def get_unused_address(self):
|
||||
|
||||
@@ -3,14 +3,14 @@ from torba.client.basenetwork import BaseNetwork
|
||||
|
||||
class Network(BaseNetwork):
|
||||
|
||||
def get_server_height(self):
|
||||
return self.rpc('blockchain.block.get_server_height', [])
|
||||
|
||||
def get_claims_by_ids(self, claim_ids):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
|
||||
|
||||
def resolve(self, urls):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
||||
|
||||
def get_name_proofs(self, block_hash, *names):
|
||||
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
|
||||
|
||||
def claim_search(self, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
||||
@@ -68,7 +68,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
|
||||
self.sql: SQLDB = self.db.sql
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.search_cache = {}
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
self.sql.begin()
|
||||
@@ -79,12 +78,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
raise
|
||||
finally:
|
||||
self.sql.commit()
|
||||
if self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -11,210 +11,3 @@ STREAM_TYPES = {
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = [
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
]
|
||||
|
||||
COMMON_TAGS = {
|
||||
"gaming": "gaming",
|
||||
"people & blogs": "people_and_blogs",
|
||||
"pop culture": "pop_culture",
|
||||
"entertainment": "entertainment",
|
||||
"technology": "technology",
|
||||
"music": "music",
|
||||
"funny": "funny",
|
||||
"education": "education",
|
||||
"learning": "learning",
|
||||
"news": "news",
|
||||
"gameplay": "gameplay",
|
||||
"science & technology": "science_and_technology",
|
||||
"playstation 4": "playstation_4",
|
||||
"beliefs": "beliefs",
|
||||
"nature": "nature",
|
||||
"news & politics": "news_and_politics",
|
||||
"comedy": "comedy",
|
||||
"games": "games",
|
||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
||||
"film & animation": "film_and_animation",
|
||||
"game": "game",
|
||||
"howto & style": "howto_and_style",
|
||||
"weapons": "weapons",
|
||||
"blockchain": "blockchain",
|
||||
"video game": "video_game",
|
||||
"sports": "sports",
|
||||
"walkthrough": "walkthrough",
|
||||
"ps4live": "ps4live",
|
||||
"art": "art",
|
||||
"pc": "pc",
|
||||
"economics": "economics",
|
||||
"automotive": "automotive",
|
||||
"minecraft": "minecraft",
|
||||
"playthrough": "playthrough",
|
||||
"ps4share": "ps4share",
|
||||
"tutorial": "tutorial",
|
||||
"play": "play",
|
||||
"twitch": "twitch",
|
||||
"how to": "how_to",
|
||||
"ps4": "ps4",
|
||||
"bitcoin": "bitcoin",
|
||||
"fortnite": "fortnite",
|
||||
"commentary": "commentary",
|
||||
"lets play": "lets_play",
|
||||
"fun": "fun",
|
||||
"politics": "politics",
|
||||
"xbox": "xbox",
|
||||
"autos & vehicles": "autos_and_vehicles",
|
||||
"travel & events": "travel_and_events",
|
||||
"food": "food",
|
||||
"science": "science",
|
||||
"mature": "mature",
|
||||
"xbox one": "xbox_one",
|
||||
"liberal": "liberal",
|
||||
"democrat": "democrat",
|
||||
"progressive": "progressive",
|
||||
"survival": "survival",
|
||||
"nonprofits & activism": "nonprofits_and_activism",
|
||||
"cryptocurrency": "cryptocurrency",
|
||||
"playstation": "playstation",
|
||||
"nintendo": "nintendo",
|
||||
"government": "government",
|
||||
"steam": "steam",
|
||||
"podcast": "podcast",
|
||||
"horror": "horror",
|
||||
"conservative": "conservative",
|
||||
"reaction": "reaction",
|
||||
"trailer": "trailer",
|
||||
"love": "love",
|
||||
"cnn": "cnn",
|
||||
"republican": "republican",
|
||||
"gamer": "gamer",
|
||||
"political": "political",
|
||||
"hangoutsonair": "hangoutsonair",
|
||||
"hoa": "hoa",
|
||||
"msnbc": "msnbc",
|
||||
"cbs": "cbs",
|
||||
"donald trump": "donald_trump",
|
||||
"fiction": "fiction",
|
||||
"fox news": "fox_news",
|
||||
"anime": "anime",
|
||||
"crypto": "crypto",
|
||||
"ethereum": "ethereum",
|
||||
"call of duty": "call_of_duty",
|
||||
"multiplayer": "multiplayer",
|
||||
"android": "android",
|
||||
"epic": "epic",
|
||||
"rpg": "rpg",
|
||||
"adventure": "adventure",
|
||||
"secular talk": "secular_talk",
|
||||
"btc": "btc",
|
||||
"atheist": "atheist",
|
||||
"atheism": "atheism",
|
||||
"ps3": "ps3",
|
||||
"video games": "video_games",
|
||||
"cod": "cod",
|
||||
"agnostic": "agnostic",
|
||||
"movie": "movie",
|
||||
"online": "online",
|
||||
"fps": "fps",
|
||||
"mod": "mod",
|
||||
"reviews": "reviews",
|
||||
"sharefactory": "sharefactory",
|
||||
"world": "world",
|
||||
"space": "space",
|
||||
"hilarious": "hilarious",
|
||||
"stream": "stream",
|
||||
"lol": "lol",
|
||||
"sony": "sony",
|
||||
"god": "god",
|
||||
"lets": "lets",
|
||||
"dance": "dance",
|
||||
"pvp": "pvp",
|
||||
"tech": "tech",
|
||||
"zombies": "zombies",
|
||||
"pokemon": "pokemon",
|
||||
"fail": "fail",
|
||||
"xbox 360": "xbox_360",
|
||||
"film": "film",
|
||||
"unboxing": "unboxing",
|
||||
"animation": "animation",
|
||||
"travel": "travel",
|
||||
"money": "money",
|
||||
"wwe": "wwe",
|
||||
"how": "how",
|
||||
"mods": "mods",
|
||||
"pubg": "pubg",
|
||||
"indie": "indie",
|
||||
"strategy": "strategy",
|
||||
"history": "history",
|
||||
"rap": "rap",
|
||||
"ios": "ios",
|
||||
"sony computer entertainment": "sony_computer_entertainment",
|
||||
"mobile": "mobile",
|
||||
"trump": "trump",
|
||||
"flat earth": "flat_earth",
|
||||
"hack": "hack",
|
||||
"trap": "trap",
|
||||
"fox": "fox",
|
||||
"vlogging": "vlogging",
|
||||
"news radio": "news_radio",
|
||||
"humor": "humor",
|
||||
"facebook": "facebook",
|
||||
"edm": "edm",
|
||||
"fitness": "fitness",
|
||||
"vaping": "vaping",
|
||||
"hip hop": "hip_hop",
|
||||
"secular": "secular",
|
||||
"jesus": "jesus",
|
||||
"vape": "vape",
|
||||
"song": "song",
|
||||
"remix": "remix",
|
||||
"guitar": "guitar",
|
||||
"daily": "daily",
|
||||
"mining": "mining",
|
||||
"diy": "diy",
|
||||
"videogame": "videogame",
|
||||
"pets & animals": "pets_and_animals",
|
||||
"funny moments": "funny_moments",
|
||||
"religion": "religion",
|
||||
"death": "death",
|
||||
"media": "media",
|
||||
"nbc": "nbc",
|
||||
"war": "war",
|
||||
"freedom": "freedom",
|
||||
"viral": "viral",
|
||||
"meme": "meme",
|
||||
"family": "family",
|
||||
"gold": "gold",
|
||||
"photography": "photography",
|
||||
"chill": "chill",
|
||||
"zombie": "zombie",
|
||||
"computer": "computer",
|
||||
"sniper": "sniper",
|
||||
"bible": "bible",
|
||||
"linux": "linux",
|
||||
"overwatch": "overwatch",
|
||||
"pro": "pro",
|
||||
"dragon": "dragon",
|
||||
"litecoin": "litecoin",
|
||||
"gta": "gta",
|
||||
"iphone": "iphone",
|
||||
"house": "house",
|
||||
"bass": "bass",
|
||||
"bitcoin news": "bitcoin_news",
|
||||
"wii": "wii",
|
||||
"crash": "crash",
|
||||
"league of legends": "league_of_legends",
|
||||
"grand theft auto v": "grand_theft_auto_v",
|
||||
"mario": "mario",
|
||||
"mmorpg": "mmorpg",
|
||||
"satire": "satire",
|
||||
"fire": "fire",
|
||||
"racing": "racing",
|
||||
"apple": "apple",
|
||||
"health": "health",
|
||||
"instrumental": "instrumental",
|
||||
"destiny": "destiny",
|
||||
"truth": "truth",
|
||||
"race": "race"
|
||||
}
|
||||
@@ -1,34 +1,17 @@
|
||||
import time
|
||||
import struct
|
||||
import sqlite3
|
||||
import logging
|
||||
from typing import Tuple, List, Dict, Union, Type, Optional
|
||||
import struct
|
||||
from typing import Tuple, List
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from dataclasses import dataclass
|
||||
|
||||
from torba.client.basedatabase import query, interpolate
|
||||
from torba.client.basedatabase import query
|
||||
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
class SQLiteOperationalError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query errored')
|
||||
self.metrics = metrics
|
||||
|
||||
|
||||
class SQLiteInterruptedError(sqlite3.OperationalError):
|
||||
def __init__(self, metrics):
|
||||
super().__init__('sqlite query interrupted')
|
||||
self.metrics = metrics
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
@@ -56,7 +39,7 @@ SEARCH_PARAMS = {
|
||||
|
||||
|
||||
ORDER_FIELDS = {
|
||||
'name', 'claim_hash'
|
||||
'name',
|
||||
} | INTEGER_PARAMS
|
||||
|
||||
|
||||
@@ -65,119 +48,25 @@ PRAGMAS = """
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReaderState:
|
||||
db: sqlite3.Connection
|
||||
stack: List[List]
|
||||
metrics: Dict
|
||||
is_tracking_metrics: bool
|
||||
ledger: Type[BaseLedger]
|
||||
query_timeout: float
|
||||
log: logging.Logger
|
||||
|
||||
def close(self):
|
||||
self.db.close()
|
||||
|
||||
def reset_metrics(self):
|
||||
self.stack = []
|
||||
self.metrics = {}
|
||||
|
||||
def set_query_timeout(self):
|
||||
stop_at = time.perf_counter() + self.query_timeout
|
||||
|
||||
def interruptor():
|
||||
if time.perf_counter() >= stop_at:
|
||||
self.db.interrupt()
|
||||
return
|
||||
|
||||
self.db.set_progress_handler(interruptor, 100)
|
||||
db = ContextVar('db')
|
||||
ledger = ContextVar('ledger')
|
||||
|
||||
|
||||
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
|
||||
|
||||
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
|
||||
db = sqlite3.connect(_path, isolation_level=None, uri=True)
|
||||
db.row_factory = sqlite3.Row
|
||||
ctx.set(
|
||||
ReaderState(
|
||||
db=db, stack=[], metrics={}, is_tracking_metrics=_measure,
|
||||
ledger=MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger,
|
||||
query_timeout=query_timeout, log=log
|
||||
)
|
||||
)
|
||||
def initializer(_path, _ledger_name):
|
||||
_db = sqlite3.connect(_path, isolation_level=None, uri=True)
|
||||
_db.row_factory = sqlite3.Row
|
||||
db.set(_db)
|
||||
from lbry.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
ledger.set(MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger)
|
||||
|
||||
|
||||
def cleanup():
|
||||
ctx.get().close()
|
||||
ctx.set(None)
|
||||
db.get().close()
|
||||
db.set(None)
|
||||
ledger.set(None)
|
||||
|
||||
|
||||
def measure(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
state = ctx.get()
|
||||
if not state.is_tracking_metrics:
|
||||
return func(*args, **kwargs)
|
||||
metric = {}
|
||||
state.metrics.setdefault(func.__name__, []).append(metric)
|
||||
state.stack.append([])
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
elapsed = int((time.perf_counter()-start)*1000)
|
||||
metric['total'] = elapsed
|
||||
metric['isolated'] = (elapsed-sum(state.stack.pop()))
|
||||
if state.stack:
|
||||
state.stack[-1].append(elapsed)
|
||||
return wrapper
|
||||
|
||||
|
||||
def reports_metrics(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
state = ctx.get()
|
||||
if not state.is_tracking_metrics:
|
||||
return func(*args, **kwargs)
|
||||
state.reset_metrics()
|
||||
r = func(*args, **kwargs)
|
||||
return r, state.metrics
|
||||
return wrapper
|
||||
|
||||
|
||||
@reports_metrics
|
||||
def search_to_bytes(constraints) -> Union[bytes, Tuple[bytes, Dict]]:
|
||||
return encode_result(search(constraints))
|
||||
|
||||
|
||||
@reports_metrics
|
||||
def resolve_to_bytes(urls) -> Union[bytes, Tuple[bytes, Dict]]:
|
||||
return encode_result(resolve(urls))
|
||||
|
||||
|
||||
def encode_result(result):
|
||||
return Outputs.to_bytes(*result)
|
||||
|
||||
|
||||
@measure
|
||||
def execute_query(sql, values) -> List:
|
||||
context = ctx.get()
|
||||
context.set_query_timeout()
|
||||
try:
|
||||
return context.db.execute(sql, values).fetchall()
|
||||
except sqlite3.OperationalError as err:
|
||||
plain_sql = interpolate(sql, values)
|
||||
if context.is_tracking_metrics:
|
||||
context.metrics['execute_query'][-1]['sql'] = plain_sql
|
||||
if str(err) == "interrupted":
|
||||
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
|
||||
raise SQLiteInterruptedError(context.metrics)
|
||||
context.log.exception('failed running query', exc_info=err)
|
||||
raise SQLiteOperationalError(context.metrics)
|
||||
|
||||
|
||||
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
def get_claims(cols, for_count=False, **constraints):
|
||||
if 'order_by' in constraints:
|
||||
sql_order_by = []
|
||||
for order_by in constraints['order_by']:
|
||||
@@ -231,7 +120,15 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['claim.public_key_hash'] = sqlite3.Binary(
|
||||
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
ledger.get().address_to_hash160(constraints.pop('public_key_id')))
|
||||
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = _resolve_one(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
if 'channel_hash' in constraints:
|
||||
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
|
||||
if 'channel_ids' in constraints:
|
||||
@@ -300,23 +197,10 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
""", **constraints
|
||||
)
|
||||
return sql, values
|
||||
return db.get().execute(sql, values).fetchall()
|
||||
|
||||
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = resolve_url(channel_url)
|
||||
if isinstance(match, sqlite3.Row):
|
||||
constraints['channel_hash'] = match['claim_hash']
|
||||
else:
|
||||
return [[0]] if cols == 'count(*)' else []
|
||||
sql, values = _get_claims(cols, for_count, **constraints)
|
||||
return execute_query(sql, values)
|
||||
|
||||
|
||||
@measure
|
||||
def get_claims_count(**constraints) -> int:
|
||||
def get_claims_count(**constraints):
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
@@ -324,7 +208,6 @@ def get_claims_count(**constraints) -> int:
|
||||
return count[0][0]
|
||||
|
||||
|
||||
@measure
|
||||
def search(constraints) -> Tuple[List, List, int, int]:
|
||||
assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
|
||||
@@ -334,17 +217,19 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
constraints['order_by'] = ["height", "^name"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
|
||||
)
|
||||
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
|
||||
return txo_rows, extra_txo_rows, constraints['offset'], total
|
||||
|
||||
|
||||
def search_to_bytes(constraints) -> bytes:
|
||||
return Outputs.to_bytes(*search(constraints))
|
||||
|
||||
|
||||
def _search(**constraints):
|
||||
return get_claims(
|
||||
"""
|
||||
@@ -364,26 +249,25 @@ def _search(**constraints):
|
||||
)
|
||||
|
||||
|
||||
@measure
|
||||
def resolve(urls) -> Tuple[List, List]:
|
||||
result = []
|
||||
channel_hashes = set()
|
||||
for raw_url in urls:
|
||||
match = resolve_url(raw_url)
|
||||
match = _resolve_one(raw_url)
|
||||
result.append(match)
|
||||
if isinstance(match, sqlite3.Row) and match['channel_hash']:
|
||||
channel_hashes.add(match['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
|
||||
)
|
||||
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
|
||||
return result, extra_txo_rows
|
||||
|
||||
|
||||
@measure
|
||||
def resolve_url(raw_url):
|
||||
def resolve_to_bytes(urls) -> bytes:
|
||||
return Outputs.to_bytes(*resolve(urls))
|
||||
|
||||
|
||||
def _resolve_one(raw_url):
|
||||
try:
|
||||
url = URL.parse(raw_url)
|
||||
except ValueError as e:
|
||||
@@ -408,7 +292,7 @@ def resolve_url(raw_url):
|
||||
if channel is not None:
|
||||
if set(query) == {'name'}:
|
||||
# temporarily emulate is_controlling for claims in channel
|
||||
query['order_by'] = ['effective_amount', '^height']
|
||||
query['order_by'] = ['effective_amount']
|
||||
else:
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['channel_hash'] = channel['claim_hash']
|
||||
@@ -425,68 +309,28 @@ def resolve_url(raw_url):
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
not_items = set(cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
|
||||
all_items = {item for item in all_items if item not in not_items}
|
||||
any_items = {item for item in any_items if item not in not_items}
|
||||
|
||||
any_queries = {}
|
||||
|
||||
if attr == 'tag':
|
||||
common_tags = any_items & COMMON_TAGS.keys()
|
||||
if common_tags:
|
||||
any_items -= common_tags
|
||||
if len(common_tags) < 5:
|
||||
for item in common_tags:
|
||||
index_name = COMMON_TAGS[item]
|
||||
any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag = '{item}'
|
||||
)
|
||||
"""
|
||||
elif len(common_tags) >= 5:
|
||||
constraints.update({
|
||||
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_common_tag{i}' for i in range(len(common_tags))
|
||||
)
|
||||
any_queries[f'#_any_common_tags'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
any_items = cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
|
||||
if any_items:
|
||||
|
||||
constraints.update({
|
||||
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
if for_count:
|
||||
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
constraints[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if len(any_queries) == 1:
|
||||
constraints.update(any_queries)
|
||||
elif len(any_queries) > 1:
|
||||
constraints[f'ORed_{attr}_queries__any'] = any_queries
|
||||
|
||||
all_items = cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
|
||||
if all_items:
|
||||
constraints[f'$all_{attr}_count'] = len(all_items)
|
||||
constraints.update({
|
||||
@@ -509,6 +353,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
)
|
||||
"""
|
||||
|
||||
not_items = cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
|
||||
if not_items:
|
||||
constraints.update({
|
||||
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
|
||||
|
||||
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
|
||||
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
|
||||
)
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
@@ -76,12 +76,33 @@ class SQLDB:
|
||||
trending_global integer not null default 0
|
||||
);
|
||||
|
||||
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized);
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash);
|
||||
create index if not exists claim_release_time_idx on claim (release_time);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_height_idx on claim (height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
|
||||
create index if not exists claim_effective_amount_idx on claim (effective_amount);
|
||||
create index if not exists claim_trending_group_idx on claim (trending_group);
|
||||
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
|
||||
create index if not exists claim_trending_local_idx on claim (trending_local);
|
||||
create index if not exists claim_trending_global_idx on claim (trending_global);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -92,6 +113,7 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
amount integer not null
|
||||
);
|
||||
create index if not exists support_txo_hash_idx on support (txo_hash);
|
||||
create index if not exists support_claim_hash_idx on support (claim_hash, height);
|
||||
"""
|
||||
|
||||
@@ -101,7 +123,9 @@ class SQLDB:
|
||||
claim_hash bytes not null,
|
||||
height integer not null
|
||||
);
|
||||
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
|
||||
create index if not exists tag_tag_idx on tag (tag);
|
||||
create index if not exists tag_claim_hash_idx on tag (claim_hash);
|
||||
create index if not exists tag_height_idx on tag (height);
|
||||
"""
|
||||
|
||||
CREATE_CLAIMTRIE_TABLE = """
|
||||
@@ -113,38 +137,6 @@ class SQLDB:
|
||||
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
|
||||
"""
|
||||
|
||||
SEARCH_INDEXES = """
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
create index if not exists claim_fee_amount_idx on claim (fee_amount);
|
||||
create index if not exists claim_fee_currency_idx on claim (fee_currency);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
TAG_INDEXES = '\n'.join(
|
||||
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
|
||||
for tag_value, tag_key in COMMON_TAGS.items()
|
||||
)
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
PRAGMAS +
|
||||
CREATE_CLAIM_TABLE +
|
||||
@@ -208,7 +200,7 @@ class SQLDB:
|
||||
self.execute('commit;')
|
||||
|
||||
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
|
||||
claim_hashes, claims, tags = [], [], {}
|
||||
claim_hashes, claims, tags = [], [], []
|
||||
for txo in txos:
|
||||
tx = txo.tx_ref.tx
|
||||
|
||||
@@ -262,14 +254,14 @@ class SQLDB:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
for tag in clean_tags(claim.message.tags):
|
||||
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
|
||||
tags.append((tag, claim_hash, tx.height))
|
||||
|
||||
if clear_first:
|
||||
self._clear_claim_metadata(claim_hashes)
|
||||
|
||||
if tags:
|
||||
self.db.executemany(
|
||||
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags
|
||||
)
|
||||
|
||||
return claims
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
import time
|
||||
import math
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
def calculate_elapsed(start) -> int:
|
||||
return int((time.perf_counter() - start) * 1000)
|
||||
|
||||
|
||||
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
|
||||
if not data:
|
||||
return 0, 0, 0, 0, 0, 0, 0, 0
|
||||
data.sort()
|
||||
size = len(data)
|
||||
return (
|
||||
int(sum(data) / size),
|
||||
data[0],
|
||||
data[math.ceil(size * .05) - 1],
|
||||
data[math.ceil(size * .25) - 1],
|
||||
data[math.ceil(size * .50) - 1],
|
||||
data[math.ceil(size * .75) - 1],
|
||||
data[math.ceil(size * .95) - 1],
|
||||
data[-1]
|
||||
)
|
||||
|
||||
|
||||
def remove_select_list(sql) -> str:
|
||||
return sql[sql.index('FROM'):]
|
||||
|
||||
|
||||
class APICallMetrics:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
# total requests received
|
||||
self.receive_count = 0
|
||||
self.cache_response_count = 0
|
||||
|
||||
# millisecond timings for query based responses
|
||||
self.query_response_times = []
|
||||
self.query_intrp_times = []
|
||||
self.query_error_times = []
|
||||
|
||||
self.query_python_times = []
|
||||
self.query_wait_times = []
|
||||
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
|
||||
|
||||
self.individual_sql_times = [] # every SQL query run on server
|
||||
|
||||
# actual queries
|
||||
self.errored_queries = set()
|
||||
self.interrupted_queries = set()
|
||||
|
||||
def to_json_and_reset(self):
|
||||
return {
|
||||
# total requests received
|
||||
"receive_count": self.receive_count,
|
||||
# sum of these is total responses made
|
||||
"cache_response_count": self.cache_response_count,
|
||||
"query_response_count": len(self.query_response_times),
|
||||
"intrp_response_count": len(self.query_intrp_times),
|
||||
"error_response_count": len(self.query_error_times),
|
||||
# millisecond timings for non-cache responses
|
||||
"response": calculate_avg_percentiles(self.query_response_times),
|
||||
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
|
||||
"error": calculate_avg_percentiles(self.query_error_times),
|
||||
# response, interrupt and error each also report the python, wait and sql stats:
|
||||
"python": calculate_avg_percentiles(self.query_python_times),
|
||||
"wait": calculate_avg_percentiles(self.query_wait_times),
|
||||
"sql": calculate_avg_percentiles(self.query_sql_times),
|
||||
# extended timings for individual sql executions
|
||||
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
|
||||
"individual_sql_count": len(self.individual_sql_times),
|
||||
# actual queries
|
||||
"errored_queries": list(self.errored_queries),
|
||||
"interrupted_queries": list(self.interrupted_queries),
|
||||
}
|
||||
|
||||
def start(self):
|
||||
self.receive_count += 1
|
||||
|
||||
def cache_response(self):
|
||||
self.cache_response_count += 1
|
||||
|
||||
def _add_query_timings(self, request_total_time, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
sub_process_total = metrics[self.name][0]['total']
|
||||
individual_query_times = [f['total'] for f in metrics['execute_query']]
|
||||
aggregated_query_time = sum(individual_query_times)
|
||||
self.individual_sql_times.extend(individual_query_times)
|
||||
self.query_sql_times.append(aggregated_query_time)
|
||||
self.query_python_times.append(sub_process_total - aggregated_query_time)
|
||||
self.query_wait_times.append(request_total_time - sub_process_total)
|
||||
|
||||
@staticmethod
|
||||
def _add_queries(query_set, metrics):
|
||||
if metrics and 'execute_query' in metrics:
|
||||
for execute_query in metrics['execute_query']:
|
||||
if 'sql' in execute_query:
|
||||
query_set.add(remove_select_list(execute_query['sql']))
|
||||
|
||||
def query_response(self, start, metrics):
|
||||
self.query_response_times.append(calculate_elapsed(start))
|
||||
self._add_query_timings(self.query_response_times[-1], metrics)
|
||||
|
||||
def query_interrupt(self, start, metrics):
|
||||
self.query_intrp_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.interrupted_queries, metrics)
|
||||
self._add_query_timings(self.query_intrp_times[-1], metrics)
|
||||
|
||||
def query_error(self, start, metrics):
|
||||
self.query_error_times.append(calculate_elapsed(start))
|
||||
self._add_queries(self.errored_queries, metrics)
|
||||
self._add_query_timings(self.query_error_times[-1], metrics)
|
||||
|
||||
|
||||
class ServerLoadData:
|
||||
|
||||
def __init__(self):
|
||||
self._apis = {}
|
||||
|
||||
def for_api(self, name) -> APICallMetrics:
|
||||
if name not in self._apis:
|
||||
self._apis[name] = APICallMetrics(name)
|
||||
return self._apis[name]
|
||||
|
||||
def to_json_and_reset(self, status):
|
||||
try:
|
||||
return {
|
||||
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
|
||||
'status': status
|
||||
}
|
||||
finally:
|
||||
self._apis = {}
|
||||
@@ -1,40 +1,17 @@
|
||||
import os
|
||||
import math
|
||||
import time
|
||||
import base64
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from torba.rpc.jsonrpc import RPCError, JSONRPC
|
||||
from torba.rpc.jsonrpc import RPCError
|
||||
from torba.server.session import ElectrumX, SessionManager
|
||||
from torba.server import util
|
||||
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYDB
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
|
||||
|
||||
class ResultCacheItem:
|
||||
__slots__ = '_result', 'lock', 'has_result'
|
||||
|
||||
def __init__(self):
|
||||
self.has_result = asyncio.Event()
|
||||
self.lock = asyncio.Lock()
|
||||
self._result = None
|
||||
|
||||
@property
|
||||
def result(self) -> str:
|
||||
return self._result
|
||||
|
||||
@result.setter
|
||||
def result(self, result: str):
|
||||
self._result = result
|
||||
if result is not None:
|
||||
self.has_result.set()
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
|
||||
class LBRYSessionManager(SessionManager):
|
||||
@@ -42,54 +19,23 @@ class LBRYSessionManager(SessionManager):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.query_executor = None
|
||||
self.websocket = None
|
||||
self.metrics = ServerLoadData()
|
||||
self.metrics_loop = None
|
||||
self.running = False
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
self.search_cache = self.bp.search_cache
|
||||
self.search_cache['search'] = lrucache(10000)
|
||||
self.search_cache['resolve'] = lrucache(10000)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
|
||||
if self.websocket is not None:
|
||||
self.websocket.send_message(data)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def start_other(self):
|
||||
self.running = True
|
||||
args = dict(
|
||||
initializer=reader.initializer,
|
||||
initargs=(self.logger, 'claims.db', self.env.coin.NET, self.env.database_query_timeout,
|
||||
self.env.track_metrics)
|
||||
)
|
||||
args = dict(initializer=reader.initializer, initargs=('claims.db', self.env.coin.NET))
|
||||
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
|
||||
self.query_executor = ThreadPoolExecutor(max_workers=1, **args)
|
||||
else:
|
||||
self.query_executor = ProcessPoolExecutor(
|
||||
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args
|
||||
)
|
||||
if self.websocket is not None:
|
||||
await self.websocket.start()
|
||||
if self.env.track_metrics:
|
||||
self.metrics_loop = asyncio.create_task(self.process_metrics())
|
||||
|
||||
async def stop_other(self):
|
||||
self.running = False
|
||||
if self.env.track_metrics:
|
||||
self.metrics_loop.cancel()
|
||||
if self.websocket is not None:
|
||||
await self.websocket.stop()
|
||||
self.query_executor.shutdown()
|
||||
|
||||
|
||||
class LBRYElectrumX(ElectrumX):
|
||||
PROTOCOL_MIN = (0, 0) # temporary, for supporting 0.10 protocol
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -104,71 +50,26 @@ class LBRYElectrumX(ElectrumX):
|
||||
'blockchain.transaction.get_height': self.transaction_get_height,
|
||||
'blockchain.claimtrie.search': self.claimtrie_search,
|
||||
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
|
||||
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
|
||||
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
|
||||
'blockchain.block.get_server_height': self.get_server_height,
|
||||
}
|
||||
self.request_handlers.update(handlers)
|
||||
|
||||
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics.query_error(start, error.metrics)
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except:
|
||||
metrics.query_error(start, {})
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
|
||||
if self.env.track_metrics:
|
||||
(result, metrics_data) = result
|
||||
metrics.query_response(start, metrics_data)
|
||||
|
||||
return base64.b64encode(result).decode()
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
if self.env.track_metrics:
|
||||
metrics = self.session_mgr.metrics.for_api(query_name)
|
||||
else:
|
||||
metrics = APICallMetrics(query_name)
|
||||
metrics.start()
|
||||
cache = self.session_mgr.search_cache[query_name]
|
||||
cache_key = str(kwargs)
|
||||
cache_item = cache.get(cache_key)
|
||||
if cache_item is None:
|
||||
cache_item = cache[cache_key] = ResultCacheItem()
|
||||
elif cache_item.result is not None:
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
async with cache_item.lock:
|
||||
if cache_item.result is None:
|
||||
cache_item.result = await self.run_in_executor(
|
||||
metrics, function, kwargs
|
||||
)
|
||||
else:
|
||||
metrics.cache_response()
|
||||
return cache_item.result
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
|
||||
if 'claim_id' in kwargs:
|
||||
self.assert_claim_id(kwargs['claim_id'])
|
||||
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, reader.search_to_bytes, kwargs
|
||||
)).decode()
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
|
||||
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, reader.resolve_to_bytes, urls
|
||||
)).decode()
|
||||
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
|
||||
def claimtrie_getnameproofs(self, block_hash, *names):
|
||||
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
|
||||
|
||||
async def transaction_get_height(self, tx_hash):
|
||||
self.assert_tx_hash(tx_hash)
|
||||
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import asyncio
|
||||
from weakref import WeakSet
|
||||
|
||||
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
|
||||
from aiohttp.http_websocket import WSMsgType, WSCloseCode
|
||||
|
||||
|
||||
class AdminWebSocket:
|
||||
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
self.app = Application()
|
||||
self.app['websockets'] = WeakSet()
|
||||
self.app.router.add_get('/', self.on_connect)
|
||||
self.app.on_shutdown.append(self.on_shutdown)
|
||||
self.runner = AppRunner(self.app)
|
||||
|
||||
async def on_status(self, _):
|
||||
if not self.app['websockets']:
|
||||
return
|
||||
self.send_message({
|
||||
'type': 'status',
|
||||
'height': self.manager.daemon.cached_height(),
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
for web_socket in self.app['websockets']:
|
||||
asyncio.create_task(web_socket.send_json(msg))
|
||||
|
||||
async def start(self):
|
||||
await self.runner.setup()
|
||||
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def on_connect(self, request):
|
||||
web_socket = WebSocketResponse()
|
||||
await web_socket.prepare(request)
|
||||
self.app['websockets'].add(web_socket)
|
||||
try:
|
||||
async for msg in web_socket:
|
||||
if msg.type == WSMsgType.TEXT:
|
||||
await self.on_status(None)
|
||||
elif msg.type == WSMsgType.ERROR:
|
||||
print('web socket connection closed with exception %s' %
|
||||
web_socket.exception())
|
||||
finally:
|
||||
self.app['websockets'].discard(web_socket)
|
||||
return web_socket
|
||||
|
||||
@staticmethod
|
||||
async def on_shutdown(app):
|
||||
for web_socket in set(app['websockets']):
|
||||
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
|
||||
@@ -1,146 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
|
||||
from lbry.wallet.ledger import MainNetLedger
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
DEFAULT_ANY_TAGS = [
|
||||
'blockchain',
|
||||
'news',
|
||||
'learning',
|
||||
'technology',
|
||||
'automotive',
|
||||
'economics',
|
||||
'food',
|
||||
'science',
|
||||
'art',
|
||||
'nature'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE = [
|
||||
'gaming',
|
||||
'ufos'
|
||||
]
|
||||
|
||||
COMMON_AND_RARE2 = [
|
||||
'city fix',
|
||||
'gaming'
|
||||
]
|
||||
|
||||
RARE_ANY_TAGS = [
|
||||
'city fix',
|
||||
'ufos',
|
||||
]
|
||||
|
||||
CITY_FIX = [
|
||||
'city fix'
|
||||
]
|
||||
|
||||
MATURE_TAGS = [
|
||||
'porn',
|
||||
'nsfw',
|
||||
'mature',
|
||||
'xxx'
|
||||
]
|
||||
|
||||
ORDER_BY = [
|
||||
[
|
||||
"trending_global",
|
||||
"trending_mixed",
|
||||
],
|
||||
[
|
||||
"release_time"
|
||||
],
|
||||
[
|
||||
"effective_amount"
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
def get_args(limit=20):
|
||||
args = []
|
||||
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
|
||||
not_tags_combinations = [MATURE_TAGS, []]
|
||||
for no_totals in [True]:
|
||||
for offset in [0, 100]:
|
||||
for any_tags in any_tags_combinations:
|
||||
for not_tags in not_tags_combinations:
|
||||
for order_by in ORDER_BY:
|
||||
kw = {
|
||||
'order_by': order_by,
|
||||
'offset': offset,
|
||||
'limit': limit,
|
||||
'no_totals': no_totals
|
||||
}
|
||||
if not_tags:
|
||||
kw['not_tags'] = not_tags
|
||||
if any_tags:
|
||||
kw['any_tags'] = any_tags
|
||||
args.append(kw)
|
||||
print(len(args), "argument combinations")
|
||||
return args
|
||||
|
||||
|
||||
def _search(kwargs):
|
||||
start = time.time()
|
||||
try:
|
||||
search_to_bytes(kwargs)
|
||||
t = time.time() - start
|
||||
return t, kwargs
|
||||
except Exception as err:
|
||||
return -1, f"failed: error={str(type(err))}({str(err)})"
|
||||
|
||||
|
||||
async def search(executor, kwargs):
|
||||
try:
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
executor, _search, kwargs
|
||||
)
|
||||
except Exception as err:
|
||||
return f"failed (err={str(type(err))}({err}))- {kwargs}"
|
||||
|
||||
|
||||
async def main(db_path, max_query_time):
|
||||
args = dict(initializer=initializer, initargs=(log, db_path, MainNetLedger, 0.25))
|
||||
workers = max(os.cpu_count(), 4)
|
||||
log.info(f"using {workers} reader processes")
|
||||
query_executor = ProcessPoolExecutor(workers, **args)
|
||||
tasks = [search(query_executor, constraints) for constraints in get_args()]
|
||||
try:
|
||||
results = await asyncio.gather(*tasks)
|
||||
for ts, constraints in results:
|
||||
if ts >= max_query_time:
|
||||
sql = interpolate(*_get_claims("""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
""", **constraints))
|
||||
print(f"Query took {int(ts * 1000)}ms\n{sql}")
|
||||
finally:
|
||||
query_executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
|
||||
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
|
||||
args = parser.parse_args()
|
||||
db_path = args.db_path
|
||||
max_query_time = args.max_time
|
||||
asyncio.run(main(db_path, max_query_time))
|
||||
@@ -1,107 +0,0 @@
|
||||
import time
|
||||
import asyncio
|
||||
import random
|
||||
from argparse import ArgumentParser
|
||||
from torba.client.basenetwork import ClientSession
|
||||
|
||||
|
||||
class AgentSmith(ClientSession):
|
||||
|
||||
async def do_nefarious_things(self):
|
||||
await self.send_request('blockchain.claimtrie.search', {
|
||||
'no_totals': True,
|
||||
'offset': random.choice(range(0, 300, 20)),
|
||||
'limit': 20,
|
||||
'any_tags': (
|
||||
random.choice([[
|
||||
random.choice(['gaming', 'games', 'game']) +
|
||||
random.choice(['entertainment', 'playthrough', 'funny']) +
|
||||
random.choice(['xbox', 'xbox one', 'xbox news'])
|
||||
], [
|
||||
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
|
||||
random.choice(['news', 'sighting', 'sightings'])
|
||||
], [
|
||||
random.choice(['art', 'automotive']),
|
||||
random.choice(['blockchain', 'economics', 'food']),
|
||||
random.choice(['funny', 'learnings', 'nature']),
|
||||
random.choice(['news', 'science', 'technology'])
|
||||
]
|
||||
])
|
||||
),
|
||||
'not_tags': random.choice([[], [
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
]]),
|
||||
'order_by': random.choice([
|
||||
['release_time'],
|
||||
['trending_global', 'trending_mixed'],
|
||||
['effective_amount']
|
||||
])
|
||||
})
|
||||
|
||||
|
||||
class AgentSmithProgram:
|
||||
|
||||
def __init__(self, host, port):
|
||||
self.host, self.port = host, port
|
||||
self.agent_smiths = []
|
||||
|
||||
async def make_one_more_of_them(self):
|
||||
smith = AgentSmith(network=None, server=(self.host, self.port))
|
||||
await smith.create_connection()
|
||||
self.agent_smiths.append(smith)
|
||||
|
||||
async def coordinate_nefarious_activity(self):
|
||||
start = time.perf_counter()
|
||||
await asyncio.gather(
|
||||
*(s.do_nefarious_things() for s in self.agent_smiths),
|
||||
return_exceptions=True
|
||||
)
|
||||
return time.perf_counter() - start
|
||||
|
||||
def __len__(self):
|
||||
return len(self.agent_smiths)
|
||||
|
||||
async def delete_one_smith(self):
|
||||
if self.agent_smiths:
|
||||
await self.agent_smiths.pop().close()
|
||||
|
||||
async def delete_program(self):
|
||||
await asyncio.gather(*(
|
||||
s.close() for s in self.agent_smiths
|
||||
))
|
||||
|
||||
|
||||
async def main(host, port):
|
||||
smiths = AgentSmithProgram(host, port)
|
||||
await smiths.make_one_more_of_them()
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
ease_off = 0
|
||||
for i in range(1000):
|
||||
await asyncio.sleep(1)
|
||||
if activity.done() and activity.result() < .9:
|
||||
print('more, more, more...')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
|
||||
))
|
||||
else:
|
||||
print('!!!!!!!!!!!!!!')
|
||||
print('IS NEO LOSING?')
|
||||
print('!!!!!!!!!!!!!!')
|
||||
await asyncio.gather(*(
|
||||
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
|
||||
))
|
||||
print(f'coordinate all {len(smiths)} smiths to action')
|
||||
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
|
||||
print('finishing up any remaining actions')
|
||||
await activity
|
||||
print('neo has won, deleting agents...')
|
||||
await smiths.delete_program()
|
||||
print('done.')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('--host', dest='host', default='localhost', type=str)
|
||||
parser.add_argument('--port', dest='port', default=50001, type=int)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.host, args.port))
|
||||
+61
-32
@@ -74,19 +74,28 @@ def get_release_text(desc: str):
|
||||
yield line[len(RELEASE_TEXT):]
|
||||
|
||||
|
||||
def get_previous_final(repo, current_release):
|
||||
assert current_release.rc is not None, "Need an rc to find the previous final release."
|
||||
previous = None
|
||||
for tag in repo.tags(current_release.rc+1):
|
||||
previous = tag
|
||||
return previous
|
||||
|
||||
|
||||
class Version:
|
||||
|
||||
def __init__(self, major=0, minor=0, micro=0):
|
||||
def __init__(self, major=0, minor=0, micro=0, rc=None):
|
||||
self.major = int(major)
|
||||
self.minor = int(minor)
|
||||
self.micro = int(micro)
|
||||
self.rc = rc if rc is None else int(rc)
|
||||
|
||||
@classmethod
|
||||
def from_string(cls, version_string):
|
||||
(major, minor, micro), rc = version_string.split('.'), None
|
||||
if 'rc' in micro:
|
||||
micro, rc = micro.split('rc')
|
||||
return cls(major, minor, micro)
|
||||
return cls(major, minor, micro, rc)
|
||||
|
||||
@classmethod
|
||||
def from_content(cls, content):
|
||||
@@ -97,12 +106,20 @@ class Version:
|
||||
def increment(self, action):
|
||||
cls = self.__class__
|
||||
|
||||
if action == 'major':
|
||||
return cls(self.major+1)
|
||||
elif action == 'minor':
|
||||
return cls(self.major, self.minor+1)
|
||||
elif action == 'micro':
|
||||
return cls(self.major, self.minor, self.micro+1)
|
||||
if action == '*-rc':
|
||||
assert self.rc is not None, f"Can't drop rc designation because {self} is already not an rc."
|
||||
return cls(self.major, self.minor, self.micro)
|
||||
elif action == '*+rc':
|
||||
assert self.rc is not None, "Must already be an rc to increment."
|
||||
return cls(self.major, self.minor, self.micro, self.rc+1)
|
||||
|
||||
assert self.rc is None, f"Can't start a new rc because {self} is already an rc."
|
||||
if action == 'major+rc':
|
||||
return cls(self.major+1, rc=1)
|
||||
elif action == 'minor+rc':
|
||||
return cls(self.major, self.minor+1, rc=1)
|
||||
elif action == 'micro+rc':
|
||||
return cls(self.major, self.minor, self.micro+1, 1)
|
||||
|
||||
raise ValueError(f'unknown action: {action}')
|
||||
|
||||
@@ -111,7 +128,10 @@ class Version:
|
||||
return f'v{self}'
|
||||
|
||||
def __str__(self):
|
||||
return '.'.join(str(p) for p in [self.major, self.minor, self.micro])
|
||||
version = '.'.join(str(p) for p in [self.major, self.minor, self.micro])
|
||||
if self.rc is not None:
|
||||
version += f'rc{self.rc}'
|
||||
return version
|
||||
|
||||
|
||||
def release(args):
|
||||
@@ -119,15 +139,15 @@ def release(args):
|
||||
repo = gh.repository('lbryio', 'lbry-sdk')
|
||||
version_file = repo.file_contents('lbry/lbry/__init__.py')
|
||||
|
||||
if not args.confirm:
|
||||
print("\nDRY RUN ONLY. RUN WITH --confirm TO DO A REAL RELEASE.\n")
|
||||
|
||||
current_version = Version.from_content(version_file)
|
||||
print(f'Current Version: {current_version}')
|
||||
new_version = current_version.increment(args.action)
|
||||
print(f' New Version: {new_version}')
|
||||
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
if args.action == '*-rc':
|
||||
previous_release = repo.release_from_tag(args.start_tag or get_previous_final(repo, current_version))
|
||||
else:
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
|
||||
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
|
||||
print()
|
||||
@@ -145,7 +165,7 @@ def release(args):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
for release_text in get_release_text(pr.body):
|
||||
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
|
||||
if type_label != 'fixup':
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
else:
|
||||
@@ -183,7 +203,7 @@ def release(args):
|
||||
for skipped in unlabeled:
|
||||
print(skipped)
|
||||
|
||||
if args.confirm:
|
||||
if not args.dry_run:
|
||||
|
||||
commit = version_file.update(
|
||||
new_version.tag,
|
||||
@@ -203,10 +223,9 @@ def release(args):
|
||||
name=new_version.tag,
|
||||
body=body.getvalue(),
|
||||
draft=True,
|
||||
prerelease=new_version.rc is not None
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class TestReleaseTool(unittest.TestCase):
|
||||
|
||||
@@ -216,34 +235,44 @@ class TestReleaseTool(unittest.TestCase):
|
||||
|
||||
def test_version_increment(self):
|
||||
v = Version.from_string('1.2.3')
|
||||
self.assertTrue(str(v.increment('major')), '2.0.0')
|
||||
self.assertTrue(str(v.increment('minor')), '1.3.0')
|
||||
self.assertTrue(str(v.increment('micro')), '1.2.4')
|
||||
self.assertTrue(str(v.increment('major+rc')), '2.0.0rc1')
|
||||
self.assertTrue(str(v.increment('minor+rc')), '1.3.0rc1')
|
||||
self.assertTrue(str(v.increment('micro+rc')), '1.2.4rc1')
|
||||
with self.assertRaisesRegex(AssertionError, "Must already be an rc to increment."):
|
||||
v.increment('*+rc')
|
||||
with self.assertRaisesRegex(AssertionError, "Can't drop rc designation"):
|
||||
v.increment('*-rc')
|
||||
|
||||
v = Version.from_string('1.2.3rc3')
|
||||
self.assertTrue(str(v.increment('*+rc')), '1.2.3rc4')
|
||||
self.assertTrue(str(v.increment('*-rc')), '1.2.3')
|
||||
with self.assertRaisesRegex(AssertionError, "already an rc"):
|
||||
v.increment('major+rc')
|
||||
with self.assertRaisesRegex(AssertionError, "already an rc"):
|
||||
v.increment('minor+rc')
|
||||
with self.assertRaisesRegex(AssertionError, "already an rc"):
|
||||
v.increment('micro+rc')
|
||||
|
||||
|
||||
def test():
|
||||
runner = unittest.TextTestRunner(verbosity=2)
|
||||
loader = unittest.TestLoader()
|
||||
suite = loader.loadTestsFromTestCase(TestReleaseTool)
|
||||
return 0 if runner.run(suite).wasSuccessful() else 1
|
||||
runner.run(suite)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--confirm", default=False, action="store_true",
|
||||
help="without this flag, it will only print what it will do but will not actually do it")
|
||||
parser.add_argument("--test", default=False, action="store_true", help="run unit tests")
|
||||
parser.add_argument("--dry-run", default=False, action="store_true", help="show what will be done")
|
||||
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
|
||||
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
|
||||
parser.add_argument("action", nargs="?", choices=['major+rc', 'minor+rc', 'micro+rc', '*+rc', '*-rc'])
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.action == "test":
|
||||
code = test()
|
||||
if args.test:
|
||||
test()
|
||||
else:
|
||||
code = release(args)
|
||||
|
||||
print()
|
||||
return code
|
||||
release(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
main()
|
||||
@@ -1,60 +1,81 @@
|
||||
import uvloop, asyncio, time, sys, logging
|
||||
import uvloop, asyncio, sqlite3, time, sys
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.metrics import calculate_avg_percentiles
|
||||
from contextvars import ContextVar
|
||||
|
||||
|
||||
db_path = '../../../lbryconf/wallet-server/claims.db'
|
||||
default_query_timout = 0.25
|
||||
log = logging.getLogger(__name__)
|
||||
log.addHandler(logging.StreamHandler())
|
||||
db_path = '/tmp/wallet-server/claims.db'
|
||||
db = ContextVar('db')
|
||||
|
||||
|
||||
def init():
|
||||
conn = sqlite3.connect(db_path)
|
||||
db.set(conn)
|
||||
|
||||
|
||||
def reader():
|
||||
conn = db.get()
|
||||
start = time.time()
|
||||
conn.execute("""
|
||||
SELECT
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
FROM claim
|
||||
LEFT JOIN claimtrie USING (claim_hash)
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
WHERE
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag IN ('alexandria ocasio-cortez', 'Alien', 'alt news', 'art', 'audio',
|
||||
'automotive', 'beliefs', 'blockchain', 'dog grooming', 'economics', 'food',
|
||||
'learning', 'mature', 'nature', 'news', 'physics', 'science', 'technology')
|
||||
)
|
||||
AND NOT EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash AND tag IN ('nsfw', 'xxx', 'mature')
|
||||
)
|
||||
ORDER BY claim.height DESC, claim.normalized ASC
|
||||
LIMIT 20 OFFSET 100
|
||||
""").fetchall()
|
||||
elapsed = time.time() - start
|
||||
return elapsed
|
||||
|
||||
|
||||
async def run_times(executor, iterations, show=True):
|
||||
start = time.perf_counter()
|
||||
timings = await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(
|
||||
executor, reader.search_to_bytes, {
|
||||
'no_totals': True,
|
||||
'offset': 0,
|
||||
'limit': 20,
|
||||
'any_tags': [
|
||||
'ufos', 'city fix'
|
||||
],
|
||||
'not_tags': [
|
||||
'porn', 'mature', 'xxx', 'nsfw'
|
||||
],
|
||||
'order_by': [
|
||||
'release_time'
|
||||
]
|
||||
}
|
||||
) for _ in range(iterations)))
|
||||
timings = [r[1]['execute_query'][0]['total'] for r in timings]
|
||||
total = int((time.perf_counter() - start) * 100)
|
||||
start = time.time()
|
||||
timings = await asyncio.gather(*(
|
||||
asyncio.get_running_loop().run_in_executor(executor, reader) for _ in range(iterations)
|
||||
))
|
||||
total = time.time() - start
|
||||
if show:
|
||||
avg = sum(timings)/len(timings)
|
||||
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
|
||||
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
|
||||
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
|
||||
stats = calculate_avg_percentiles(timings)
|
||||
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
|
||||
print(f"{iterations:4}: {total:.5f}ms total concurrent, {len(timings)*avg:.5f}ms total sequential (avg*runs)")
|
||||
print(f" {total/len(timings):.5f}ms/query concurrent (total/runs)")
|
||||
print(f" {avg:.5f}ms/query actual average (sum(queries)/runs)")
|
||||
sys.stdout.write(' sample:')
|
||||
for i, t in zip(range(10), timings[::-1]):
|
||||
sys.stdout.write(f' {t}ms')
|
||||
sys.stdout.write(f' {t:.5f}ms')
|
||||
print(' ...\n' if len(timings) > 10 else '\n')
|
||||
|
||||
|
||||
async def main():
|
||||
executor = ProcessPoolExecutor(
|
||||
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
|
||||
)
|
||||
#await run_times(executor, 4, show=False)
|
||||
#await run_times(executor, 1)
|
||||
executor = ProcessPoolExecutor(4, initializer=init)
|
||||
await run_times(executor, 4, show=False)
|
||||
await run_times(executor, 1)
|
||||
await run_times(executor, 2**3)
|
||||
await run_times(executor, 2**5)
|
||||
await run_times(executor, 2**7)
|
||||
#await run_times(executor, 2**9)
|
||||
#await run_times(executor, 2**11)
|
||||
#await run_times(executor, 2**13)
|
||||
await run_times(executor, 2**9)
|
||||
await run_times(executor, 2**11)
|
||||
await run_times(executor, 2**13)
|
||||
executor.shutdown(True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -89,7 +89,6 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([claim], **kwargs)
|
||||
|
||||
async def assertFindsClaims(self, claims, **kwargs):
|
||||
kwargs.setdefault('order_by', ['height', '^name'])
|
||||
results = await self.claim_search(**kwargs)
|
||||
self.assertEqual(len(claims), len(results))
|
||||
for claim, result in zip(claims, results):
|
||||
@@ -166,15 +165,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.create_channel()
|
||||
await self.create_lots_of_streams()
|
||||
|
||||
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
|
||||
page = await self.claim_search(page_size=20, channel='@abc')
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams)
|
||||
|
||||
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page = await self.claim_search(page_size=6, channel='@abc')
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[:6])
|
||||
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
|
||||
page = await self.claim_search(page=2, page_size=6, channel='@abc')
|
||||
page_claim_ids = [item['name'] for item in page]
|
||||
self.assertEqual(page_claim_ids, self.streams[6:])
|
||||
|
||||
@@ -226,7 +225,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([claim4, claim3, claim2], all_tags=['abc'], any_tags=['def', 'ghi'])
|
||||
|
||||
async def test_order_by(self):
|
||||
height = self.ledger.network.remote_height
|
||||
height = await self.ledger.network.get_server_height()
|
||||
claims = [await self.stream_create(f'claim{i}') for i in range(5)]
|
||||
|
||||
await self.assertFindsClaims(claims, order_by=["^height"])
|
||||
|
||||
@@ -1,31 +1,130 @@
|
||||
import asyncio
|
||||
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.orchstr8 import Conductor
|
||||
from torba.testcase import IntegrationTestCase
|
||||
import socket
|
||||
import logging
|
||||
from unittest.mock import Mock
|
||||
from torba.testcase import IntegrationTestCase, Conductor
|
||||
import lbry.wallet
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
def wrap_callback_event(fn, callback):
|
||||
def inner(*a, **kw):
|
||||
callback()
|
||||
return fn(*a, **kw)
|
||||
return inner
|
||||
|
||||
|
||||
class TestSessionBloat(IntegrationTestCase):
|
||||
"""
|
||||
Tests that server cleans up stale connections after session timeout and client times out too.
|
||||
ERROR:asyncio:Fatal read error on socket transport
|
||||
protocol: <lbrynet.wallet.server.session.LBRYElectrumX object at 0x7f7e3bfcaf60>
|
||||
transport: <_SelectorSocketTransport fd=3236 read=polling write=<idle, bufsize=0>>
|
||||
Traceback (most recent call last):
|
||||
File "/usr/lib/python3.7/asyncio/selector_events.py", line 801, in _read_ready__data_received
|
||||
data = self._sock.recv(self.max_size)
|
||||
TimeoutError: [Errno 110] Connection timed out
|
||||
"""
|
||||
|
||||
LEDGER = lbry.wallet
|
||||
|
||||
async def test_session_bloat_from_socket_timeout(self):
|
||||
await self.conductor.stop_spv()
|
||||
await self.ledger.stop()
|
||||
self.conductor.spv_node.session_timeout = 1
|
||||
async def asyncSetUp(self):
|
||||
self.conductor = Conductor(
|
||||
ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY
|
||||
)
|
||||
await self.conductor.start_blockchain()
|
||||
self.addCleanup(self.conductor.stop_blockchain)
|
||||
|
||||
await self.conductor.start_spv()
|
||||
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
|
||||
await session.create_connection()
|
||||
session.ping_task.cancel()
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 1)
|
||||
self.assertFalse(session.is_closing())
|
||||
await asyncio.sleep(1.1)
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await session.send_request('server.banner', ())
|
||||
self.assertTrue(session.is_closing())
|
||||
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
|
||||
|
||||
self.session_manager = self.conductor.spv_node.server.session_mgr
|
||||
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64)
|
||||
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 64)
|
||||
|
||||
self.addCleanup(self.conductor.stop_spv)
|
||||
await self.conductor.start_wallet()
|
||||
self.addCleanup(self.conductor.stop_wallet)
|
||||
|
||||
self.client_session = list(self.session_manager.sessions)[0]
|
||||
self.client_session.transport.set_write_buffer_limits(0, 0)
|
||||
|
||||
self.paused_session = asyncio.Event(loop=self.loop)
|
||||
self.resumed_session = asyncio.Event(loop=self.loop)
|
||||
|
||||
def paused():
|
||||
self.resumed_session.clear()
|
||||
self.paused_session.set()
|
||||
|
||||
def delayed_resume():
|
||||
self.paused_session.clear()
|
||||
|
||||
self.resumed_session.set()
|
||||
|
||||
self.client_session.pause_writing = wrap_callback_event(self.client_session.pause_writing, paused)
|
||||
self.client_session.resume_writing = wrap_callback_event(self.client_session.resume_writing, delayed_resume)
|
||||
|
||||
self.blockchain = self.conductor.blockchain_node
|
||||
self.wallet_node = self.conductor.wallet_node
|
||||
self.manager = self.wallet_node.manager
|
||||
self.ledger = self.wallet_node.ledger
|
||||
self.wallet = self.wallet_node.wallet
|
||||
self.account = self.wallet_node.wallet.default_account
|
||||
|
||||
async def test_session_bloat_from_socket_timeout(self):
|
||||
await self.account.ensure_address_gap()
|
||||
|
||||
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
|
||||
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
|
||||
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
|
||||
|
||||
await self.blockchain.generate(1)
|
||||
await asyncio.wait([
|
||||
self.on_transaction_id(sendtxid1),
|
||||
self.on_transaction_id(sendtxid2)
|
||||
])
|
||||
|
||||
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
|
||||
|
||||
channel = Claim()
|
||||
channel_txo = Output.pay_claim_name_pubkey_hash(
|
||||
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
|
||||
)
|
||||
channel_txo.generate_channel_private_key()
|
||||
channel_txo.script.generate()
|
||||
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
|
||||
|
||||
stream = Claim()
|
||||
stream.stream.description = "0" * 8000
|
||||
stream_txo = Output.pay_claim_name_pubkey_hash(
|
||||
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
|
||||
)
|
||||
stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account)
|
||||
stream_txo.sign(channel_txo)
|
||||
await stream_tx.sign([self.account])
|
||||
self.paused_session.clear()
|
||||
self.resumed_session.clear()
|
||||
|
||||
await self.broadcast(channel_tx)
|
||||
await self.broadcast(stream_tx)
|
||||
await asyncio.wait_for(self.paused_session.wait(), 2)
|
||||
self.assertEqual(1, len(self.session_manager.sessions))
|
||||
|
||||
real_sock = self.client_session.transport._extra.pop('socket')
|
||||
mock_sock = Mock(spec=socket.socket)
|
||||
|
||||
for attr in dir(real_sock):
|
||||
if not attr.startswith('__'):
|
||||
setattr(mock_sock, attr, getattr(real_sock, attr))
|
||||
|
||||
def recv(*a, **kw):
|
||||
raise TimeoutError("[Errno 110] Connection timed out")
|
||||
|
||||
mock_sock.recv = recv
|
||||
self.client_session.transport._sock = mock_sock
|
||||
self.client_session.transport._extra['socket'] = mock_sock
|
||||
self.assertFalse(self.resumed_session.is_set())
|
||||
self.assertFalse(self.session_manager.session_event.is_set())
|
||||
await self.session_manager.session_event.wait()
|
||||
self.assertEqual(0, len(self.session_manager.sessions))
|
||||
@@ -17,18 +17,6 @@ class TestBlobManager(AsyncioTestCase):
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
|
||||
for save_blobs in (False, True):
|
||||
await self.setup_blob_manager(save_blobs=save_blobs)
|
||||
# add a blob file
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
blob.save_verified_blob(blob_bytes)
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.blob_manager.blob_completed(blob)
|
||||
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
|
||||
|
||||
async def test_sync_blob_file_manager_on_startup(self):
|
||||
await self.setup_blob_manager(save_blobs=True)
|
||||
|
||||
|
||||
@@ -72,13 +72,6 @@ class TestLanguages(TestCase):
|
||||
self.assertEqual(stream.languages[2].region, 'US')
|
||||
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US'])
|
||||
|
||||
stream.languages.append('es-419')
|
||||
self.assertEqual(stream.languages[3].langtag, 'es-419')
|
||||
self.assertEqual(stream.languages[3].language, 'es')
|
||||
self.assertEqual(stream.languages[3].script, None)
|
||||
self.assertEqual(stream.languages[3].region, '419')
|
||||
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US', 'es-419'])
|
||||
|
||||
stream = Stream()
|
||||
stream.languages.extend(['en-Latn-US', 'es-ES', 'de-DE'])
|
||||
self.assertEqual(stream.languages[0].language, 'en')
|
||||
|
||||
@@ -11,13 +11,9 @@ class TestTagNormalization(unittest.TestCase):
|
||||
def test_normalize_tag(self):
|
||||
tag = self.assertNormalizedTag
|
||||
tag('', ' \t #!~')
|
||||
tag('tag', 'T\'ag')
|
||||
tag('tag', 'Tag')
|
||||
tag('t ag', '\tT \nAG ')
|
||||
tag('tag hash', '#tag~#hash!')
|
||||
|
||||
def test_clean_tags(self):
|
||||
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))
|
||||
cleaned = clean_tags(['fOo', '!taG', 'FoO'])
|
||||
self.assertIn('tag', cleaned)
|
||||
self.assertIn('foo', cleaned)
|
||||
self.assertEqual(len(cleaned), 2)
|
||||
@@ -40,15 +40,6 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir
|
||||
)
|
||||
|
||||
async def test_status_file_completed(self):
|
||||
await self._test_transfer_stream(10)
|
||||
self.assertTrue(self.stream.output_file_exists)
|
||||
self.assertTrue(self.stream.as_dict()['completed'])
|
||||
with open(self.stream.full_path, 'w+b') as outfile:
|
||||
outfile.truncate(1)
|
||||
self.assertTrue(self.stream.output_file_exists)
|
||||
self.assertFalse(self.stream.as_dict()['completed'])
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
|
||||
await self.setup_stream(blob_count)
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
@@ -61,7 +52,7 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
await self.stream.save_file(node=mock_node)
|
||||
await self.stream.finished_write_attempt.wait()
|
||||
await self.stream.finished_writing.wait()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
if stop_when_done:
|
||||
await self.stream.stop()
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
import time
|
||||
import unittest
|
||||
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
|
||||
|
||||
|
||||
class TestPercentileCalculation(unittest.TestCase):
|
||||
|
||||
def test_calculate_percentiles(self):
|
||||
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
|
||||
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
|
||||
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
|
||||
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
|
||||
self.assertEqual(calculate_avg_percentiles(
|
||||
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
|
||||
|
||||
|
||||
class TestCollectingMetrics(unittest.TestCase):
|
||||
|
||||
def test_happy_path(self):
|
||||
self.maxDiff = None
|
||||
load = ServerLoadData()
|
||||
search = load.for_api('search')
|
||||
self.assertEqual(search.name, 'search')
|
||||
search.start()
|
||||
search.cache_response()
|
||||
search.cache_response()
|
||||
metrics = {
|
||||
'search': [{'total': 40}],
|
||||
'execute_query': [
|
||||
{'total': 20},
|
||||
{'total': 10}
|
||||
]
|
||||
}
|
||||
for x in range(5):
|
||||
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
|
||||
metrics['execute_query'][0]['total'] = 10
|
||||
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
|
||||
search.query_interrupt(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.050, metrics)
|
||||
search.query_error(time.perf_counter() - 0.052, {})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
|
||||
"receive_count": 1,
|
||||
"cache_response_count": 2,
|
||||
"query_response_count": 5,
|
||||
"intrp_response_count": 1,
|
||||
"error_response_count": 2,
|
||||
"response": (53, 51, 51, 52, 53, 54, 55, 55),
|
||||
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
|
||||
"error": (51, 50, 50, 50, 50, 52, 52, 52),
|
||||
"python": (12, 10, 10, 10, 10, 20, 20, 20),
|
||||
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
|
||||
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
|
||||
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
|
||||
"individual_sql_count": 14,
|
||||
"errored_queries": ['FROM claim where something=1'],
|
||||
"interrupted_queries": ['FROM claim where something=1'],
|
||||
}}})
|
||||
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
|
||||
@@ -1,7 +1,6 @@
|
||||
import unittest
|
||||
import ecdsa
|
||||
import hashlib
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from torba.client.constants import COIN, NULL_HASH32
|
||||
|
||||
@@ -37,7 +36,6 @@ class OldWalletServerTransaction:
|
||||
|
||||
|
||||
class TestSQLDB(unittest.TestCase):
|
||||
query_timeout = 0.25
|
||||
|
||||
def setUp(self):
|
||||
self.first_sync = False
|
||||
@@ -46,7 +44,7 @@ class TestSQLDB(unittest.TestCase):
|
||||
db_url = 'file:test_sqldb?mode=memory&cache=shared'
|
||||
self.sql = writer.SQLDB(self, db_url)
|
||||
self.addCleanup(self.sql.close)
|
||||
reader.initializer(logging.getLogger(__name__), db_url, 'regtest', self.query_timeout)
|
||||
reader.initializer(db_url, 'regtest')
|
||||
self.addCleanup(reader.cleanup)
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.sql.open()
|
||||
|
||||
+55
-35
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import socket
|
||||
from unittest.mock import Mock
|
||||
|
||||
from torba.client.basenetwork import BaseNetwork
|
||||
@@ -7,15 +8,6 @@ from torba.rpc import RPCSession
|
||||
from torba.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
|
||||
|
||||
class NetworkTests(IntegrationTestCase):
|
||||
|
||||
async def test_remote_height_updated_automagically(self):
|
||||
initial_height = self.ledger.network.remote_height
|
||||
await self.blockchain.generate(1)
|
||||
await self.ledger.network.on_header.first
|
||||
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
@@ -51,47 +43,75 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_socket_timeout_then_reconnect(self):
|
||||
# TODO: test reconnecting on an rpc request
|
||||
# TODO: test rolling over to a working server when an rpc request fails before raising
|
||||
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
txid = await self.blockchain.send_to_address(address1, 21)
|
||||
await self.blockchain.generate(1)
|
||||
|
||||
real_sock = self.ledger.network.client.transport._extra.pop('socket')
|
||||
mock_sock = Mock(spec=socket.socket)
|
||||
|
||||
for attr in dir(real_sock):
|
||||
if not attr.startswith('__'):
|
||||
setattr(mock_sock, attr, getattr(real_sock, attr))
|
||||
|
||||
raised = asyncio.Event(loop=self.loop)
|
||||
|
||||
def recv(*a, **kw):
|
||||
raised.set()
|
||||
raise TimeoutError("[Errno 60] Operation timed out")
|
||||
|
||||
mock_sock.recv = recv
|
||||
self.ledger.network.client.transport._sock = mock_sock
|
||||
self.ledger.network.client.transport._extra['socket'] = mock_sock
|
||||
|
||||
self.assertFalse(raised.is_set())
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await self.ledger.network.get_transaction(txid)
|
||||
|
||||
self.assertTrue(raised.is_set())
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
self.assertIsNone(self.ledger.network.client.transport)
|
||||
|
||||
txid = await self.blockchain.send_to_address(address1, 2)
|
||||
await self.blockchain.generate(1)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
self.assertIsNotNone(self.ledger.network.client.transport)
|
||||
self.assertIsNotNone(await self.ledger.network.get_transaction(txid))
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
async def _make_fake_server(self, latency=1.0, port=1337):
|
||||
# local fake server with artificial latency
|
||||
class FakeSession(RPCSession):
|
||||
async def handle_request(self, request):
|
||||
await asyncio.sleep(latency)
|
||||
return {"height": 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
proto = RPCSession()
|
||||
proto.handle_request = lambda _: asyncio.sleep(latency)
|
||||
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
async def echo(reader, writer):
|
||||
while True:
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
return '127.0.0.1', port
|
||||
return ('127.0.0.1', port)
|
||||
|
||||
async def test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
# fast but unhealthy, should be discarded
|
||||
await self._make_bad_server(),
|
||||
('localhost', 1),
|
||||
('example.that.doesnt.resolve', 9000),
|
||||
await self._make_fake_server(latency=1.2, port=1340),
|
||||
await self._make_fake_server(latency=0.5, port=1337),
|
||||
await self._make_fake_server(latency=0.7, port=1339),
|
||||
await self._make_fake_server(latency=1.5, port=1340),
|
||||
await self._make_fake_server(latency=0.1, port=1337),
|
||||
await self._make_fake_server(latency=1.0, port=1339),
|
||||
await self._make_fake_server(latency=0.5, port=1338),
|
||||
],
|
||||
'connect_timeout': 3
|
||||
'connect_timeout': 30
|
||||
})
|
||||
|
||||
network = BaseNetwork(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=3)
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
# ensure we are connected to all of them
|
||||
self.assertEqual(len(network.session_pool.sessions), 4)
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
|
||||
self.assertEqual(len(network.session_pool.sessions), 3)
|
||||
@@ -1,10 +1,11 @@
|
||||
import unittest
|
||||
import sqlite3
|
||||
from functools import wraps
|
||||
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import COIN
|
||||
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
|
||||
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
|
||||
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
|
||||
from torba.client.hash import sha256
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
@@ -166,26 +167,6 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
("select * from foo LIMIT 20 OFFSET 10", {})
|
||||
)
|
||||
|
||||
def test_query_interpolation(self):
|
||||
self.maxDiff = None
|
||||
# tests that interpolation replaces longer keys first
|
||||
self.assertEqual(
|
||||
interpolate(*query(
|
||||
"select * from foo",
|
||||
a__not='b', b__in='select * from blah where c=:$c',
|
||||
d__any={'one__like': 'o', 'two': 2},
|
||||
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
|
||||
ahash=memoryview(sha256(b'hello world')),
|
||||
limit=10, order_by='b', **{'$c': 3})
|
||||
),
|
||||
"select * from foo WHERE a != 'b' AND "
|
||||
"b IN (select * from blah where c=3) AND "
|
||||
"(one LIKE 'o' OR two = 2) AND "
|
||||
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
|
||||
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
|
||||
"ORDER BY b LIMIT 10",
|
||||
)
|
||||
|
||||
|
||||
class TestQueries(AsyncioTestCase):
|
||||
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
@@ -93,42 +91,3 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
self.assertEqual(headers.height, 32250)
|
||||
await headers.connect(len(headers), remainder)
|
||||
self.assertEqual(headers.height, 32259)
|
||||
|
||||
async def test_repair(self):
|
||||
headers = MainHeaders(':memory:')
|
||||
await headers.connect(0, self.get_bytes(block_bytes(3001)))
|
||||
self.assertEqual(headers.height, 3000)
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 3000)
|
||||
# corrupt the middle of it
|
||||
headers.io.seek(block_bytes(1500))
|
||||
headers.io.write(b"wtf")
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 1499)
|
||||
self.assertEqual(len(headers), 1500)
|
||||
# corrupt by appending
|
||||
headers.io.seek(block_bytes(len(headers)))
|
||||
headers.io.write(b"appending")
|
||||
headers._size = None
|
||||
await headers.repair()
|
||||
self.assertEqual(headers.height, 1499)
|
||||
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
|
||||
self.assertEqual(headers.height, 3000)
|
||||
|
||||
async def test_concurrency(self):
|
||||
BLOCKS = 30
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
headers = MainHeaders(headers_temporary_file)
|
||||
await headers.open()
|
||||
self.addCleanup(os.remove, headers_temporary_file)
|
||||
async def writer():
|
||||
for block_index in range(BLOCKS):
|
||||
await headers.connect(block_index, self.get_bytes(block_bytes(block_index + 1), block_bytes(block_index)))
|
||||
async def reader():
|
||||
for block_index in range(BLOCKS):
|
||||
while len(headers) < block_index:
|
||||
await asyncio.sleep(0.000001)
|
||||
assert headers[block_index]['block_height'] == block_index
|
||||
reader_task = asyncio.create_task(reader())
|
||||
await writer()
|
||||
await reader_task
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from asyncio import wrap_future
|
||||
from binascii import hexlify
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
|
||||
@@ -192,19 +191,6 @@ def query(select, **constraints):
|
||||
return ' '.join(sql), values
|
||||
|
||||
|
||||
def interpolate(sql, values):
|
||||
for k in sorted(values.keys(), reverse=True):
|
||||
value = values[k]
|
||||
if isinstance(value, memoryview):
|
||||
value = f"X'{hexlify(bytes(value)).decode()}'"
|
||||
elif isinstance(value, str):
|
||||
value = f"'{value}'"
|
||||
else:
|
||||
value = str(value)
|
||||
sql = sql.replace(f":{k}", value)
|
||||
return sql
|
||||
|
||||
|
||||
def rows_to_dict(rows, fields):
|
||||
if rows:
|
||||
return [dict(zip(fields, r)) for r in rows]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple
|
||||
@@ -34,6 +35,7 @@ class BaseHeaders:
|
||||
self.io = BytesIO()
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self._header_connect_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if self.path != ':memory:':
|
||||
@@ -82,10 +84,6 @@ class BaseHeaders:
|
||||
def height(self) -> int:
|
||||
return len(self)-1
|
||||
|
||||
@property
|
||||
def bytes_size(self):
|
||||
return len(self) * self.header_size
|
||||
|
||||
def hash(self, height=None) -> bytes:
|
||||
return self.hash_header(
|
||||
self.get_raw_header(height if height is not None else self.height)
|
||||
@@ -100,25 +98,27 @@ class BaseHeaders:
|
||||
async def connect(self, start: int, headers: bytes) -> int:
|
||||
added = 0
|
||||
bail = False
|
||||
for height, chunk in self._iterate_chunks(start, headers):
|
||||
try:
|
||||
# validate_chunk() is CPU bound and reads previous chunks from file system
|
||||
self.validate_chunk(height, chunk)
|
||||
except InvalidHeader as e:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
written = 0
|
||||
if chunk:
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(chunk) // self.header_size
|
||||
self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
added += written
|
||||
if bail:
|
||||
break
|
||||
loop = asyncio.get_running_loop()
|
||||
async with self._header_connect_lock:
|
||||
for height, chunk in self._iterate_chunks(start, headers):
|
||||
try:
|
||||
# validate_chunk() is CPU bound and reads previous chunks from file system
|
||||
await loop.run_in_executor(None, self.validate_chunk, height, chunk)
|
||||
except InvalidHeader as e:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
written = 0
|
||||
if chunk:
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(chunk) // self.header_size
|
||||
self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
await loop.run_in_executor(None, self.io.flush)
|
||||
self._size = None
|
||||
added += written
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
|
||||
def validate_chunk(self, height, chunk):
|
||||
@@ -168,35 +168,12 @@ class BaseHeaders:
|
||||
proof_of_work.value, target.value)
|
||||
)
|
||||
|
||||
async def repair(self):
|
||||
previous_header_hash = fail = None
|
||||
self.io.seek(0)
|
||||
batch_size = 10000
|
||||
for start_height in range(0, self.height, batch_size):
|
||||
headers = self.io.read(self.header_size*batch_size)
|
||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
||||
height = header['block_height']
|
||||
if height:
|
||||
if header['prev_block_hash'] != previous_header_hash:
|
||||
fail = True
|
||||
else:
|
||||
if header_hash != self.genesis_hash:
|
||||
fail = True
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = None
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@staticmethod
|
||||
def get_proof_of_work(header_hash: bytes) -> ArithUint256:
|
||||
return ArithUint256(int(b'0x' + header_hash, 16))
|
||||
|
||||
def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]:
|
||||
assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}"
|
||||
assert len(headers) % self.header_size == 0
|
||||
start = 0
|
||||
end = (self.chunk_size - height % self.chunk_size) * self.header_size
|
||||
while start < end:
|
||||
|
||||
@@ -263,8 +263,8 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
|
||||
async def join_network(self, *args):
|
||||
log.info("Subscribing and updating accounts.")
|
||||
async with self._header_processing_lock:
|
||||
await self.update_headers()
|
||||
await self.update_headers()
|
||||
await self.network.subscribe_headers()
|
||||
await self.subscribe_accounts()
|
||||
await self._update_tasks.done.wait()
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import socket
|
||||
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
|
||||
|
||||
from torba import __version__
|
||||
from torba.rpc.util import protocol_exception_handler
|
||||
from torba.stream import StreamController
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -15,36 +16,28 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
|
||||
def __init__(self, *args, network, server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network, server, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
self.max_seconds_idle = 60
|
||||
self.ping_task = None
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
try:
|
||||
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
|
||||
return await super().send_request(method, args)
|
||||
except RPCError as e:
|
||||
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
|
||||
raise e
|
||||
except asyncio.TimeoutError:
|
||||
self.abort()
|
||||
raise
|
||||
|
||||
async def ping_forever(self):
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
while not self.is_closing():
|
||||
if (time() - self.last_send) > self.max_seconds_idle:
|
||||
try:
|
||||
await self.send_request('server.banner')
|
||||
except:
|
||||
self.abort()
|
||||
raise
|
||||
await self.send_request('server.banner')
|
||||
await asyncio.sleep(self.max_seconds_idle//3)
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
@@ -66,11 +59,11 @@ class ClientSession(BaseClientSession):
|
||||
class BaseNetwork:
|
||||
|
||||
def __init__(self, ledger):
|
||||
asyncio.get_event_loop().set_exception_handler(protocol_exception_handler)
|
||||
self.config = ledger.config
|
||||
self.client: ClientSession = None
|
||||
self.session_pool: SessionPool = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
|
||||
self._on_connected_controller = StreamController()
|
||||
self.on_connected = self._on_connected_controller.stream
|
||||
@@ -91,13 +84,11 @@ class BaseNetwork:
|
||||
connect_timeout = self.config.get('connect_timeout', 6)
|
||||
self.session_pool = SessionPool(network=self, timeout=connect_timeout)
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
while True:
|
||||
try:
|
||||
self.client = await self.pick_fastest_session()
|
||||
if self.is_connected:
|
||||
await self.ensure_server_version()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
log.info("Successfully connected to SPV wallet server: %s:%d", *self.client.server)
|
||||
self._on_connected_controller.add(True)
|
||||
await self.client.on_disconnected.first
|
||||
@@ -147,9 +138,6 @@ class BaseNetwork:
|
||||
await session.send_request('server.banner')
|
||||
return session
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
def ensure_server_version(self, required='1.2'):
|
||||
return self.rpc('server.version', [__version__, required])
|
||||
|
||||
@@ -227,14 +215,10 @@ class SessionPool:
|
||||
self._dead_servers = []
|
||||
|
||||
async def ensure_connection(self, session):
|
||||
self._dead_servers.append(session)
|
||||
self.sessions.remove(session)
|
||||
if not session.is_closing():
|
||||
return
|
||||
try:
|
||||
if session.is_closing():
|
||||
await session.create_connection(self.timeout)
|
||||
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
|
||||
self.sessions.append(session)
|
||||
self._dead_servers.remove(session)
|
||||
return await session.create_connection(self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timeout connecting to %s:%d", *session.server)
|
||||
except asyncio.CancelledError: # pylint: disable=try-except-raise
|
||||
@@ -246,9 +230,11 @@ class SessionPool:
|
||||
log.warning("Could not connect to %s:%d", *session.server)
|
||||
else:
|
||||
log.exception("Connecting to %s:%d raised an exception:", *session.server)
|
||||
self._dead_servers.append(session)
|
||||
self.sessions.remove(session)
|
||||
|
||||
async def get_online_sessions(self):
|
||||
self._lost_master.set()
|
||||
while not self.online:
|
||||
self._lost_master.set()
|
||||
await asyncio.sleep(0.5)
|
||||
await asyncio.sleep(0.1)
|
||||
return self.sessions
|
||||
@@ -195,7 +195,6 @@ class SPVNode:
|
||||
self.server = None
|
||||
self.hostname = 'localhost'
|
||||
self.port = 50001 + 1 # avoid conflict with default daemon
|
||||
self.session_timeout = 600
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode'):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
@@ -205,9 +204,7 @@ class SPVNode:
|
||||
'REORG_LIMIT': '100',
|
||||
'HOST': self.hostname,
|
||||
'TCP_PORT': str(self.port),
|
||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
'MAX_QUERY_WORKERS': '0'
|
||||
}
|
||||
# TODO: don't use os.environ
|
||||
os.environ.update(conf)
|
||||
@@ -266,7 +263,6 @@ class BlockchainNode:
|
||||
self.data_path = None
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.blockchain_address = None
|
||||
self._block_expected = 0
|
||||
self.hostname = 'localhost'
|
||||
self.peerport = 9246 + 2 # avoid conflict with default peer port
|
||||
@@ -330,7 +326,7 @@ class BlockchainNode:
|
||||
self.daemon_bin,
|
||||
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
|
||||
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
|
||||
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
|
||||
f'-port={self.peerport}'
|
||||
)
|
||||
self.log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
@@ -365,10 +361,9 @@ class BlockchainNode:
|
||||
self.log.info(out.decode().strip())
|
||||
return out.decode().strip()
|
||||
|
||||
async def generate(self, blocks, address=None):
|
||||
def generate(self, blocks):
|
||||
self._block_expected += blocks
|
||||
address = await self.get_raw_change_address()
|
||||
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
|
||||
return self._cli_cmnd('generate', str(blocks))
|
||||
|
||||
def invalidate_block(self, blockhash):
|
||||
return self._cli_cmnd('invalidateblock', blockhash)
|
||||
@@ -377,7 +372,7 @@ class BlockchainNode:
|
||||
return self._cli_cmnd('getblockhash', str(block))
|
||||
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress', 'legacy')
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
|
||||
async def get_balance(self):
|
||||
return float(await self._cli_cmnd('getbalance'))
|
||||
|
||||
@@ -165,8 +165,6 @@ class JSONRPC(object):
|
||||
METHOD_NOT_FOUND = -32601
|
||||
INVALID_ARGS = -32602
|
||||
INTERNAL_ERROR = -32603
|
||||
QUERY_TIMEOUT = -32000
|
||||
|
||||
# Codes specific to this library
|
||||
ERROR_CODE_UNAVAILABLE = -100
|
||||
|
||||
|
||||
@@ -93,3 +93,13 @@ class Concurrency(object):
|
||||
else:
|
||||
for _ in range(-diff):
|
||||
await self.semaphore.acquire()
|
||||
|
||||
|
||||
def protocol_exception_handler(loop: asyncio.AbstractEventLoop, context):
|
||||
message = context['message']
|
||||
transport = context.get('transport')
|
||||
if transport and message in ("Fatal read error on socket transport", "Fatal write error on socket transport"):
|
||||
transport.abort()
|
||||
transport.close()
|
||||
else:
|
||||
return loop.default_exception_handler(context)
|
||||
@@ -57,11 +57,6 @@ class Daemon:
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
await self.connector.close()
|
||||
self.connector = None
|
||||
|
||||
def set_url(self, url):
|
||||
"""Set the URLS to the given list, and switch to the first one."""
|
||||
urls = url.split(',')
|
||||
@@ -98,8 +93,6 @@ class Daemon:
|
||||
return aiohttp.ClientSession(connector=self.connector, connector_owner=False)
|
||||
|
||||
async def _send_data(self, data):
|
||||
if not self.connector:
|
||||
raise asyncio.CancelledError('Tried to send request during shutdown.')
|
||||
async with self.workqueue_semaphore:
|
||||
async with self.client_session() as session:
|
||||
async with session.post(self.current_url(), data=data) as resp:
|
||||
|
||||
@@ -38,10 +38,6 @@ class Env:
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
|
||||
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
|
||||
self.daemon_url = self.required('DAEMON_URL')
|
||||
if coin is not None:
|
||||
assert issubclass(coin, Coin)
|
||||
@@ -51,6 +47,7 @@ class Env:
|
||||
network = self.default('NET', 'mainnet').strip()
|
||||
self.coin = Coin.lookup_coin_class(coin_name, network)
|
||||
self.cache_MB = self.integer('CACHE_MB', 1200)
|
||||
self.host = self.default('HOST', 'localhost')
|
||||
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
|
||||
# Server stuff
|
||||
self.tcp_port = self.integer('TCP_PORT', None)
|
||||
@@ -87,7 +84,6 @@ class Env:
|
||||
self.identities = [identity
|
||||
for identity in (clearnet_identity, tor_identity)
|
||||
if identity is not None]
|
||||
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 250)) / 1000.0
|
||||
|
||||
@classmethod
|
||||
def default(cls, envvar, default):
|
||||
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
import torba
|
||||
from torba.rpc.util import protocol_exception_handler
|
||||
from torba.server.mempool import MemPool, MemPoolAPI
|
||||
|
||||
|
||||
@@ -89,6 +90,7 @@ class Server:
|
||||
)
|
||||
|
||||
async def start(self):
|
||||
asyncio.get_event_loop().set_exception_handler(protocol_exception_handler)
|
||||
env = self.env
|
||||
min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
|
||||
self.log.info(f'software version: {torba.__version__}')
|
||||
@@ -113,7 +115,6 @@ class Server:
|
||||
task.cancel()
|
||||
await asyncio.wait(self.cancellable_tasks)
|
||||
self.shutdown_event.set()
|
||||
await self.daemon.close()
|
||||
|
||||
def run(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
@@ -255,10 +255,9 @@ class SessionManager:
|
||||
|
||||
async def _clear_stale_sessions(self):
|
||||
"""Cut off sessions that haven't done anything for 10 minutes."""
|
||||
session_timeout = self.env.session_timeout
|
||||
while True:
|
||||
await sleep(session_timeout // 10)
|
||||
stale_cutoff = time.time() - session_timeout
|
||||
await sleep(60)
|
||||
stale_cutoff = time.time() - self.env.session_timeout
|
||||
stale_sessions = [session for session in self.sessions
|
||||
if session.last_recv < stale_cutoff]
|
||||
if stale_sessions:
|
||||
@@ -268,7 +267,7 @@ class SessionManager:
|
||||
# Give the sockets some time to close gracefully
|
||||
if stale_sessions:
|
||||
await asyncio.wait([
|
||||
session.close(force_after=session_timeout // 10) for session in stale_sessions
|
||||
session.close() for session in stale_sessions
|
||||
])
|
||||
|
||||
# Consolidate small groups
|
||||
|
||||
Reference in new issue
Block a user