Compare commits

..
65 changed files with 827 additions and 3834 deletions

No files matched your search

+1 -2
View File
@@ -78,8 +78,7 @@ jobs:
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
deploy:
provider: releases
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
api_key: $GITHUB_OAUTH_TOKEN
file: lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
-4
View File
@@ -9,10 +9,6 @@ lint:
cd torba && pylint --rcfile=setup.cfg torba
cd torba && mypy --ignore-missing-imports torba
test:
cd lbry && tox
cd torba && tox
idea:
mkdir -p .idea
cp -r lbry/scripts/idea/* .idea
+1 -2
View File
@@ -1,6 +1,5 @@
import 'package:lbry/src/load_generator.dart' as load;
//import 'package:lbry/src/server_connection.dart' as load;
main(List<String> arguments) {
load.generate_load();
load.cli();
}
+1 -2
View File
@@ -1,2 +1 @@
export 'src/load_generator.dart';
export 'src/server_connection.dart';
export 'src/load_generator.dart' show LoadGenerator, LoadDataPoint;
+23 -43
View File
@@ -9,7 +9,6 @@ class LoadRequest {
final Map payload;
Completer completer;
Stopwatch timer;
Socket socket;
bool get isDone => completer.isCompleted;
int get elapsed => timer.elapsedMilliseconds;
@@ -22,7 +21,6 @@ class LoadRequest {
completer.future.whenComplete(() => timer.stop());
try {
Socket.connect(this.host, this.port).then((socket) {
this.socket = socket;
utf8.decoder.bind(socket).listen((r) {
if (r.contains('"jsonrpc": "2.0", "result": ')) {
socket.close();
@@ -46,33 +44,24 @@ class LoadRequest {
}
}
typedef bool LoadTickCallback(ClientLoadGenerator load_generator, ClientLoadDataPoint stats);
typedef bool LoadTestCallback(LoadGenerator load_generator, LoadDataPoint stats);
class ClientLoadGenerator {
Timer _timer;
final String host;
final int port;
final LoadTickCallback tickCallback;
class LoadGenerator {
int load = 1;
Timer _timer;
String host;
int port;
Map query;
List<LoadRequest> backlog = [];
ClientLoadGenerator(this.host, this.port, {this.tickCallback, this.query}) {
if (query == null) {
query = {
'id': 1,
'method': 'blockchain.claimtrie.resolve',
'params': ['one', 'two', 'three']
};
}
}
LoadTestCallback cb;
LoadGenerator(this.host, this.port, this.query, this.cb);
start() {
var previous = spawn_requests();
backlog = <LoadRequest>[];
var backlog = <LoadRequest>[];
_timer = Timer.periodic(Duration(seconds: 1), (t) {
var stat = ClientLoadDataPoint(t.tick);
var stat = LoadDataPoint();
backlog.removeWhere((r) {
if (r.isDone) stat.addCatchup(r);
return r.isDone;
@@ -86,20 +75,16 @@ class ClientLoadGenerator {
}
stat.backlog = backlog.length;
stat.load = load;
stat.close();
if (tickCallback(this, stat)) {
if (cb(this, stat)) {
previous = spawn_requests();
} else {
stop();
t.cancel();
}
});
}
stop() {
_timer.cancel();
for (var remaining in backlog) {
remaining?.socket?.close();
}
}
List<LoadRequest> spawn_requests() {
@@ -112,20 +97,18 @@ class ClientLoadGenerator {
}
class ClientLoadDataPoint {
final int tick;
class LoadDataPoint {
final DateTime time = new DateTime.now();
int success = 0;
int errored = 0;
int backlog = 0;
int catchup = 0;
int _success_total = 0;
int avg_success = 0;
int _catchup_total = 0;
int avg_catchup = 0;
int load = 0;
ClientLoadDataPoint(this.tick);
ClientLoadDataPoint.empty(): this(0);
int get avg_success => _success_total > 0 ? (_success_total/success).round() : 0;
int get avg_catchup => _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
addSuccess(LoadRequest r) {
success++; _success_total += r.elapsed;
@@ -134,19 +117,16 @@ class ClientLoadDataPoint {
addCatchup(LoadRequest r) {
catchup++; _catchup_total += r.elapsed;
}
close() {
avg_success = _success_total > 0 ? (_success_total/success).round() : 0;
avg_catchup = _catchup_total > 0 ? (_catchup_total/catchup).round() : 0;
}
}
generate_load() {
cli() {
var runs = 1;
ClientLoadGenerator('localhost', 50001, tickCallback: (t, stats) {
print("run ${runs}: ${stats.load} load, ${stats.success} success, ${stats.backlog} backlog");
LoadGenerator('localhost', 50001, {
'id': 1,
'method': 'blockchain.claimtrie.resolve',
'params': ['one', 'two', 'three']
}, (t, stats) {
print("run ${runs}: ${stats}");
t.load = (runs < 4 ? t.load*2 : t.load/2).round();
return runs++ < 10;
}).start();
@@ -1,159 +0,0 @@
import 'dart:async';
import 'dart:convert';
import 'package:web_socket_channel/io.dart';
import 'package:web_socket_channel/status.dart' as status;
class ServerConnection {
String url = "";
IOWebSocketChannel channel;
bool get isConnected => channel != null && channel.closeCode == null;
final StreamController<ServerLoadDataPoint> _loadDataController = StreamController.broadcast();
Stream<ServerLoadDataPoint> get load_data => _loadDataController.stream;
ServerConnection({this.url});
open() {
if (isConnected) return Future.value('already open');
channel = IOWebSocketChannel.connect(this.url);
int tick = 1;
channel.stream.listen((message) {
var data = json.decode(message);
print(data);
_loadDataController.add(
ServerLoadDataPoint.from_map(
tick, data
)
);
tick++;
});
}
close() {
if (isConnected) {
return channel.sink.close(status.goingAway);
}
return Future.value('already closed');
}
subscribe_to_server_load_data() {
if (isConnected) channel.sink.add('subscribe');
}
unsubscribe_from_server_load_data() {
if (isConnected) channel.sink.add('unsubscribe');
}
}
class TimeStats {
final int avg, min, max;
// percentiles
final int five;
final int twenty_five;
final int fifty;
final int seventy_five;
final int ninety_five;
TimeStats(
this.avg, this.min, this.five, this.twenty_five,
this.fifty, this.seventy_five, this.ninety_five,
this.max
);
TimeStats.from_list(List l): this(
l[0], l[1], l[2], l[3], l[4], l[5], l[6], l[7]
);
TimeStats.from_zeros(): this(
0, 0, 0, 0, 0, 0, 0, 0
);
factory TimeStats.from_list_or_zeros(List l) =>
l != null ? TimeStats.from_list(l): TimeStats.from_zeros();
}
class APICallMetrics {
// total requests received
final int receive_count;
// sum of these is total responses made
final int cache_response_count;
final int query_response_count;
final int intrp_response_count;
final int error_response_count;
// stacked values for chart
final int cache_response_stack;
final int query_response_stack;
final int intrp_response_stack;
final int error_response_stack;
// millisecond timings for non-cache responses
final TimeStats response;
final TimeStats interrupt;
final TimeStats error;
// response, interrupt and error each also report the python, wait and sql stats:
final TimeStats python;
final TimeStats wait;
final TimeStats sql;
// extended timings for individual sql executions
final TimeStats individual_sql;
final int individual_sql_count;
// actual queries
final List<String> errored_queries;
final List<String> interrupted_queries;
APICallMetrics(
this.receive_count,
this.cache_response_count, this.query_response_count,
this.intrp_response_count, this.error_response_count,
this.response, this.interrupt, this.error,
this.python, this.wait, this.sql,
this.individual_sql, this.individual_sql_count,
this.errored_queries, this.interrupted_queries
):
cache_response_stack=cache_response_count+query_response_count+intrp_response_count+error_response_count,
query_response_stack=query_response_count+intrp_response_count+error_response_count,
intrp_response_stack=intrp_response_count+error_response_count,
error_response_stack=error_response_count;
APICallMetrics.from_map(Map data): this(
data["receive_count"] ?? 0,
data["cache_response_count"] ?? 0,
data["query_response_count"] ?? 0,
data["intrp_response_count"] ?? 0,
data["error_response_count"] ?? 0,
TimeStats.from_list_or_zeros(data["response"]),
TimeStats.from_list_or_zeros(data["interrupt"]),
TimeStats.from_list_or_zeros(data["error"]),
TimeStats.from_list_or_zeros(data["python"]),
TimeStats.from_list_or_zeros(data["wait"]),
TimeStats.from_list_or_zeros(data["sql"]),
TimeStats.from_list_or_zeros(data["individual_sql"]),
data["individual_sql_count"] ?? 0,
List<String>.from(data["errored_queries"] ?? const []),
List<String>.from(data["interrupted_queries"] ?? const []),
);
}
class ServerLoadDataPoint {
final int tick;
final int sessions;
final APICallMetrics search;
final APICallMetrics resolve;
const ServerLoadDataPoint(
this.tick, this.sessions, this.search, this.resolve
);
ServerLoadDataPoint.from_map(int tick, Map data): this(
tick, (data['status'] ?? const {})['sessions'] ?? 0,
APICallMetrics.from_map((data['api'] ?? const {})['search'] ?? const {}),
APICallMetrics.from_map((data['api'] ?? const {})['resolve'] ?? const {})
);
ServerLoadDataPoint.empty(): this(
0, 0, APICallMetrics.from_map(const {}), APICallMetrics.from_map(const {})
);
}
connect_and_listen_for_load_data() {
ServerConnection(url: 'ws://localhost:8181/')..open()..load_data.listen((m) {
print(m.search);
});
}
+22 -22
View File
@@ -7,7 +7,7 @@ packages:
name: analyzer
url: "https://pub.dartlang.org"
source: hosted
version: "0.37.0"
version: "0.36.4"
args:
dependency: transitive
description:
@@ -63,14 +63,14 @@ packages:
name: csslib
url: "https://pub.dartlang.org"
source: hosted
version: "0.16.1"
version: "0.16.0"
dart_style:
dependency: transitive
description:
name: dart_style
url: "https://pub.dartlang.org"
source: hosted
version: "1.2.9"
version: "1.2.8"
fixnum:
dependency: transitive
description:
@@ -84,7 +84,7 @@ packages:
name: front_end
url: "https://pub.dartlang.org"
source: hosted
version: "0.1.20"
version: "0.1.19"
glob:
dependency: transitive
description:
@@ -134,13 +134,20 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "0.6.1+1"
json_rpc_2:
dependency: transitive
description:
name: json_rpc_2
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.0"
kernel:
dependency: transitive
description:
name: kernel
url: "https://pub.dartlang.org"
source: hosted
version: "0.3.20"
version: "0.3.19"
matcher:
dependency: transitive
description:
@@ -203,7 +210,7 @@ packages:
name: pedantic
url: "https://pub.dartlang.org"
source: hosted
version: "1.8.0+1"
version: "1.7.0"
pool:
dependency: transitive
description:
@@ -217,14 +224,14 @@ packages:
name: protobuf
url: "https://pub.dartlang.org"
source: hosted
version: "0.13.15"
version: "0.13.12"
protoc_plugin:
dependency: "direct dev"
description:
name: protoc_plugin
url: "https://pub.dartlang.org"
source: hosted
version: "17.0.4"
version: "17.0.2"
pub_semver:
dependency: transitive
description:
@@ -295,13 +302,6 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.0"
stream_transform:
dependency: transitive
description:
name: stream_transform
url: "https://pub.dartlang.org"
source: hosted
version: "0.0.19"
string_scanner:
dependency: transitive
description:
@@ -322,7 +322,7 @@ packages:
name: test
url: "https://pub.dartlang.org"
source: hosted
version: "1.6.5"
version: "1.6.4"
test_api:
dependency: transitive
description:
@@ -336,7 +336,7 @@ packages:
name: test_core
url: "https://pub.dartlang.org"
source: hosted
version: "0.2.7"
version: "0.2.6"
typed_data:
dependency: transitive
description:
@@ -344,13 +344,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "1.1.6"
vm_service_lib:
vm_service_client:
dependency: transitive
description:
name: vm_service_lib
name: vm_service_client
url: "https://pub.dartlang.org"
source: hosted
version: "3.22.0"
version: "0.2.6+2"
watcher:
dependency: transitive
description:
@@ -359,12 +359,12 @@ packages:
source: hosted
version: "0.9.7+10"
web_socket_channel:
dependency: "direct main"
dependency: transitive
description:
name: web_socket_channel
url: "https://pub.dartlang.org"
source: hosted
version: "1.0.14"
version: "1.0.13"
yaml:
dependency: transitive
description:
-1
View File
@@ -11,7 +11,6 @@ dependencies:
http: ^0.12.0+2
path: ^1.4.1
protobuf: ^0.13.12
web_socket_channel: ^1.0.4
dev_dependencies:
pedantic: ^1.0.0
+35 -69
View File
@@ -1,83 +1,49 @@
import 'package:flutter/material.dart';
import 'package:flutter/foundation.dart' show debugDefaultTargetPlatformOverride;
import 'package:provider/provider.dart';
import 'package:ver/src/servers.dart';
import 'package:ver/src/models/server.dart';
import 'package:ver/utils.dart';
import 'package:ver/time_series_chart.dart';
class UnderConstructionPage extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(title: Text('Under Construction')),
body: SizedBox.expand(
child: Center(child: Text('Under Construction')),
),
);
}
class VerApp extends StatelessWidget {
@override
Widget build(BuildContext context) {
return MaterialApp(
title: 'Ver',
theme: ThemeData(
brightness: Brightness.light,
primarySwatch: Colors.lightBlue,
fontFamily: 'Roboto',
),
home: VerHomePage(title: 'Wallet Server'),
);
}
}
class MainPage extends StatefulWidget {
@override
_MainPageState createState() => _MainPageState();
class VerHomePage extends StatefulWidget {
VerHomePage({Key key, this.title}) : super(key: key);
final String title;
@override
_VerHomePageState createState() => _VerHomePageState();
}
class _MainPageState extends State<MainPage> {
int _currentIndex = 3;
@override
Widget build(BuildContext context) {
return Scaffold(
body: IndexedStack(
index: _currentIndex,
children: [
UnderConstructionPage(),
UnderConstructionPage(),
UnderConstructionPage(),
ServersSectionNavigation(),
UnderConstructionPage(),
],
),
bottomNavigationBar: BottomNavigationBar(
items: const <BottomNavigationBarItem>[
BottomNavigationBarItem(icon: Icon(Icons.home), title: Text('Home')),
BottomNavigationBarItem(icon: Icon(Icons.trending_up), title: Text('Trending')),
BottomNavigationBarItem(icon: Icon(Icons.subscriptions), title: Text('Subscriptions')),
BottomNavigationBarItem(icon: Icon(Icons.router), title: Text('Servers')),
BottomNavigationBarItem(icon: Icon(Icons.folder), title: Text('Library')),
],
currentIndex: _currentIndex,
onTap: (int index) {
setState(() {
_currentIndex = index;
});
},
selectedItemColor: Colors.amber[800],
),
);
}
class _VerHomePageState extends State<VerHomePage> {
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(
title: Text(widget.title),
),
body: new Padding(
padding: const EdgeInsets.all(8.0),
child: SimpleTimeSeriesChart()
),
);
}
}
void main() {
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
runApp(
MaterialApp(
title: 'Ver',
theme: ThemeData(
brightness: Brightness.dark,
//primaryColor: Colors.lightBlue[800],
//accentColor: Colors.cyan[600],
fontFamily: 'Roboto',
),
home: MultiProvider(
providers: [
ChangeNotifierProvider<ServerManager>(builder: (context) => ServerManager())
],
child: MainPage()
)
)
);
debugDefaultTargetPlatformOverride = getTargetPlatformForDesktop();
runApp(new VerApp());
}
@@ -1,190 +0,0 @@
import 'dart:async';
import 'dart:collection';
import 'package:flutter/foundation.dart';
import 'package:lbry/lbry.dart';
class ServerManager extends ChangeNotifier {
final List<Server> _servers = [];
UnmodifiableListView<Server> get items => UnmodifiableListView(_servers);
add(Server server) {
_servers.add(server);
notifyListeners();
}
remove(Server server) {
server.dispose();
_servers.remove(server);
notifyListeners();
}
@override
void dispose() {
super.dispose();
for (var server in _servers) {
server.dispose();
}
}
}
class Server extends ChangeNotifier {
String _label = "";
String get label => _label;
String get labelOrHost => _label.length > 0 ? _label : host;
String _host = "";
String get host => _host;
int _port = 8181;
int get port => _port;
bool _ssl = false;
bool get ssl => _ssl;
String get url => _connection.url;
_setURL(String host, int port, bool ssl) {
_host = host;
_port = port;
_connection.url = "ws${ssl?'s':''}://$host:$port";
}
DateTime _added = new DateTime.now();
String get added => _added.toIso8601String();
bool _isDefault = false;
bool get isDefault => _isDefault;
bool _isEnabled = false;
bool get isEnabled => _isEnabled;
bool _isTrackingServerLoad = false;
bool get isTrackingServerLoad => _isTrackingServerLoad;
_setIsTrackingServerLoad(bool toggle_tracking) {
if (_isTrackingServerLoad && !toggle_tracking) {
_connection.unsubscribe_from_server_load_data();
} else if (!_isTrackingServerLoad && toggle_tracking) {
_connection.subscribe_to_server_load_data();
}
_isTrackingServerLoad = toggle_tracking;
}
ClientLoadManager clientLoadManager;
final ServerConnection _connection = ServerConnection();
bool get isConnected => _connection.isConnected;
Stream<ServerLoadDataPoint> get serverLoadStream => _connection.load_data;
final List<ServerLoadDataPoint> serverLoadData = [ServerLoadDataPoint.empty()];
Server() {
clientLoadManager = ClientLoadManager(this);
serverLoadStream.listen(serverLoadData.add);
}
update({String host, int port, bool ssl, String label,
bool isDefault, bool isEnabled, bool isTrackingServerLoad}) {
if (host != null && port != null && ssl != null) {
_setURL(host, port, ssl);
}
if (isTrackingServerLoad != null) {
_setIsTrackingServerLoad(isTrackingServerLoad);
}
_label = label ?? _label;
_isDefault = isDefault ?? _isDefault;
_isEnabled = isEnabled ?? _isEnabled;
notifyListeners();
}
connect() async {
await _connection.open();
if (_isTrackingServerLoad) {
_connection.subscribe_to_server_load_data();
}
notifyListeners();
}
disconnect() {
clientLoadManager.stop();
_connection.close();
notifyListeners();
}
@override
void dispose() {
disconnect();
super.dispose();
}
}
class ClientLoadManager extends ChangeNotifier {
final Server _server;
ClientLoadGenerator clientLoadGenerator;
final StreamController<ClientLoadDataPoint> _loadDataController = StreamController.broadcast();
Stream<ClientLoadDataPoint> get clientLoadStream => _loadDataController.stream;
final List<ClientLoadDataPoint> clientLoadData = [ClientLoadDataPoint.empty()];
int _load = 1;
int get load => _load;
int _offset = 0;
int get offset => _offset;
bool _noTotals = false;
bool get noTotals => _noTotals;
ClientLoadManager(this._server) {
clientLoadStream.listen(clientLoadData.add);
}
update({int load, int offset, bool noTotals}) {
_load = load ?? _load;
_offset = offset ?? _offset;
_noTotals = noTotals ?? _noTotals;
notifyListeners();
}
start() {
clientLoadData.clear();
clientLoadData.add(ClientLoadDataPoint.empty());
clientLoadGenerator = ClientLoadGenerator(
_server.host, _server.port,
query: {
'id': 1,
'method': 'blockchain.claimtrie.search',
'params': {
'no_totals': _noTotals,
'offset': _offset,
'limit': 20,
'fee_amount': '<1',
//'all_tags': ['funny'],
'any_tags': [
'crypto',
'outdoors',
'cars',
'automotive'
]
}
}, tickCallback: (t, stats) {
_loadDataController.add(stats);
//increase = max(1, min(100, increase+2)-stats.backlog);
//increase += 1;
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
t.load = _load;//rand.nextInt(10)+5;
return true;
})..start();
}
stop() {
if (clientLoadGenerator != null) {
clientLoadGenerator.stop();
}
}
@override
void dispose() {
super.dispose();
stop();
}
}
-73
View File
@@ -1,73 +0,0 @@
import 'package:flutter/material.dart';
import 'package:ver/src/widgets/server.dart';
import 'package:provider/provider.dart';
import 'models/server.dart';
class ServerListPage extends StatelessWidget {
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text('Servers')),
body: ServerList(),
floatingActionButton: FloatingActionButton(
child: Icon(Icons.add),
onPressed: () => Navigator.of(context).pushNamed('/edit', arguments: true),
)
);
}
class ServerFormPage extends StatelessWidget {
final bool creating;
ServerFormPage(this.creating);
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text(creating ? 'Add Server' : 'Modify Server')),
body: ServerForm(creating),
);
}
class ServerViewPage extends StatelessWidget {
final Server server;
ServerViewPage(this.server);
@override
Widget build(BuildContext context) =>
Scaffold(
appBar: AppBar(title: Text(server.labelOrHost)),
body: ChangeNotifierProvider<Server>.value(
value: server,
child: ServerView()
)
);
}
class ServersSectionNavigation extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Navigator(
onGenerateRoute: (RouteSettings settings) {
return MaterialPageRoute(
settings: settings,
builder: (BuildContext context) {
switch (settings.name) {
case '/':
return ServerListPage();
case '/edit':
return ServerFormPage(settings.arguments);
case '/view':
return ServerViewPage(settings.arguments);
}
return ServerListPage();
},
);
},
);
}
}
@@ -1,172 +0,0 @@
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:provider/provider.dart';
import '../models/server.dart';
import 'time_series_chart.dart';
class ServerList extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Consumer<ServerManager>(
builder: (_, servers, __) =>
ListView.builder(
itemCount: servers.items.length,
itemBuilder: (context, index) =>
ChangeNotifierProvider<Server>.value(
value: servers.items[index],
child: ServerListItem()
)
)
);
}
}
class ServerListItem extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Card(
child: Consumer<Server>(
builder: (_, server, __) =>
ListTile(
leading: FlutterLogo(size: 72.0),
title: Text(server.labelOrHost),
subtitle: Text("${server.url}\nadded ${server.added}"),
trailing: Icon(Icons.more_vert),
isThreeLine: true,
onTap: ()=> Navigator.of(context).pushNamed('/view', arguments: server),
)
)
);
}
}
class ServerView extends StatelessWidget {
@override
Widget build(BuildContext context) => ServerCharts();
}
class ServerForm extends StatefulWidget {
final bool creating;
ServerForm(this.creating);
@override
_ServerFormState createState() => _ServerFormState();
}
class _ServerFormState extends State<ServerForm> {
final _formKey = GlobalKey<FormState>();
final _formData = {
'label': '',
'host': 'localhost',
'port': 8181,
'ssl': false,
'isDefault': true,
'isEnabled': false,
'isTrackingServerLoad': false
};
@override
Widget build(BuildContext context) {
return Form(
key: _formKey,
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: <Widget>[
ListTile(title: TextFormField(
initialValue: _formData['label'],
decoration: const InputDecoration(
labelText: 'Label',
hintText: 'Optional text to display in server list.'
),
onSaved: (value) => _formData['label'] = value,
)),
ListTile(title: TextFormField(
initialValue: _formData['host'],
keyboardType: TextInputType.url,
inputFormatters: [
WhitelistingTextInputFormatter(RegExp(r'[\w\-\.]+'))
],
decoration: const InputDecoration(
labelText: 'Host Name',
hintText: 'Enter the host name of the server.'
),
validator: (value) => value.isEmpty ? 'A host name is required.' : null,
onSaved: (value) => _formData['host'] = value,
)),
ListTile(title: TextFormField(
initialValue: _formData['port'].toString(),
keyboardType: TextInputType.number,
inputFormatters: [
WhitelistingTextInputFormatter.digitsOnly
],
decoration: const InputDecoration(
labelText: 'Port',
hintText: 'Enter the port of the server.'
),
validator: (value) => value.isEmpty ? 'A port is required.' : null,
onSaved: (value) => setState(() => _formData['port'] = int.parse(value)),
)),
SwitchListTile(
title: Text('Requires SSL.'),
value: _formData['ssl'],
onChanged: (value) => setState(() => _formData['ssl'] = value),
),
SwitchListTile(
title: Text('Use this as your primary and default server.'),
value: _formData['isDefault'],
onChanged: (value) => setState(() => _formData['isDefault'] = value),
),
SwitchListTile(
title: Text('Always stay connected.'),
value: _formData['isEnabled'],
onChanged: (value) => setState(() => _formData['isEnabled'] = value),
),
SwitchListTile(
title: Text('Track server load.'),
value: _formData['isTrackingServerLoad'],
onChanged: (value) =>
setState(() => _formData['isTrackingServerLoad'] = value),
),
ListTile(title: RaisedButton(
onPressed: () {
if (_formKey.currentState.validate()) {
_formKey.currentState.save();
var manager = Provider.of<ServerManager>(
context, listen: false
);
var server = Server();
server.update(
label: _formData['label'],
host: _formData['host'],
port: _formData['port'],
ssl: _formData['ssl'],
isDefault: _formData['isDefault'],
isEnabled: _formData['isEnabled'],
isTrackingServerLoad: _formData['isTrackingServerLoad'],
);
manager.add(server);
if (server.isEnabled) {
server.connect();
}
Navigator.of(context).pop();
}
},
child: Text(widget.creating ? 'Add Server' : 'Update Server'),
),
),
],
),
);
}
}
@@ -1,354 +0,0 @@
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:charts_flutter/flutter.dart' as charts;
import 'package:charts_flutter/src/base_chart_state.dart' as state;
import 'package:charts_common/common.dart' as common;
import 'package:provider/provider.dart';
import 'package:lbry/lbry.dart';
import '../models/server.dart';
class ServerCharts extends StatelessWidget {
@override
Widget build(BuildContext context) {
var server = Provider.of<Server>(context, listen: false);
return ListView(children: <Widget>[
SizedBox(height: 300.0, child: APILoadChart(
server, "Search", (ServerLoadDataPoint dataPoint) => dataPoint.search
)),
//SizedBox(height: 300.0, child: APILoadChart(
// server, "Resolve", (ServerLoadDataPoint dataPoint) => dataPoint.resolve
//)),
SizedBox(height: 300.0, child: ServerPerformanceChart(server)),
//SizedBox(height: 220.0, child: ClientLoadChart(server.clientLoadManager)),
//SizedBox(height: 220.0, child: ClientPerformanceChart(server.clientLoadManager)),
]);
}
}
typedef APICallMetrics APIGetter(ServerLoadDataPoint dataPoint);
class APILoadChart extends StatefulWidget {
final Server server;
final String name;
final APIGetter getter;
APILoadChart(this.server, this.name, this.getter);
@override
State<StatefulWidget> createState() => APILoadChartState();
}
class APILoadChartState extends State<APILoadChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
/*
charts.Series<ServerLoadDataPoint, int>(
id: 'Received',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
strokeWidthPxFn: (_, __) => 4.0,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).receive_count,
data: widget.server.serverLoadData,
),*/
charts.Series<ServerLoadDataPoint, int>(
id: 'Cache',
colorFn: (_, __) =>
charts.MaterialPalette.green.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Query',
colorFn: (_, __) =>
charts.MaterialPalette.blue.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).query_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Interrupts',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).intrp_response_stack,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Errors',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).error_response_stack,
data: widget.server.serverLoadData,
),
/*
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Interrupted',
colorFn: (_, __) =>
charts.MaterialPalette.pink.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).interrupted,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} Errored',
colorFn: (_, __) =>
charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).errored,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 5.0,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: '${widget.name} From Cache',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => widget.getter(load).cache_hits,
strokeWidthPxFn: (ServerLoadDataPoint load, _) => 3.0,
data: widget.server.serverLoadData,
),
*/
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData,
//renderer: new charts.LineRendererConfig<num>(includeArea: true)
)
);
}
}
class ServerPerformanceChart extends StatefulWidget {
final Server server;
ServerPerformanceChart(this.server);
@override
State<StatefulWidget> createState() => ServerPerformanceChartState();
}
class ServerPerformanceChartState extends State<ServerPerformanceChart> {
List<charts.Series<ServerLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ServerLoadDataPoint, int>(
id: 'Waiting 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Waiting',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.wait.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. Executing',
colorFn: (_, __) => charts.MaterialPalette.teal.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.python.avg,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'SQLite 95 Percentile',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.lighter,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.ninety_five,
data: widget.server.serverLoadData,
),
charts.Series<ServerLoadDataPoint, int>(
id: 'Avg. SQLite',
colorFn: (_, __) => charts.MaterialPalette.blue.shadeDefault.darker,
domainFn: (ServerLoadDataPoint load, _) => load.tick,
measureFn: (ServerLoadDataPoint load, _) => load.search.sql.avg,
data: widget.server.serverLoadData,
),
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ServerLoadDataPoint>(
stream: widget.server.serverLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class ClientLoadChart extends StatefulWidget {
final ClientLoadManager client;
ClientLoadChart(this.client);
@override
State<StatefulWidget> createState() => ClientLoadChartState();
}
class ClientLoadChartState extends State<ClientLoadChart> {
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ClientLoadDataPoint, int>(
id: 'Load',
colorFn: (_, __) => charts.MaterialPalette.black.darker,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.load,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Success',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.success,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Backlog',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.backlog,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Catch-up',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.catchup,
data: widget.client.clientLoadData,
)
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ClientLoadDataPoint>(
stream: widget.client.clientLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class ClientPerformanceChart extends StatefulWidget {
final ClientLoadManager client;
ClientPerformanceChart(this.client);
@override
State<StatefulWidget> createState() => ClientPerformanceChartState();
}
class ClientPerformanceChartState extends State<ClientPerformanceChart> {
List<charts.Series<ClientLoadDataPoint, int>> seriesData;
@override
void initState() {
super.initState();
seriesData = [
charts.Series<ClientLoadDataPoint, int>(
id: 'Avg. Success Time',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.avg_success,
data: widget.client.clientLoadData,
),
charts.Series<ClientLoadDataPoint, int>(
id: 'Avg. Catch-up Time',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (ClientLoadDataPoint load, _) => load.tick,
measureFn: (ClientLoadDataPoint load, _) => load.avg_catchup,
data: widget.client.clientLoadData,
),
];
}
@override
Widget build(BuildContext context) {
return StreamBuilder<ClientLoadDataPoint>(
stream: widget.client.clientLoadStream,
builder: (BuildContext context, _) => BetterLineChart(seriesData)
);
}
}
class BetterLineChart extends charts.LineChart {
final int itemCount;
final Object lastItem;
BetterLineChart(List<charts.Series<dynamic, int>> seriesList, {charts.LineRendererConfig renderer}):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(
seriesList,
animate: false,
defaultRenderer: renderer,
behaviors: [charts.SeriesLegend()],
domainAxis: charts.NumericAxisSpec(
viewport: new charts.NumericExtents(
max(0, seriesList[0].data.last.tick - 60), seriesList[0].data.last.tick
),
renderSpec: new charts.SmallTickRendererSpec(
labelStyle: new charts.TextStyleSpec(
color: charts.MaterialPalette.gray.shade50
),
lineStyle: new charts.LineStyleSpec(
color: charts.MaterialPalette.black
)
),
),
primaryMeasureAxis: new charts.NumericAxisSpec(
renderSpec: new charts.GridlineRendererSpec(
labelStyle: new charts.TextStyleSpec(
color: charts.MaterialPalette.white
),
lineStyle: new charts.LineStyleSpec(
color: charts.MaterialPalette.gray.shade100
),
),
tickProviderSpec: new charts.BasicNumericTickProviderSpec(
dataIsInWholeNumbers: true,
desiredTickCount: 5
)
),
);
@override
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
state.BaseChartState chartState) {
super.updateCommonChart(baseChart, oldWidget, chartState);
final prev = oldWidget as BetterLineChart;
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
chartState.markChartDirty();
}
}
}
@@ -0,0 +1,149 @@
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:charts_flutter/flutter.dart' as charts;
import 'package:charts_flutter/src/base_chart_state.dart' as state;
import 'package:charts_common/common.dart' as common;
import 'package:lbry/lbry.dart';
class SimpleTimeSeriesChart extends StatefulWidget {
SimpleTimeSeriesChart({Key key}) : super(key: key);
@override
_SimpleTimeSeriesChartState createState() => _SimpleTimeSeriesChartState();
}
class _SimpleTimeSeriesChartState extends State<SimpleTimeSeriesChart> {
final List<LoadDataPoint> loadData = [];
final List<charts.Series<LoadDataPoint, DateTime>> loadSeries = [];
final List<charts.Series<LoadDataPoint, DateTime>> timeSeries = [];
final Random rand = Random();
LoadGenerator loadGenerator;
@override
void initState() {
super.initState();
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Load',
colorFn: (_, __) => charts.MaterialPalette.black.darker,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.load,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Success',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.success,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Backlog',
colorFn: (_, __) => charts.MaterialPalette.red.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.backlog,
data: loadData,
)
);
loadSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Catch-up',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.catchup,
data: loadData,
)
);
timeSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Avg. Success Time',
colorFn: (_, __) => charts.MaterialPalette.green.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.avg_success,
data: loadData,
)
);
timeSeries.add(
charts.Series<LoadDataPoint, DateTime>(
id: 'Avg. Catch-up Time',
colorFn: (_, __) => charts.MaterialPalette.yellow.shadeDefault,
domainFn: (LoadDataPoint load, _) => load.time,
measureFn: (LoadDataPoint load, _) => load.avg_catchup,
data: loadData,
)
);
var increase = 1;
loadData.add(LoadDataPoint());
loadGenerator = LoadGenerator('spv2.lbry.com', 50001, {
'id': 1,
'method': 'blockchain.claimtrie.search',
'params': {
'offset': 0,
'limit': 20,
'fee_amount': '<1',
'all_tags': ['funny'],
'any_tags': [
'crypto',
'outdoors',
'cars',
'automotive'
]
}
}, (t, stats) {
setState(() {
//if (loadData.length > 60) loadData.removeAt(0);
loadData.add(stats);
});
//increase = max(1, min(30, (increase*1.1).ceil())-stats.backlog);
increase += 1;
//t.query['params']['offset'] = (increase/2).ceil()*t.query['params']['limit'];
t.load = increase;//rand.nextInt(10)+5;
return true;
})..start();
}
@override
void dispose() {
loadGenerator.stop();
super.dispose();
}
@override
Widget build(BuildContext context) {
return Column(children: <Widget>[
SizedBox(height: 250.0, child: BetterTimeSeriesChart(loadSeries)),
SizedBox(height: 250.0, child: BetterTimeSeriesChart(timeSeries)),
]);
}
}
class BetterTimeSeriesChart extends charts.TimeSeriesChart {
final int itemCount;
final Object lastItem;
BetterTimeSeriesChart(
List<charts.Series<dynamic, DateTime>> seriesList):
itemCount = seriesList[0].data.length,
lastItem = seriesList[0].data.last,
super(seriesList, behaviors: [charts.SeriesLegend()]);
@override
void updateCommonChart(common.BaseChart baseChart, charts.BaseChart oldWidget,
state.BaseChartState chartState) {
super.updateCommonChart(baseChart, oldWidget, chartState);
final prev = oldWidget as BetterTimeSeriesChart;
if (itemCount != prev?.itemCount || lastItem != prev?.lastItem) {
chartState.markChartDirty();
}
}
}
@@ -60,7 +60,7 @@ int main(int argc, char **argv) {
flutter::FlutterWindowController flutter_controller(icu_data_path);
// Start the engine.
if (!flutter_controller.CreateWindow(800, 1200, "ver - lbry workbench",
if (!flutter_controller.CreateWindow(800, 600, "ver - lbry workbench",
assets_path, arguments)) {
return EXIT_FAILURE;
}
-28
View File
@@ -43,20 +43,6 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "1.14.11"
convert:
dependency: transitive
description:
name: convert
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.1"
crypto:
dependency: transitive
description:
name: crypto
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.6"
cupertino_icons:
dependency: "direct main"
description:
@@ -151,13 +137,6 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "0.13.15"
provider:
dependency: "direct main"
description:
name: provider
url: "https://pub.dartlang.org"
source: hosted
version: "3.0.0+1"
quiver:
dependency: transitive
description:
@@ -226,12 +205,5 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.0.8"
web_socket_channel:
dependency: transitive
description:
name: web_socket_channel
url: "https://pub.dartlang.org"
source: hosted
version: "1.0.14"
sdks:
dart: ">=2.2.2 <3.0.0"
-1
View File
@@ -12,7 +12,6 @@ dependencies:
sdk: flutter
cupertino_icons: ^0.1.2
charts_flutter: ^0.6.0
provider: ^3.0.0
lbry:
path: ../lbry
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.3"
__version__ = "0.38.0"
version = tuple(__version__.split('.'))
+2 -2
View File
@@ -69,10 +69,10 @@ class BlobManager:
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
+3 -3
View File
@@ -22,6 +22,9 @@ CURRENCIES = {
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
HEADERS_FILE_SHA256_CHECKSUM = (
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
)
class Setting(typing.Generic[T]):
@@ -551,9 +554,6 @@ class Config(CLIConfig):
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
@@ -163,6 +163,3 @@ class ComponentManager:
if component.component_name == component_name:
return component.component
raise NameError(component_name)
def has_component(self, component_name):
return any(component for component in self.components if component_name == component.component_name)
+80 -61
View File
@@ -4,6 +4,8 @@ import logging
import math
import binascii
import typing
from hashlib import sha256
from types import SimpleNamespace
import base58
from aioupnp import __version__ as aioupnp_version
@@ -11,6 +13,7 @@ from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
from lbry import utils
from lbry.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbry.dht.node import Node
from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager
@@ -20,7 +23,8 @@ from lbry.extras.daemon.Component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import LbryWalletManager
from lbry.wallet.header import Headers
from lbry.wallet import Network
log = logging.getLogger(__name__)
@@ -108,34 +112,21 @@ class HeadersComponent(Component):
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
self.headers_file = os.path.join(self.headers_dir, 'headers')
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
self.headers = Headers(self.headers_file)
self.is_downloading_headers = False
self._downloading_headers = None
self._headers_progress_percent = 0
@property
def component(self):
return self
def _round_progress(self, local_height, remote_height):
return min(max(math.ceil(float(local_height) / float(remote_height) * 100), 0), 100)
async def get_status(self) -> dict:
progress = None
if self.is_downloading_headers:
progress = self._headers_progress_percent
elif self.component_manager.has_component(WALLET_COMPONENT):
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
if wallet_manager and wallet_manager.ledger.network.remote_height > 0:
local_height = wallet_manager.ledger.headers.height
remote_height = wallet_manager.ledger.network.remote_height
progress = self._round_progress(local_height, remote_height)
return {
'downloading_headers': True,
'download_progress': progress
} if progress is not None and progress < 100 else {}
async def get_status(self):
return {} if not self._downloading_headers else {
'downloading_headers': self._downloading_headers,
'download_progress': self._headers_progress_percent
}
async def fetch_headers_from_s3(self):
local_header_size = self.headers.bytes_size
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
@@ -145,68 +136,94 @@ class HeadersComponent(Component):
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
buffer, header_size = b'', self.headers.header_size
async for chunk in response.content.iter_any():
chunk = buffer + chunk
remaining = len(chunk) % header_size
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
if not chunk:
continue
if not await self.headers.connect(len(self.headers), chunk):
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
return
self._headers_progress_percent = self._round_progress(
self.headers.bytes_size, final_size_after_download
)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while True:
chunk = await response.content.read(512)
if not chunk:
break
fd.write(chunk)
self._headers_progress_percent = math.ceil(
float(fd.tell()) / float(final_size_after_download) * 100
)
log.info("fetched headers from s3, now verifying integrity after download.")
self._check_header_file_integrity()
def local_header_file_size(self) -> int:
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
def local_header_file_size(self):
if os.path.isfile(self.headers_file):
return os.stat(self.headers_file).st_size
return 0
async def get_downloadable_header_height(self) -> typing.Optional[int]:
try:
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return response.content_length // HEADER_SIZE
except OSError:
log.exception("Failed to download headers using https.")
async def get_remote_height(self):
ledger = SimpleNamespace()
ledger.config = {
'default_servers': self.conf.lbryum_servers,
'data_path': self.conf.wallet_dir
}
net = Network(ledger)
first_connection = net.on_connected.first
asyncio.ensure_future(net.start()) # TODO: SKETCHY! it might be trapping a CancelledError and not raising it
await first_connection
remote_height = await net.get_server_height()
await net.stop()
return remote_height
async def should_download_headers_from_s3(self) -> bool:
async def should_download_headers_from_s3(self):
if self.conf.blockchain_name != "lbrycrd_main":
return False
self._check_header_file_integrity()
s3_headers_depth = self.conf.s3_headers_depth
if not s3_headers_depth:
return False
local_height = self.headers.height
remote_height = await self.get_downloadable_header_height()
if remote_height is not None:
log.info("remote height: %i, local height: %i", remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
return True
local_height = self.local_header_file_height()
remote_height = await self.get_remote_height()
log.info("remote height: %i, local height: %i", remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
return True
return False
def _check_header_file_integrity(self):
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
if self.conf.blockchain_name != "lbrycrd_main":
return
hashsum = sha256()
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
checksum_length_in_bytes = checksum_height * HEADER_SIZE
if self.local_header_file_size() < checksum_length_in_bytes:
return
with open(self.headers_file, "rb") as headers_file:
hashsum.update(headers_file.read(checksum_length_in_bytes))
current_checksum = hashsum.hexdigest()
if current_checksum != checksum:
msg = f"Expected checksum {checksum}, got {current_checksum}"
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
log.warning("Deleting header file so it can be downloaded again.")
os.unlink(self.headers_file)
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
with open(self.headers_file, "rb+") as headers_file:
headers_file.truncate(checksum_length_in_bytes)
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
await self.headers.open()
await self.headers.repair()
if await self.should_download_headers_from_s3():
self._downloading_headers = await self.should_download_headers_from_s3()
if self._downloading_headers:
try:
self.is_downloading_headers = True
await self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
await self.headers.close()
self._downloading_headers = False
async def stop(self):
pass
@@ -225,9 +242,9 @@ class WalletComponent(Component):
return self.wallet_manager
async def get_status(self):
if self.wallet_manager and self.wallet_manager.ledger.network.remote_height:
if self.wallet_manager and self.running:
local_height = self.wallet_manager.ledger.headers.height
remote_height = self.wallet_manager.ledger.network.remote_height
remote_height = await self.wallet_manager.ledger.network.get_server_height()
best_hash = self.wallet_manager.get_best_blockhash()
return {
'blocks': max(local_height, 0),
@@ -399,8 +416,10 @@ class StreamManagerComponent(Component):
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
node = self.component_manager.get_component(DHT_COMPONENT)\
if self.component_manager.has_component(DHT_COMPONENT) else None
try:
node = self.component_manager.get_component(DHT_COMPONENT)
except NameError:
node = None
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
+18 -23
View File
@@ -1742,12 +1742,12 @@ class Daemon(metaclass=JSONRPCServerType):
multiple channels at the same time,
includes claims with invalid signatures,
use in conjunction with --valid_channel_signature
--channel_ids=<channel_ids> : (list) claims signed by any of these channels
--channel_ids=<channel_ids> : (str) claims signed by any of these channels
(arguments must be claim ids of the channels),
includes claims with invalid signatures,
implies --has_channel_signature,
use in conjunction with --valid_channel_signature
--not_channel_ids=<not_channel_ids>: (list) exclude claims signed by any of these channels
--not_channel_ids=<not_channel_ids>: (str) exclude claims signed by any of these channels
(arguments must be claim ids of the channels)
--has_channel_signature : (bool) claims with a channel signature (valid or invalid)
--valid_channel_signature : (bool) claims with a valid channel signature or no signature,
@@ -1815,7 +1815,7 @@ class Daemon(metaclass=JSONRPCServerType):
--not_locations=<not_locations> : (list) find claims not containing any of these locations
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--order_by=<order_by> : (list) field to order by, default is descending order, to do an
--order_by=<order_by> : (str) field to order by, default is descending order, to do an
ascending order prepend ^ to the field name, eg. '^amount'
available fields: 'name', 'height', 'release_time',
'publish_time', 'amount', 'effective_amount',
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
]
}
"""
result = await jsonrpc_post(
cmnt_list: dict = await jsonrpc_post(
self.conf.comment_server,
"get_claim_comments",
claim_id=claim_id,
@@ -3425,25 +3425,20 @@ class Daemon(metaclass=JSONRPCServerType):
page_size=page_size,
top_level=not include_replies
)
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
comment['is_channel_signature_valid'] = False
if is_channel_signature_valid:
result['items'] = [
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
]
return result
if 'items' in cmnt_list:
items = cmnt_list['items']
chnl_uris = {cmnt['channel_url'] for cmnt in items if 'channel_url' in cmnt}
claims = await self.resolve(tuple(chnl_uris))
for cmnt in cmnt_list['items']:
if 'channel_url' in cmnt:
if cmnt['channel_url'] in claims:
channel = claims[cmnt['channel_url']]
cmnt['is_channel_signature_valid'] = is_comment_signed_by_channel(cmnt, channel)
continue
cmnt['is_channel_signature_valid'] = False
if is_channel_signature_valid:
cmnt_list['items'] = [c for c in cmnt_list['items'] if c['is_channel_signature_valid']]
return cmnt_list
async def jsonrpc_comment_create(self, claim_id, comment, parent_id=None, channel_account_id=None,
channel_name=None, channel_id=None) -> dict:
+2 -7
View File
@@ -379,9 +379,7 @@ class Language(Metadata):
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2 and parts[0].isalpha():
self.region = parts.pop(0)
if parts and len(parts[0]) == 3 and parts[0].isdigit():
if parts and len(parts[0]) == 2:
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@@ -406,13 +404,10 @@ class Language(Metadata):
@property
def region(self) -> str:
if self.message.region:
r = LocationMessage.Country.Name(self.message.region)
return r[1:] if r.startswith('R') else r
return LocationMessage.Country.Name(self.message.region)
@region.setter
def region(self, region: str):
if len(region) == 3:
region = 'R'+region
self.message.region = LocationMessage.Country.Value(region)
+2 -2
View File
@@ -6,8 +6,8 @@ WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
def clean_tags(tags: List[str]):
return [tag for tag in set(normalize_tag(tag) for tag in tags) if tag]
return [tag for tag in (normalize_tag(tag) for tag in tags) if tag]
File diff suppressed because it is too large. Load diff
+16 -7
View File
@@ -56,6 +56,7 @@ class ManagedStream:
'stream_claim_info',
'download_id',
'rowid',
'written_bytes',
'content_fee',
'downloader',
'analytics_manager',
@@ -88,6 +89,7 @@ class ManagedStream:
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.written_bytes = 0
self.content_fee = content_fee
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
@@ -119,10 +121,6 @@ class ManagedStream:
def status(self) -> str:
return self._status
@property
def written_bytes(self) -> int:
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
@@ -207,13 +205,21 @@ class ManagedStream:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if not self.output_file_exists:
if self.full_path and self.output_file_exists:
if self.written_bytes:
written_bytes = self.written_bytes
else:
written_bytes = os.stat(self.full_path).st_size
else:
full_path = None
file_name = None
download_directory = None
written_bytes = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': self.written_bytes >= self.descriptor.lower_bound_decrypted_length(),
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
@@ -227,7 +233,7 @@ class ManagedStream:
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
'written_bytes': self.written_bytes,
'written_bytes': written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
@@ -365,6 +371,7 @@ class ManagedStream:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
@@ -380,6 +387,7 @@ class ManagedStream:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
self.written_bytes = 0
if isinstance(err, asyncio.TimeoutError):
self.downloader.stop()
await self.blob_manager.storage.change_file_download_dir_and_file_name(
@@ -417,6 +425,7 @@ class ManagedStream:
self.stream_hash, self.download_directory, self.file_name
)
await self.update_status(ManagedStream.STATUS_RUNNING)
self.written_bytes = 0
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
await self.started_writing.wait()
-2
View File
@@ -214,9 +214,7 @@ class CommandTestCase(IntegrationTestCase):
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
height = self.ledger.headers.height
await self.generate(1)
await self.on_header(height + 1)
await self.on_transaction_dict(claim)
return claim
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.0/lbrycrd-linux.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.1/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+2 -46
View File
@@ -3,11 +3,10 @@ import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL, normalize_name
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
@@ -73,50 +72,7 @@ class MainNetLedger(BaseLedger):
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return await self.validate_resolutions(result)
async def validate_resolutions(self, resolutions: dict):
provables = {}
for url, txo in resolutions.items():
if isinstance(txo, dict) and 'error' in txo:
continue
parsed_url = URL.parse(url)
# cases we check: names without sequence, claim id or any modifier
# except: @channel/name as the signature is enough, instead we check '@channel'
if parsed_url.has_channel:
if not parsed_url.has_stream_in_channel and parsed_url.channel.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.channel.name), []).append((url, txo.id))
elif parsed_url.has_stream and parsed_url.stream.to_dict().keys() == {'name'}:
provables.setdefault(normalize_name(parsed_url.stream.name), []).append((url, txo.id))
if not provables:
return resolutions
root_hash = self.headers.claim_trie_root.decode()
names = list(provables.keys())
proofs = await self.network.get_name_proofs(self.headers.hash().decode(), *names)
for proof, name in zip(proofs, names):
all_failed = True
for url, txid in provables[name]:
if not proof.get('txhash'):
resolutions[url] = {'error': f"Proof mismatch on {url}: no claims under {name} but we got {txid}"}
continue
expected_txid = f"{proof['txhash']}:{proof['nOut']}"
if txid != expected_txid:
resolutions[url] = {
'error': f"Proof mismatch: {url} resolved to {txid} instead of {expected_txid} on {name}"
}
else:
all_failed = False
if not all_failed:
try:
verify_proof(proof, root_hash, name)
continue
except InvalidProofError as error:
for url, _ in provables[name]:
resolution = resolutions[url]
if isinstance(resolution, dict) and 'error' in resolution:
continue
resolutions[url] = {'error': f'Invalid proof for {url}: {str(error)}'}
return resolutions
return result
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
-2
View File
@@ -184,8 +184,6 @@ class LbryWalletManager(BaseWalletManager):
len(set(receiving_addresses).difference(set(migrated_receiving))))
def get_best_blockhash(self):
if len(self.ledger.headers) <= 0:
return self.ledger.genesis_hash
return self.ledger.headers.hash(self.ledger.headers.height).decode()
def get_unused_address(self):
+3 -3
View File
@@ -3,14 +3,14 @@ from torba.client.basenetwork import BaseNetwork
class Network(BaseNetwork):
def get_server_height(self):
return self.rpc('blockchain.block.get_server_height', [])
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def get_name_proofs(self, block_hash, *names):
return self.rpc('blockchain.claimtrie.getnameproofs', (block_hash, *names))
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
@@ -68,7 +68,6 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.search_cache = {}
def advance_blocks(self, blocks):
self.sql.begin()
@@ -79,12 +78,6 @@ class LBRYBlockProcessor(BlockProcessor):
raise
finally:
self.sql.commit()
if self.db.first_sync and self.height == self.daemon.cached_height():
self.timer.run(self.sql.db.executescript, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.db.executescript, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
-207
View File
@@ -11,210 +11,3 @@ STREAM_TYPES = {
'binary': 5,
'model': 6
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
+53 -208
View File
@@ -1,34 +1,17 @@
import time
import struct
import sqlite3
import logging
from typing import Tuple, List, Dict, Union, Type, Optional
import struct
from typing import Tuple, List
from binascii import unhexlify
from decimal import Decimal
from contextvars import ContextVar
from functools import wraps
from dataclasses import dataclass
from torba.client.basedatabase import query, interpolate
from torba.client.basedatabase import query
from lbry.schema.url import URL, normalize_name
from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
class SQLiteOperationalError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query errored')
self.metrics = metrics
class SQLiteInterruptedError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query interrupted')
self.metrics = metrics
from .common import CLAIM_TYPES, STREAM_TYPES
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -56,7 +39,7 @@ SEARCH_PARAMS = {
ORDER_FIELDS = {
'name', 'claim_hash'
'name',
} | INTEGER_PARAMS
@@ -65,119 +48,25 @@ PRAGMAS = """
"""
@dataclass
class ReaderState:
db: sqlite3.Connection
stack: List[List]
metrics: Dict
is_tracking_metrics: bool
ledger: Type[BaseLedger]
query_timeout: float
log: logging.Logger
def close(self):
self.db.close()
def reset_metrics(self):
self.stack = []
self.metrics = {}
def set_query_timeout(self):
stop_at = time.perf_counter() + self.query_timeout
def interruptor():
if time.perf_counter() >= stop_at:
self.db.interrupt()
return
self.db.set_progress_handler(interruptor, 100)
db = ContextVar('db')
ledger = ContextVar('ledger')
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
db = sqlite3.connect(_path, isolation_level=None, uri=True)
db.row_factory = sqlite3.Row
ctx.set(
ReaderState(
db=db, stack=[], metrics={}, is_tracking_metrics=_measure,
ledger=MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger,
query_timeout=query_timeout, log=log
)
)
def initializer(_path, _ledger_name):
_db = sqlite3.connect(_path, isolation_level=None, uri=True)
_db.row_factory = sqlite3.Row
db.set(_db)
from lbry.wallet.ledger import MainNetLedger, RegTestLedger
ledger.set(MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger)
def cleanup():
ctx.get().close()
ctx.set(None)
db.get().close()
db.set(None)
ledger.set(None)
def measure(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
metric = {}
state.metrics.setdefault(func.__name__, []).append(metric)
state.stack.append([])
start = time.perf_counter()
try:
return func(*args, **kwargs)
finally:
elapsed = int((time.perf_counter()-start)*1000)
metric['total'] = elapsed
metric['isolated'] = (elapsed-sum(state.stack.pop()))
if state.stack:
state.stack[-1].append(elapsed)
return wrapper
def reports_metrics(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
state.reset_metrics()
r = func(*args, **kwargs)
return r, state.metrics
return wrapper
@reports_metrics
def search_to_bytes(constraints) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(search(constraints))
@reports_metrics
def resolve_to_bytes(urls) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(resolve(urls))
def encode_result(result):
return Outputs.to_bytes(*result)
@measure
def execute_query(sql, values) -> List:
context = ctx.get()
context.set_query_timeout()
try:
return context.db.execute(sql, values).fetchall()
except sqlite3.OperationalError as err:
plain_sql = interpolate(sql, values)
if context.is_tracking_metrics:
context.metrics['execute_query'][-1]['sql'] = plain_sql
if str(err) == "interrupted":
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
raise SQLiteInterruptedError(context.metrics)
context.log.exception('failed running query', exc_info=err)
raise SQLiteOperationalError(context.metrics)
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
def get_claims(cols, for_count=False, **constraints):
if 'order_by' in constraints:
sql_order_by = []
for order_by in constraints['order_by']:
@@ -231,7 +120,15 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'public_key_id' in constraints:
constraints['claim.public_key_hash'] = sqlite3.Binary(
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
ledger.get().address_to_hash160(constraints.pop('public_key_id')))
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = _resolve_one(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
if 'channel_hash' in constraints:
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_ids' in constraints:
@@ -300,23 +197,10 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)
return sql, values
return db.get().execute(sql, values).fetchall()
def get_claims(cols, for_count=False, **constraints) -> List:
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
sql, values = _get_claims(cols, for_count, **constraints)
return execute_query(sql, values)
@measure
def get_claims_count(**constraints) -> int:
def get_claims_count(**constraints):
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
@@ -324,7 +208,6 @@ def get_claims_count(**constraints) -> int:
return count[0][0]
@measure
def search(constraints) -> Tuple[List, List, int, int]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
@@ -334,17 +217,19 @@ def search(constraints) -> Tuple[List, List, int, int]:
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
constraints['order_by'] = ["height", "^name"]
txo_rows = _search(**constraints)
channel_hashes = set(txo['channel_hash'] for txo in txo_rows if txo['channel_hash'])
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
return txo_rows, extra_txo_rows, constraints['offset'], total
def search_to_bytes(constraints) -> bytes:
return Outputs.to_bytes(*search(constraints))
def _search(**constraints):
return get_claims(
"""
@@ -364,26 +249,25 @@ def _search(**constraints):
)
@measure
def resolve(urls) -> Tuple[List, List]:
result = []
channel_hashes = set()
for raw_url in urls:
match = resolve_url(raw_url)
match = _resolve_one(raw_url)
result.append(match)
if isinstance(match, sqlite3.Row) and match['channel_hash']:
channel_hashes.add(match['channel_hash'])
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
extra_txo_rows = _search(**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]})
return result, extra_txo_rows
@measure
def resolve_url(raw_url):
def resolve_to_bytes(urls) -> bytes:
return Outputs.to_bytes(*resolve(urls))
def _resolve_one(raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
@@ -408,7 +292,7 @@ def resolve_url(raw_url):
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount', '^height']
query['order_by'] = ['effective_amount']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
@@ -425,68 +309,28 @@ def resolve_url(raw_url):
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
not_items = set(cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
if attr == 'tag':
common_tags = any_items & COMMON_TAGS.keys()
if common_tags:
any_items -= common_tags
if len(common_tags) < 5:
for item in common_tags:
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
elif len(common_tags) >= 5:
constraints.update({
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
})
values = ', '.join(
f':$any_common_tag{i}' for i in range(len(common_tags))
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
any_items = cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
if for_count:
constraints[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
constraints[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
all_items = cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
@@ -509,6 +353,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
)
"""
not_items = cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH]
if not_items:
constraints.update({
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
+32 -40
View File
@@ -16,7 +16,7 @@ from lbry.wallet.server.db.trending import (
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
)
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -76,12 +76,33 @@ class SQLDB:
trending_global integer not null default 0
);
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_release_time_idx on claim (release_time);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_height_idx on claim (height);
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
create index if not exists claim_effective_amount_idx on claim (effective_amount);
create index if not exists claim_trending_group_idx on claim (trending_group);
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
create index if not exists claim_trending_local_idx on claim (trending_local);
create index if not exists claim_trending_global_idx on claim (trending_global);
"""
CREATE_SUPPORT_TABLE = """
@@ -92,6 +113,7 @@ class SQLDB:
claim_hash bytes not null,
amount integer not null
);
create index if not exists support_txo_hash_idx on support (txo_hash);
create index if not exists support_claim_hash_idx on support (claim_hash, height);
"""
@@ -101,7 +123,9 @@ class SQLDB:
claim_hash bytes not null,
height integer not null
);
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
create index if not exists tag_tag_idx on tag (tag);
create index if not exists tag_claim_hash_idx on tag (claim_hash);
create index if not exists tag_height_idx on tag (height);
"""
CREATE_CLAIMTRIE_TABLE = """
@@ -113,38 +137,6 @@ class SQLDB:
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
TAG_INDEXES = '\n'.join(
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
for tag_value, tag_key in COMMON_TAGS.items()
)
CREATE_TABLES_QUERY = (
PRAGMAS +
CREATE_CLAIM_TABLE +
@@ -208,7 +200,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags = [], [], {}
claim_hashes, claims, tags = [], [], []
for txo in txos:
tx = txo.tx_ref.tx
@@ -262,14 +254,14 @@ class SQLDB:
claim_record['claim_type'] = CLAIM_TYPES['channel']
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
tags.append((tag, claim_hash, tx.height))
if clear_first:
self._clear_claim_metadata(claim_hashes)
if tags:
self.db.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
"INSERT INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags
)
return claims
-135
View File
@@ -1,135 +0,0 @@
import time
import math
from typing import Tuple
def calculate_elapsed(start) -> int:
return int((time.perf_counter() - start) * 1000)
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
if not data:
return 0, 0, 0, 0, 0, 0, 0, 0
data.sort()
size = len(data)
return (
int(sum(data) / size),
data[0],
data[math.ceil(size * .05) - 1],
data[math.ceil(size * .25) - 1],
data[math.ceil(size * .50) - 1],
data[math.ceil(size * .75) - 1],
data[math.ceil(size * .95) - 1],
data[-1]
)
def remove_select_list(sql) -> str:
return sql[sql.index('FROM'):]
class APICallMetrics:
def __init__(self, name):
self.name = name
# total requests received
self.receive_count = 0
self.cache_response_count = 0
# millisecond timings for query based responses
self.query_response_times = []
self.query_intrp_times = []
self.query_error_times = []
self.query_python_times = []
self.query_wait_times = []
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
self.individual_sql_times = [] # every SQL query run on server
# actual queries
self.errored_queries = set()
self.interrupted_queries = set()
def to_json_and_reset(self):
return {
# total requests received
"receive_count": self.receive_count,
# sum of these is total responses made
"cache_response_count": self.cache_response_count,
"query_response_count": len(self.query_response_times),
"intrp_response_count": len(self.query_intrp_times),
"error_response_count": len(self.query_error_times),
# millisecond timings for non-cache responses
"response": calculate_avg_percentiles(self.query_response_times),
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
"error": calculate_avg_percentiles(self.query_error_times),
# response, interrupt and error each also report the python, wait and sql stats:
"python": calculate_avg_percentiles(self.query_python_times),
"wait": calculate_avg_percentiles(self.query_wait_times),
"sql": calculate_avg_percentiles(self.query_sql_times),
# extended timings for individual sql executions
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
"individual_sql_count": len(self.individual_sql_times),
# actual queries
"errored_queries": list(self.errored_queries),
"interrupted_queries": list(self.interrupted_queries),
}
def start(self):
self.receive_count += 1
def cache_response(self):
self.cache_response_count += 1
def _add_query_timings(self, request_total_time, metrics):
if metrics and 'execute_query' in metrics:
sub_process_total = metrics[self.name][0]['total']
individual_query_times = [f['total'] for f in metrics['execute_query']]
aggregated_query_time = sum(individual_query_times)
self.individual_sql_times.extend(individual_query_times)
self.query_sql_times.append(aggregated_query_time)
self.query_python_times.append(sub_process_total - aggregated_query_time)
self.query_wait_times.append(request_total_time - sub_process_total)
@staticmethod
def _add_queries(query_set, metrics):
if metrics and 'execute_query' in metrics:
for execute_query in metrics['execute_query']:
if 'sql' in execute_query:
query_set.add(remove_select_list(execute_query['sql']))
def query_response(self, start, metrics):
self.query_response_times.append(calculate_elapsed(start))
self._add_query_timings(self.query_response_times[-1], metrics)
def query_interrupt(self, start, metrics):
self.query_intrp_times.append(calculate_elapsed(start))
self._add_queries(self.interrupted_queries, metrics)
self._add_query_timings(self.query_intrp_times[-1], metrics)
def query_error(self, start, metrics):
self.query_error_times.append(calculate_elapsed(start))
self._add_queries(self.errored_queries, metrics)
self._add_query_timings(self.query_error_times[-1], metrics)
class ServerLoadData:
def __init__(self):
self._apis = {}
def for_api(self, name) -> APICallMetrics:
if name not in self._apis:
self._apis[name] = APICallMetrics(name)
return self._apis[name]
def to_json_and_reset(self, status):
try:
return {
'api': {name: api.to_json_and_reset() for name, api in self._apis.items()},
'status': status
}
finally:
self._apis = {}
+11 -110
View File
@@ -1,40 +1,17 @@
import os
import math
import time
import base64
import asyncio
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from torba.rpc.jsonrpc import RPCError, JSONRPC
from torba.rpc.jsonrpc import RPCError
from torba.server.session import ElectrumX, SessionManager
from torba.server import util
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
class ResultCacheItem:
__slots__ = '_result', 'lock', 'has_result'
def __init__(self):
self.has_result = asyncio.Event()
self.lock = asyncio.Lock()
self._result = None
@property
def result(self) -> str:
return self._result
@result.setter
def result(self, result: str):
self._result = result
if result is not None:
self.has_result.set()
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
class LBRYSessionManager(SessionManager):
@@ -42,54 +19,23 @@ class LBRYSessionManager(SessionManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.query_executor = None
self.websocket = None
self.metrics = ServerLoadData()
self.metrics_loop = None
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
async def process_metrics(self):
while self.running:
data = self.metrics.to_json_and_reset({'sessions': self.session_count()})
if self.websocket is not None:
self.websocket.send_message(data)
await asyncio.sleep(1)
async def start_other(self):
self.running = True
args = dict(
initializer=reader.initializer,
initargs=(self.logger, 'claims.db', self.env.coin.NET, self.env.database_query_timeout,
self.env.track_metrics)
)
args = dict(initializer=reader.initializer, initargs=('claims.db', self.env.coin.NET))
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
self.query_executor = ThreadPoolExecutor(max_workers=1, **args)
else:
self.query_executor = ProcessPoolExecutor(
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args
)
if self.websocket is not None:
await self.websocket.start()
if self.env.track_metrics:
self.metrics_loop = asyncio.create_task(self.process_metrics())
async def stop_other(self):
self.running = False
if self.env.track_metrics:
self.metrics_loop.cancel()
if self.websocket is not None:
await self.websocket.stop()
self.query_executor.shutdown()
class LBRYElectrumX(ElectrumX):
PROTOCOL_MIN = (0, 0) # temporary, for supporting 0.10 protocol
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -104,71 +50,26 @@ class LBRYElectrumX(ElectrumX):
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getnameproofs': self.claimtrie_getnameproofs,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': self.get_server_height,
}
self.request_handlers.update(handlers)
async def run_in_executor(self, metrics: APICallMetrics, func, kwargs):
start = time.perf_counter()
try:
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
except reader.SQLiteInterruptedError as error:
metrics.query_interrupt(start, error.metrics)
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics.query_error(start, error.metrics)
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except:
metrics.query_error(start, {})
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
if self.env.track_metrics:
(result, metrics_data) = result
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
async def run_and_cache_query(self, query_name, function, kwargs):
if self.env.track_metrics:
metrics = self.session_mgr.metrics.for_api(query_name)
else:
metrics = APICallMetrics(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
return cache_item.result
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
metrics, function, kwargs
)
else:
metrics.cache_response()
return cache_item.result
async def claimtrie_search(self, **kwargs):
if kwargs:
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
if 'claim_id' in kwargs:
self.assert_claim_id(kwargs['claim_id'])
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, reader.search_to_bytes, kwargs
)).decode()
async def claimtrie_resolve(self, *urls):
if urls:
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
return base64.b64encode(await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, reader.resolve_to_bytes, urls
)).decode()
async def get_server_height(self):
return self.bp.height
def claimtrie_getnameproofs(self, block_hash, *names):
return self.daemon._send_vector('getnameproof', iter((name, block_hash) for name in names))
async def transaction_get_height(self, tx_hash):
self.assert_tx_hash(tx_hash)
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
-55
View File
@@ -1,55 +0,0 @@
import asyncio
from weakref import WeakSet
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
-146
View File
@@ -1,146 +0,0 @@
import os
import time
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
DEFAULT_ANY_TAGS = [
'blockchain',
'news',
'learning',
'technology',
'automotive',
'economics',
'food',
'science',
'art',
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
'mature',
'xxx'
]
ORDER_BY = [
[
"trending_global",
"trending_mixed",
],
[
"release_time"
],
[
"effective_amount"
]
]
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
args.append(kw)
print(len(args), "argument combinations")
return args
def _search(kwargs):
start = time.time()
try:
search_to_bytes(kwargs)
t = time.time() - start
return t, kwargs
except Exception as err:
return -1, f"failed: error={str(type(err))}({str(err)})"
async def search(executor, kwargs):
try:
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
except Exception as err:
return f"failed (err={str(type(err))}({err}))- {kwargs}"
async def main(db_path, max_query_time):
args = dict(initializer=initializer, initargs=(log, db_path, MainNetLedger, 0.25))
workers = max(os.cpu_count(), 4)
log.info(f"using {workers} reader processes")
query_executor = ProcessPoolExecutor(workers, **args)
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
for ts, constraints in results:
if ts >= max_query_time:
sql = interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints))
print(f"Query took {int(ts * 1000)}ms\n{sql}")
finally:
query_executor.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
asyncio.run(main(db_path, max_query_time))
-107
View File
@@ -1,107 +0,0 @@
import time
import asyncio
import random
from argparse import ArgumentParser
from torba.client.basenetwork import ClientSession
class AgentSmith(ClientSession):
async def do_nefarious_things(self):
await self.send_request('blockchain.claimtrie.search', {
'no_totals': True,
'offset': random.choice(range(0, 300, 20)),
'limit': 20,
'any_tags': (
random.choice([[
random.choice(['gaming', 'games', 'game']) +
random.choice(['entertainment', 'playthrough', 'funny']) +
random.choice(['xbox', 'xbox one', 'xbox news'])
], [
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
random.choice(['news', 'sighting', 'sightings'])
], [
random.choice(['art', 'automotive']),
random.choice(['blockchain', 'economics', 'food']),
random.choice(['funny', 'learnings', 'nature']),
random.choice(['news', 'science', 'technology'])
]
])
),
'not_tags': random.choice([[], [
'porn', 'mature', 'xxx', 'nsfw'
]]),
'order_by': random.choice([
['release_time'],
['trending_global', 'trending_mixed'],
['effective_amount']
])
})
class AgentSmithProgram:
def __init__(self, host, port):
self.host, self.port = host, port
self.agent_smiths = []
async def make_one_more_of_them(self):
smith = AgentSmith(network=None, server=(self.host, self.port))
await smith.create_connection()
self.agent_smiths.append(smith)
async def coordinate_nefarious_activity(self):
start = time.perf_counter()
await asyncio.gather(
*(s.do_nefarious_things() for s in self.agent_smiths),
return_exceptions=True
)
return time.perf_counter() - start
def __len__(self):
return len(self.agent_smiths)
async def delete_one_smith(self):
if self.agent_smiths:
await self.agent_smiths.pop().close()
async def delete_program(self):
await asyncio.gather(*(
s.close() for s in self.agent_smiths
))
async def main(host, port):
smiths = AgentSmithProgram(host, port)
await smiths.make_one_more_of_them()
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
ease_off = 0
for i in range(1000):
await asyncio.sleep(1)
if activity.done() and activity.result() < .9:
print('more, more, more...')
await asyncio.gather(*(
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
))
else:
print('!!!!!!!!!!!!!!')
print('IS NEO LOSING?')
print('!!!!!!!!!!!!!!')
await asyncio.gather(*(
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
))
print(f'coordinate all {len(smiths)} smiths to action')
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
print('finishing up any remaining actions')
await activity
print('neo has won, deleting agents...')
await smiths.delete_program()
print('done.')
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--host', dest='host', default='localhost', type=str)
parser.add_argument('--port', dest='port', default=50001, type=int)
args = parser.parse_args()
asyncio.run(main(args.host, args.port))
+61 -32
View File
@@ -74,19 +74,28 @@ def get_release_text(desc: str):
yield line[len(RELEASE_TEXT):]
def get_previous_final(repo, current_release):
assert current_release.rc is not None, "Need an rc to find the previous final release."
previous = None
for tag in repo.tags(current_release.rc+1):
previous = tag
return previous
class Version:
def __init__(self, major=0, minor=0, micro=0):
def __init__(self, major=0, minor=0, micro=0, rc=None):
self.major = int(major)
self.minor = int(minor)
self.micro = int(micro)
self.rc = rc if rc is None else int(rc)
@classmethod
def from_string(cls, version_string):
(major, minor, micro), rc = version_string.split('.'), None
if 'rc' in micro:
micro, rc = micro.split('rc')
return cls(major, minor, micro)
return cls(major, minor, micro, rc)
@classmethod
def from_content(cls, content):
@@ -97,12 +106,20 @@ class Version:
def increment(self, action):
cls = self.__class__
if action == 'major':
return cls(self.major+1)
elif action == 'minor':
return cls(self.major, self.minor+1)
elif action == 'micro':
return cls(self.major, self.minor, self.micro+1)
if action == '*-rc':
assert self.rc is not None, f"Can't drop rc designation because {self} is already not an rc."
return cls(self.major, self.minor, self.micro)
elif action == '*+rc':
assert self.rc is not None, "Must already be an rc to increment."
return cls(self.major, self.minor, self.micro, self.rc+1)
assert self.rc is None, f"Can't start a new rc because {self} is already an rc."
if action == 'major+rc':
return cls(self.major+1, rc=1)
elif action == 'minor+rc':
return cls(self.major, self.minor+1, rc=1)
elif action == 'micro+rc':
return cls(self.major, self.minor, self.micro+1, 1)
raise ValueError(f'unknown action: {action}')
@@ -111,7 +128,10 @@ class Version:
return f'v{self}'
def __str__(self):
return '.'.join(str(p) for p in [self.major, self.minor, self.micro])
version = '.'.join(str(p) for p in [self.major, self.minor, self.micro])
if self.rc is not None:
version += f'rc{self.rc}'
return version
def release(args):
@@ -119,15 +139,15 @@ def release(args):
repo = gh.repository('lbryio', 'lbry-sdk')
version_file = repo.file_contents('lbry/lbry/__init__.py')
if not args.confirm:
print("\nDRY RUN ONLY. RUN WITH --confirm TO DO A REAL RELEASE.\n")
current_version = Version.from_content(version_file)
print(f'Current Version: {current_version}')
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(current_version.tag)
if args.action == '*-rc':
previous_release = repo.release_from_tag(args.start_tag or get_previous_final(repo, current_version))
else:
previous_release = repo.release_from_tag(current_version.tag)
print(f' Changelog From: {previous_release.tag_name} ({previous_release.created_at})')
print()
@@ -145,7 +165,7 @@ def release(args):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body):
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
if type_label != 'fixup':
if not (args.action == '*-rc' and type_label == 'fixup'):
area = areas.setdefault(area_name, [])
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
else:
@@ -183,7 +203,7 @@ def release(args):
for skipped in unlabeled:
print(skipped)
if args.confirm:
if not args.dry_run:
commit = version_file.update(
new_version.tag,
@@ -203,10 +223,9 @@ def release(args):
name=new_version.tag,
body=body.getvalue(),
draft=True,
prerelease=new_version.rc is not None
)
return 0
class TestReleaseTool(unittest.TestCase):
@@ -216,34 +235,44 @@ class TestReleaseTool(unittest.TestCase):
def test_version_increment(self):
v = Version.from_string('1.2.3')
self.assertTrue(str(v.increment('major')), '2.0.0')
self.assertTrue(str(v.increment('minor')), '1.3.0')
self.assertTrue(str(v.increment('micro')), '1.2.4')
self.assertTrue(str(v.increment('major+rc')), '2.0.0rc1')
self.assertTrue(str(v.increment('minor+rc')), '1.3.0rc1')
self.assertTrue(str(v.increment('micro+rc')), '1.2.4rc1')
with self.assertRaisesRegex(AssertionError, "Must already be an rc to increment."):
v.increment('*+rc')
with self.assertRaisesRegex(AssertionError, "Can't drop rc designation"):
v.increment('*-rc')
v = Version.from_string('1.2.3rc3')
self.assertTrue(str(v.increment('*+rc')), '1.2.3rc4')
self.assertTrue(str(v.increment('*-rc')), '1.2.3')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('major+rc')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('minor+rc')
with self.assertRaisesRegex(AssertionError, "already an rc"):
v.increment('micro+rc')
def test():
runner = unittest.TextTestRunner(verbosity=2)
loader = unittest.TestLoader()
suite = loader.loadTestsFromTestCase(TestReleaseTool)
return 0 if runner.run(suite).wasSuccessful() else 1
runner.run(suite)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--confirm", default=False, action="store_true",
help="without this flag, it will only print what it will do but will not actually do it")
parser.add_argument("--test", default=False, action="store_true", help="run unit tests")
parser.add_argument("--dry-run", default=False, action="store_true", help="show what will be done")
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
parser.add_argument("action", nargs="?", choices=['major+rc', 'minor+rc', 'micro+rc', '*+rc', '*-rc'])
args = parser.parse_args()
if args.action == "test":
code = test()
if args.test:
test()
else:
code = release(args)
print()
return code
release(args)
if __name__ == "__main__":
sys.exit(main())
main()
+61 -40
View File
@@ -1,60 +1,81 @@
import uvloop, asyncio, time, sys, logging
import uvloop, asyncio, sqlite3, time, sys
from concurrent.futures import ProcessPoolExecutor
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
from contextvars import ContextVar
db_path = '../../../lbryconf/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
db_path = '/tmp/wallet-server/claims.db'
db = ContextVar('db')
def init():
conn = sqlite3.connect(db_path)
db.set(conn)
def reader():
conn = db.get()
start = time.time()
conn.execute("""
SELECT
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
FROM claim
LEFT JOIN claimtrie USING (claim_hash)
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
WHERE
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ('alexandria ocasio-cortez', 'Alien', 'alt news', 'art', 'audio',
'automotive', 'beliefs', 'blockchain', 'dog grooming', 'economics', 'food',
'learning', 'mature', 'nature', 'news', 'physics', 'science', 'technology')
)
AND NOT EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash AND tag IN ('nsfw', 'xxx', 'mature')
)
ORDER BY claim.height DESC, claim.normalized ASC
LIMIT 20 OFFSET 100
""").fetchall()
elapsed = time.time() - start
return elapsed
async def run_times(executor, iterations, show=True):
start = time.perf_counter()
timings = await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(
executor, reader.search_to_bytes, {
'no_totals': True,
'offset': 0,
'limit': 20,
'any_tags': [
'ufos', 'city fix'
],
'not_tags': [
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query'][0]['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
start = time.time()
timings = await asyncio.gather(*(
asyncio.get_running_loop().run_in_executor(executor, reader) for _ in range(iterations)
))
total = time.time() - start
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
print(f"{iterations:4}: {total:.5f}ms total concurrent, {len(timings)*avg:.5f}ms total sequential (avg*runs)")
print(f" {total/len(timings):.5f}ms/query concurrent (total/runs)")
print(f" {avg:.5f}ms/query actual average (sum(queries)/runs)")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t}ms')
sys.stdout.write(f' {t:.5f}ms')
print(' ...\n' if len(timings) > 10 else '\n')
async def main():
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
executor = ProcessPoolExecutor(4, initializer=init)
await run_times(executor, 4, show=False)
await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
await run_times(executor, 2**9)
await run_times(executor, 2**11)
await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
@@ -89,7 +89,6 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim], **kwargs)
async def assertFindsClaims(self, claims, **kwargs):
kwargs.setdefault('order_by', ['height', '^name'])
results = await self.claim_search(**kwargs)
self.assertEqual(len(claims), len(results))
for claim, result in zip(claims, results):
@@ -166,15 +165,15 @@ class ClaimSearchCommand(ClaimTestCase):
await self.create_channel()
await self.create_lots_of_streams()
page = await self.claim_search(page_size=20, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page_size=20, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams)
page = await self.claim_search(page_size=6, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[:6])
page = await self.claim_search(page=2, page_size=6, channel='@abc', order_by=['height', '^name'])
page = await self.claim_search(page=2, page_size=6, channel='@abc')
page_claim_ids = [item['name'] for item in page]
self.assertEqual(page_claim_ids, self.streams[6:])
@@ -226,7 +225,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim4, claim3, claim2], all_tags=['abc'], any_tags=['def', 'ghi'])
async def test_order_by(self):
height = self.ledger.network.remote_height
height = await self.ledger.network.get_server_height()
claims = [await self.stream_create(f'claim{i}') for i in range(5)]
await self.assertFindsClaims(claims, order_by=["^height"])
@@ -1,31 +1,130 @@
import asyncio
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import socket
import logging
from unittest.mock import Mock
from torba.testcase import IntegrationTestCase, Conductor
import lbry.wallet
from lbry.schema.claim import Claim
from lbry.wallet.transaction import Transaction, Output
from lbry.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
log = logging.getLogger(__name__)
def wrap_callback_event(fn, callback):
def inner(*a, **kw):
callback()
return fn(*a, **kw)
return inner
class TestSessionBloat(IntegrationTestCase):
"""
Tests that server cleans up stale connections after session timeout and client times out too.
ERROR:asyncio:Fatal read error on socket transport
protocol: <lbrynet.wallet.server.session.LBRYElectrumX object at 0x7f7e3bfcaf60>
transport: <_SelectorSocketTransport fd=3236 read=polling write=<idle, bufsize=0>>
Traceback (most recent call last):
File "/usr/lib/python3.7/asyncio/selector_events.py", line 801, in _read_ready__data_received
data = self._sock.recv(self.max_size)
TimeoutError: [Errno 110] Connection timed out
"""
LEDGER = lbry.wallet
async def test_session_bloat_from_socket_timeout(self):
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
async def asyncSetUp(self):
self.conductor = Conductor(
ledger_module=self.LEDGER, manager_module=self.MANAGER, verbosity=self.VERBOSITY
)
await self.conductor.start_blockchain()
self.addCleanup(self.conductor.stop_blockchain)
await self.conductor.start_spv()
session = ClientSession(network=None, server=self.ledger.network.client.server, timeout=0.2)
await session.create_connection()
session.ping_task.cancel()
await session.send_request('server.banner', ())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 1)
self.assertFalse(session.is_closing())
await asyncio.sleep(1.1)
with self.assertRaises(asyncio.TimeoutError):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
self.session_manager = self.conductor.spv_node.server.session_mgr
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64)
self.session_manager.servers['TCP'].sockets[0].setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 64)
self.addCleanup(self.conductor.stop_spv)
await self.conductor.start_wallet()
self.addCleanup(self.conductor.stop_wallet)
self.client_session = list(self.session_manager.sessions)[0]
self.client_session.transport.set_write_buffer_limits(0, 0)
self.paused_session = asyncio.Event(loop=self.loop)
self.resumed_session = asyncio.Event(loop=self.loop)
def paused():
self.resumed_session.clear()
self.paused_session.set()
def delayed_resume():
self.paused_session.clear()
self.resumed_session.set()
self.client_session.pause_writing = wrap_callback_event(self.client_session.pause_writing, paused)
self.client_session.resume_writing = wrap_callback_event(self.client_session.resume_writing, delayed_resume)
self.blockchain = self.conductor.blockchain_node
self.wallet_node = self.conductor.wallet_node
self.manager = self.wallet_node.manager
self.ledger = self.wallet_node.ledger
self.wallet = self.wallet_node.wallet
self.account = self.wallet_node.wallet.default_account
async def test_session_bloat_from_socket_timeout(self):
await self.account.ensure_address_gap()
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
await self.blockchain.generate(1)
await asyncio.wait([
self.on_transaction_id(sendtxid1),
self.on_transaction_id(sendtxid2)
])
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
channel = Claim()
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.description = "0" * 8000
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
stream_tx = await Transaction.create([], [stream_txo], [self.account], self.account)
stream_txo.sign(channel_txo)
await stream_tx.sign([self.account])
self.paused_session.clear()
self.resumed_session.clear()
await self.broadcast(channel_tx)
await self.broadcast(stream_tx)
await asyncio.wait_for(self.paused_session.wait(), 2)
self.assertEqual(1, len(self.session_manager.sessions))
real_sock = self.client_session.transport._extra.pop('socket')
mock_sock = Mock(spec=socket.socket)
for attr in dir(real_sock):
if not attr.startswith('__'):
setattr(mock_sock, attr, getattr(real_sock, attr))
def recv(*a, **kw):
raise TimeoutError("[Errno 110] Connection timed out")
mock_sock.recv = recv
self.client_session.transport._sock = mock_sock
self.client_session.transport._extra['socket'] = mock_sock
self.assertFalse(self.resumed_session.is_set())
self.assertFalse(self.session_manager.session_event.is_set())
await self.session_manager.session_event.wait()
self.assertEqual(0, len(self.session_manager.sessions))
-12
View File
@@ -17,18 +17,6 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
async def test_sync_blob_file_manager_on_startup(self):
await self.setup_blob_manager(save_blobs=True)
-7
View File
@@ -72,13 +72,6 @@ class TestLanguages(TestCase):
self.assertEqual(stream.languages[2].region, 'US')
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US'])
stream.languages.append('es-419')
self.assertEqual(stream.languages[3].langtag, 'es-419')
self.assertEqual(stream.languages[3].language, 'es')
self.assertEqual(stream.languages[3].script, None)
self.assertEqual(stream.languages[3].region, '419')
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US', 'es-419'])
stream = Stream()
stream.languages.extend(['en-Latn-US', 'es-ES', 'de-DE'])
self.assertEqual(stream.languages[0].language, 'en')
+1 -5
View File
@@ -11,13 +11,9 @@ class TestTagNormalization(unittest.TestCase):
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'T\'ag')
tag('tag', 'Tag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
def test_clean_tags(self):
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))
cleaned = clean_tags(['fOo', '!taG', 'FoO'])
self.assertIn('tag', cleaned)
self.assertIn('foo', cleaned)
self.assertEqual(len(cleaned), 2)
+1 -10
View File
@@ -40,15 +40,6 @@ class TestManagedStream(BlobExchangeTestBase):
self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir
)
async def test_status_file_completed(self):
await self._test_transfer_stream(10)
self.assertTrue(self.stream.output_file_exists)
self.assertTrue(self.stream.as_dict()['completed'])
with open(self.stream.full_path, 'w+b') as outfile:
outfile.truncate(1)
self.assertTrue(self.stream.output_file_exists)
self.assertFalse(self.stream.as_dict()['completed'])
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
await self.setup_stream(blob_count)
mock_node = mock.Mock(spec=Node)
@@ -61,7 +52,7 @@ class TestManagedStream(BlobExchangeTestBase):
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
await self.stream.finished_write_attempt.wait()
await self.stream.finished_writing.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
if stop_when_done:
await self.stream.stop()
@@ -1,60 +0,0 @@
import time
import unittest
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
class TestPercentileCalculation(unittest.TestCase):
def test_calculate_percentiles(self):
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
self.assertEqual(calculate_avg_percentiles(
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
class TestCollectingMetrics(unittest.TestCase):
def test_happy_path(self):
self.maxDiff = None
load = ServerLoadData()
search = load.for_api('search')
self.assertEqual(search.name, 'search')
search.start()
search.cache_response()
search.cache_response()
metrics = {
'search': [{'total': 40}],
'execute_query': [
{'total': 20},
{'total': 10}
]
}
for x in range(5):
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
metrics['execute_query'][0]['total'] = 10
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
search.query_interrupt(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.052, {})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
"receive_count": 1,
"cache_response_count": 2,
"query_response_count": 5,
"intrp_response_count": 1,
"error_response_count": 2,
"response": (53, 51, 51, 52, 53, 54, 55, 55),
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
"error": (51, 50, 50, 50, 50, 52, 52, 52),
"python": (12, 10, 10, 10, 10, 20, 20, 20),
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
"individual_sql_count": 14,
"errored_queries": ['FROM claim where something=1'],
"interrupted_queries": ['FROM claim where something=1'],
}}})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
+1 -3
View File
@@ -1,7 +1,6 @@
import unittest
import ecdsa
import hashlib
import logging
from binascii import hexlify
from torba.client.constants import COIN, NULL_HASH32
@@ -37,7 +36,6 @@ class OldWalletServerTransaction:
class TestSQLDB(unittest.TestCase):
query_timeout = 0.25
def setUp(self):
self.first_sync = False
@@ -46,7 +44,7 @@ class TestSQLDB(unittest.TestCase):
db_url = 'file:test_sqldb?mode=memory&cache=shared'
self.sql = writer.SQLDB(self, db_url)
self.addCleanup(self.sql.close)
reader.initializer(logging.getLogger(__name__), db_url, 'regtest', self.query_timeout)
reader.initializer(db_url, 'regtest')
self.addCleanup(reader.cleanup)
self.timer = Timer('BlockProcessor')
self.sql.open()
@@ -1,5 +1,6 @@
import logging
import asyncio
import socket
from unittest.mock import Mock
from torba.client.basenetwork import BaseNetwork
@@ -7,15 +8,6 @@ from torba.rpc import RPCSession
from torba.testcase import IntegrationTestCase, AsyncioTestCase
class NetworkTests(IntegrationTestCase):
async def test_remote_height_updated_automagically(self):
initial_height = self.ledger.network.remote_height
await self.blockchain.generate(1)
await self.ledger.network.on_header.first
self.assertEqual(self.ledger.network.remote_height, initial_height + 1)
class ReconnectTests(IntegrationTestCase):
VERBOSITY = logging.WARN
@@ -51,47 +43,75 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_socket_timeout_then_reconnect(self):
# TODO: test reconnecting on an rpc request
# TODO: test rolling over to a working server when an rpc request fails before raising
self.assertTrue(self.ledger.network.is_connected)
address1 = await self.account.receiving.get_or_create_usable_address()
txid = await self.blockchain.send_to_address(address1, 21)
await self.blockchain.generate(1)
real_sock = self.ledger.network.client.transport._extra.pop('socket')
mock_sock = Mock(spec=socket.socket)
for attr in dir(real_sock):
if not attr.startswith('__'):
setattr(mock_sock, attr, getattr(real_sock, attr))
raised = asyncio.Event(loop=self.loop)
def recv(*a, **kw):
raised.set()
raise TimeoutError("[Errno 60] Operation timed out")
mock_sock.recv = recv
self.ledger.network.client.transport._sock = mock_sock
self.ledger.network.client.transport._extra['socket'] = mock_sock
self.assertFalse(raised.is_set())
with self.assertRaises(asyncio.CancelledError):
await self.ledger.network.get_transaction(txid)
self.assertTrue(raised.is_set())
self.assertFalse(self.ledger.network.is_connected)
self.assertIsNone(self.ledger.network.client.transport)
txid = await self.blockchain.send_to_address(address1, 2)
await self.blockchain.generate(1)
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
self.assertIsNotNone(self.ledger.network.client.transport)
self.assertIsNotNone(await self.ledger.network.get_transaction(txid))
class ServerPickingTestCase(AsyncioTestCase):
async def _make_fake_server(self, latency=1.0, port=1):
async def _make_fake_server(self, latency=1.0, port=1337):
# local fake server with artificial latency
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
return {"height": 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
proto = RPCSession()
proto.handle_request = lambda _: asyncio.sleep(latency)
server = await self.loop.create_server(lambda: proto, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
async def _make_bad_server(self, port=42420):
async def echo(reader, writer):
while True:
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
return ('127.0.0.1', port)
async def test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
# fast but unhealthy, should be discarded
await self._make_bad_server(),
('localhost', 1),
('example.that.doesnt.resolve', 9000),
await self._make_fake_server(latency=1.2, port=1340),
await self._make_fake_server(latency=0.5, port=1337),
await self._make_fake_server(latency=0.7, port=1339),
await self._make_fake_server(latency=1.5, port=1340),
await self._make_fake_server(latency=0.1, port=1337),
await self._make_fake_server(latency=1.0, port=1339),
await self._make_fake_server(latency=0.5, port=1338),
],
'connect_timeout': 3
'connect_timeout': 30
})
network = BaseNetwork(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=3)
await asyncio.wait_for(network.on_connected.first, timeout=1)
self.assertTrue(network.is_connected)
self.assertEqual(network.client.server, ('127.0.0.1', 1337))
# ensure we are connected to all of them
self.assertEqual(len(network.session_pool.sessions), 4)
self.assertTrue(all([not session.is_closing() for session in network.session_pool.sessions]))
self.assertEqual(len(network.session_pool.sessions), 3)
+2 -21
View File
@@ -1,10 +1,11 @@
import unittest
import sqlite3
from functools import wraps
from torba.client.wallet import Wallet
from torba.client.constants import COIN
from torba.coin.bitcoinsegwit import MainNetLedger as ledger_class
from torba.client.basedatabase import query, interpolate, constraints_to_sql, AIOSQLite
from torba.client.basedatabase import query, constraints_to_sql, AIOSQLite
from torba.client.hash import sha256
from torba.testcase import AsyncioTestCase
@@ -166,26 +167,6 @@ class TestQueryBuilder(unittest.TestCase):
("select * from foo LIMIT 20 OFFSET 10", {})
)
def test_query_interpolation(self):
self.maxDiff = None
# tests that interpolation replaces longer keys first
self.assertEqual(
interpolate(*query(
"select * from foo",
a__not='b', b__in='select * from blah where c=:$c',
d__any={'one__like': 'o', 'two': 2},
a0=3, a00=1, a00a=2, a00aa=4, # <-- breaks without correct interpolation key order
ahash=memoryview(sha256(b'hello world')),
limit=10, order_by='b', **{'$c': 3})
),
"select * from foo WHERE a != 'b' AND "
"b IN (select * from blah where c=3) AND "
"(one LIKE 'o' OR two = 2) AND "
"a0 = 3 AND a00 = 1 AND a00a = 2 AND a00aa = 4 "
"AND ahash = X'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' "
"ORDER BY b LIMIT 10",
)
class TestQueries(AsyncioTestCase):
@@ -1,6 +1,4 @@
import asyncio
import os
import tempfile
from urllib.request import Request, urlopen
from torba.testcase import AsyncioTestCase
@@ -93,42 +91,3 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
self.assertEqual(headers.height, 32250)
await headers.connect(len(headers), remainder)
self.assertEqual(headers.height, 32259)
async def test_repair(self):
headers = MainHeaders(':memory:')
await headers.connect(0, self.get_bytes(block_bytes(3001)))
self.assertEqual(headers.height, 3000)
await headers.repair()
self.assertEqual(headers.height, 3000)
# corrupt the middle of it
headers.io.seek(block_bytes(1500))
headers.io.write(b"wtf")
await headers.repair()
self.assertEqual(headers.height, 1499)
self.assertEqual(len(headers), 1500)
# corrupt by appending
headers.io.seek(block_bytes(len(headers)))
headers.io.write(b"appending")
headers._size = None
await headers.repair()
self.assertEqual(headers.height, 1499)
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
self.assertEqual(headers.height, 3000)
async def test_concurrency(self):
BLOCKS = 30
headers_temporary_file = tempfile.mktemp()
headers = MainHeaders(headers_temporary_file)
await headers.open()
self.addCleanup(os.remove, headers_temporary_file)
async def writer():
for block_index in range(BLOCKS):
await headers.connect(block_index, self.get_bytes(block_bytes(block_index + 1), block_bytes(block_index)))
async def reader():
for block_index in range(BLOCKS):
while len(headers) < block_index:
await asyncio.sleep(0.000001)
assert headers[block_index]['block_height'] == block_index
reader_task = asyncio.create_task(reader())
await writer()
await reader_task
-14
View File
@@ -1,7 +1,6 @@
import logging
import asyncio
from asyncio import wrap_future
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable
@@ -192,19 +191,6 @@ def query(select, **constraints):
return ' '.join(sql), values
def interpolate(sql, values):
for k in sorted(values.keys(), reverse=True):
value = values[k]
if isinstance(value, memoryview):
value = f"X'{hexlify(bytes(value)).decode()}'"
elif isinstance(value, str):
value = f"'{value}'"
else:
value = str(value)
sql = sql.replace(f":{k}", value)
return sql
def rows_to_dict(rows, fields):
if rows:
return [dict(zip(fields, r)) for r in rows]
+24 -47
View File
@@ -1,4 +1,5 @@
import os
import asyncio
import logging
from io import BytesIO
from typing import Optional, Iterator, Tuple
@@ -34,6 +35,7 @@ class BaseHeaders:
self.io = BytesIO()
self.path = path
self._size: Optional[int] = None
self._header_connect_lock = asyncio.Lock()
async def open(self):
if self.path != ':memory:':
@@ -82,10 +84,6 @@ class BaseHeaders:
def height(self) -> int:
return len(self)-1
@property
def bytes_size(self):
return len(self) * self.header_size
def hash(self, height=None) -> bytes:
return self.hash_header(
self.get_raw_header(height if height is not None else self.height)
@@ -100,25 +98,27 @@ class BaseHeaders:
async def connect(self, start: int, headers: bytes) -> int:
added = 0
bail = False
for height, chunk in self._iterate_chunks(start, headers):
try:
# validate_chunk() is CPU bound and reads previous chunks from file system
self.validate_chunk(height, chunk)
except InvalidHeader as e:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
written = 0
if chunk:
self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(chunk) // self.header_size
self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = None
added += written
if bail:
break
loop = asyncio.get_running_loop()
async with self._header_connect_lock:
for height, chunk in self._iterate_chunks(start, headers):
try:
# validate_chunk() is CPU bound and reads previous chunks from file system
await loop.run_in_executor(None, self.validate_chunk, height, chunk)
except InvalidHeader as e:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
written = 0
if chunk:
self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(chunk) // self.header_size
self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
await loop.run_in_executor(None, self.io.flush)
self._size = None
added += written
if bail:
break
return added
def validate_chunk(self, height, chunk):
@@ -168,35 +168,12 @@ class BaseHeaders:
proof_of_work.value, target.value)
)
async def repair(self):
previous_header_hash = fail = None
self.io.seek(0)
batch_size = 10000
for start_height in range(0, self.height, batch_size):
headers = self.io.read(self.header_size*batch_size)
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
if header['prev_block_hash'] != previous_header_hash:
fail = True
else:
if header_hash != self.genesis_hash:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
return
previous_header_hash = header_hash
@staticmethod
def get_proof_of_work(header_hash: bytes) -> ArithUint256:
return ArithUint256(int(b'0x' + header_hash, 16))
def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]:
assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}"
assert len(headers) % self.header_size == 0
start = 0
end = (self.chunk_size - height % self.chunk_size) * self.header_size
while start < end:
+2 -2
View File
@@ -263,8 +263,8 @@ class BaseLedger(metaclass=LedgerRegistry):
async def join_network(self, *args):
log.info("Subscribing and updating accounts.")
async with self._header_processing_lock:
await self.update_headers()
await self.update_headers()
await self.network.subscribe_headers()
await self.subscribe_accounts()
await self._update_tasks.done.wait()
+13 -27
View File
@@ -8,6 +8,7 @@ import socket
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
from torba import __version__
from torba.rpc.util import protocol_exception_handler
from torba.stream import StreamController
log = logging.getLogger(__name__)
@@ -15,36 +16,28 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, **kwargs):
def __init__(self, *args, network, server, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.max_seconds_idle = 60
self.ping_task = None
async def send_request(self, method, args=()):
try:
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
return await super().send_request(method, args)
except RPCError as e:
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
raise e
except asyncio.TimeoutError:
self.abort()
raise
async def ping_forever(self):
# TODO: change to 'ping' on newer protocol (above 1.2)
while not self.is_closing():
if (time() - self.last_send) > self.max_seconds_idle:
try:
await self.send_request('server.banner')
except:
self.abort()
raise
await self.send_request('server.banner')
await asyncio.sleep(self.max_seconds_idle//3)
async def create_connection(self, timeout=6):
@@ -66,11 +59,11 @@ class ClientSession(BaseClientSession):
class BaseNetwork:
def __init__(self, ledger):
asyncio.get_event_loop().set_exception_handler(protocol_exception_handler)
self.config = ledger.config
self.client: ClientSession = None
self.session_pool: SessionPool = None
self.running = False
self.remote_height: int = 0
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
@@ -91,13 +84,11 @@ class BaseNetwork:
connect_timeout = self.config.get('connect_timeout', 6)
self.session_pool = SessionPool(network=self, timeout=connect_timeout)
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
while True:
try:
self.client = await self.pick_fastest_session()
if self.is_connected:
await self.ensure_server_version()
self._update_remote_height((await self.subscribe_headers(),))
log.info("Successfully connected to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
await self.client.on_disconnected.first
@@ -147,9 +138,6 @@ class BaseNetwork:
await session.send_request('server.banner')
return session
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
def ensure_server_version(self, required='1.2'):
return self.rpc('server.version', [__version__, required])
@@ -227,14 +215,10 @@ class SessionPool:
self._dead_servers = []
async def ensure_connection(self, session):
self._dead_servers.append(session)
self.sessions.remove(session)
if not session.is_closing():
return
try:
if session.is_closing():
await session.create_connection(self.timeout)
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
self.sessions.append(session)
self._dead_servers.remove(session)
return await session.create_connection(self.timeout)
except asyncio.TimeoutError:
log.warning("Timeout connecting to %s:%d", *session.server)
except asyncio.CancelledError: # pylint: disable=try-except-raise
@@ -246,9 +230,11 @@ class SessionPool:
log.warning("Could not connect to %s:%d", *session.server)
else:
log.exception("Connecting to %s:%d raised an exception:", *session.server)
self._dead_servers.append(session)
self.sessions.remove(session)
async def get_online_sessions(self):
self._lost_master.set()
while not self.online:
self._lost_master.set()
await asyncio.sleep(0.5)
await asyncio.sleep(0.1)
return self.sessions
+5 -10
View File
@@ -195,7 +195,6 @@ class SPVNode:
self.server = None
self.hostname = 'localhost'
self.port = 50001 + 1 # avoid conflict with default daemon
self.session_timeout = 600
async def start(self, blockchain_node: 'BlockchainNode'):
self.data_path = tempfile.mkdtemp()
@@ -205,9 +204,7 @@ class SPVNode:
'REORG_LIMIT': '100',
'HOST': self.hostname,
'TCP_PORT': str(self.port),
'SESSION_TIMEOUT': str(self.session_timeout),
'MAX_QUERY_WORKERS': '0',
'INDIVIDUAL_TAG_INDEXES': '',
'MAX_QUERY_WORKERS': '0'
}
# TODO: don't use os.environ
os.environ.update(conf)
@@ -266,7 +263,6 @@ class BlockchainNode:
self.data_path = None
self.protocol = None
self.transport = None
self.blockchain_address = None
self._block_expected = 0
self.hostname = 'localhost'
self.peerport = 9246 + 2 # avoid conflict with default peer port
@@ -330,7 +326,7 @@ class BlockchainNode:
self.daemon_bin,
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
f'-port={self.peerport}', '-addresstype=legacy', '-vbparams=segwit:0:999999999999'
f'-port={self.peerport}'
)
self.log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(
@@ -365,10 +361,9 @@ class BlockchainNode:
self.log.info(out.decode().strip())
return out.decode().strip()
async def generate(self, blocks, address=None):
def generate(self, blocks):
self._block_expected += blocks
address = await self.get_raw_change_address()
return await self._cli_cmnd('generatetoaddress', str(blocks), str(address))
return self._cli_cmnd('generate', str(blocks))
def invalidate_block(self, blockhash):
return self._cli_cmnd('invalidateblock', blockhash)
@@ -377,7 +372,7 @@ class BlockchainNode:
return self._cli_cmnd('getblockhash', str(block))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress', 'legacy')
return self._cli_cmnd('getrawchangeaddress')
async def get_balance(self):
return float(await self._cli_cmnd('getbalance'))
-2
View File
@@ -165,8 +165,6 @@ class JSONRPC(object):
METHOD_NOT_FOUND = -32601
INVALID_ARGS = -32602
INTERNAL_ERROR = -32603
QUERY_TIMEOUT = -32000
# Codes specific to this library
ERROR_CODE_UNAVAILABLE = -100
+10
View File
@@ -93,3 +93,13 @@ class Concurrency(object):
else:
for _ in range(-diff):
await self.semaphore.acquire()
def protocol_exception_handler(loop: asyncio.AbstractEventLoop, context):
message = context['message']
transport = context.get('transport')
if transport and message in ("Fatal read error on socket transport", "Fatal write error on socket transport"):
transport.abort()
transport.close()
else:
return loop.default_exception_handler(context)
-7
View File
@@ -57,11 +57,6 @@ class Daemon:
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
async def close(self):
if self.connector:
await self.connector.close()
self.connector = None
def set_url(self, url):
"""Set the URLS to the given list, and switch to the first one."""
urls = url.split(',')
@@ -98,8 +93,6 @@ class Daemon:
return aiohttp.ClientSession(connector=self.connector, connector_owner=False)
async def _send_data(self, data):
if not self.connector:
raise asyncio.CancelledError('Tried to send request during shutdown.')
async with self.workqueue_semaphore:
async with self.client_session() as session:
async with session.post(self.current_url(), data=data) as resp:
+1 -5
View File
@@ -38,10 +38,6 @@ class Env:
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
self.websocket_host = self.default('WEBSOCKET_HOST', self.host)
self.websocket_port = self.integer('WEBSOCKET_PORT', None)
self.daemon_url = self.required('DAEMON_URL')
if coin is not None:
assert issubclass(coin, Coin)
@@ -51,6 +47,7 @@ class Env:
network = self.default('NET', 'mainnet').strip()
self.coin = Coin.lookup_coin_class(coin_name, network)
self.cache_MB = self.integer('CACHE_MB', 1200)
self.host = self.default('HOST', 'localhost')
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
# Server stuff
self.tcp_port = self.integer('TCP_PORT', None)
@@ -87,7 +84,6 @@ class Env:
self.identities = [identity
for identity in (clearnet_identity, tor_identity)
if identity is not None]
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 250)) / 1000.0
@classmethod
def default(cls, envvar, default):
+2 -1
View File
@@ -4,6 +4,7 @@ import asyncio
from concurrent.futures.thread import ThreadPoolExecutor
import torba
from torba.rpc.util import protocol_exception_handler
from torba.server.mempool import MemPool, MemPoolAPI
@@ -89,6 +90,7 @@ class Server:
)
async def start(self):
asyncio.get_event_loop().set_exception_handler(protocol_exception_handler)
env = self.env
min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
self.log.info(f'software version: {torba.__version__}')
@@ -113,7 +115,6 @@ class Server:
task.cancel()
await asyncio.wait(self.cancellable_tasks)
self.shutdown_event.set()
await self.daemon.close()
def run(self):
loop = asyncio.get_event_loop()
+3 -4
View File
@@ -255,10 +255,9 @@ class SessionManager:
async def _clear_stale_sessions(self):
"""Cut off sessions that haven't done anything for 10 minutes."""
session_timeout = self.env.session_timeout
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.time() - session_timeout
await sleep(60)
stale_cutoff = time.time() - self.env.session_timeout
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions:
@@ -268,7 +267,7 @@ class SessionManager:
# Give the sockets some time to close gracefully
if stale_sessions:
await asyncio.wait([
session.close(force_after=session_timeout // 10) for session in stale_sessions
session.close() for session in stale_sessions
])
# Consolidate small groups