Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c22eed9109 | ||
|
|
9c10ee12fd | ||
|
|
7a622986d6 | ||
|
|
99b3cdd45d | ||
|
|
7b9d84f528 | ||
|
|
5f49827bd2 | ||
|
|
e1b4fe02d6 | ||
|
|
cdab127b98 | ||
|
|
bac7ea5dc0 | ||
|
|
b3b5581f2d | ||
|
|
9a26d869fe | ||
|
|
57138d9629 | ||
|
|
47d897b48d | ||
|
|
6b9f1d519e | ||
|
|
8b966c61d8 | ||
|
|
87e09efff5 | ||
|
|
65a0583c85 | ||
|
|
2000a1b334 | ||
|
|
cdd44af043 | ||
|
|
df99d06d11 | ||
|
|
b32d096be2 | ||
|
|
b9d5d1041c | ||
|
|
994dc9ff33 | ||
|
|
2ac97736ce | ||
|
|
925e106c43 | ||
|
|
59002413f8 | ||
|
|
32293ee3bb | ||
|
|
e04f35af4f | ||
|
|
c50f83400a | ||
|
|
31fa974379 | ||
|
|
aa8c362cb8 | ||
|
|
e48cc40610 | ||
|
|
a931f59dda | ||
|
|
4140fc4d28 | ||
|
|
f0d39e6081 | ||
|
|
c11872a229 | ||
|
|
f4fe730dc9 | ||
|
|
a4c83a08c7 | ||
|
|
c3277d24c3 | ||
|
|
28a23590ef | ||
|
|
162275992b | ||
|
|
08e25f4622 | ||
|
|
9a300d773f | ||
|
|
a5362f3170 | ||
|
|
7b653b90a8 | ||
|
|
f464b3ae7c | ||
|
|
d4b7064d2f | ||
|
|
2540b9969f | ||
|
|
4355b799cb | ||
|
|
a7fb434f9f | ||
|
|
b3465f3bbf | ||
|
|
4dfc3ff94c | ||
|
|
2670279b0d | ||
|
|
f4fd4245be | ||
|
|
200d239e0b | ||
|
|
41a0590182 | ||
|
|
63b2c49354 | ||
|
|
c99d7ce89d | ||
|
|
79e382e064 | ||
|
|
fa1d4dae0f | ||
|
|
35f84b4414 | ||
|
|
a882bdb08c | ||
|
|
b5c2e1c95b | ||
|
|
cfd6ea5a34 | ||
|
|
b3cf5e8698 | ||
|
|
2d07223995 | ||
|
|
32bb9fd4ec | ||
|
|
37b5f22c1a | ||
|
|
e0b00c09ce | ||
|
|
71c4dba34d | ||
|
|
9a3187e36c | ||
|
|
c714a3f91c | ||
|
|
5d4846cf60 | ||
|
|
543b327d09 | ||
|
|
1ccd2a3cc9 | ||
|
|
94398071eb | ||
|
|
143476c76c | ||
|
|
e85683e40f | ||
|
|
141162e5ae | ||
|
|
13acbfb64f | ||
|
|
a55e3d8578 | ||
|
|
2486ae6d3f | ||
|
|
ba991b47ea | ||
|
|
a096d0db37 | ||
|
|
e4a30c052e | ||
|
|
ccfe80ecfa | ||
|
|
df5b889f7d | ||
|
|
55f25c2613 | ||
|
|
34eaa822a0 | ||
|
|
5072f503c4 | ||
|
|
8e6b5a5e55 | ||
|
|
246647641a | ||
|
|
06a1bdbcba | ||
|
|
a0b75d7ada | ||
|
|
d36036a9fe | ||
|
|
a68d891770 | ||
|
|
f272a9ead2 | ||
|
|
8db2e10696 | ||
|
|
bea4f7da98 | ||
|
|
3cd74c06d6 | ||
|
|
053681e4a5 | ||
|
|
e3225f7123 | ||
|
|
ef62b0e746 | ||
|
|
7756d12993 | ||
|
|
3b92d12a44 | ||
|
|
4449bf3c75 | ||
|
|
0200ed1341 | ||
|
|
02bde900d6 | ||
|
|
36e299b4cf | ||
|
|
4e34f861c9 | ||
|
|
42df2fe242 | ||
|
|
7f11da8650 | ||
|
|
dff0fcb1fd | ||
|
|
c31d22f996 | ||
|
|
14ab2b52e9 | ||
|
|
e91f34f09b | ||
|
|
6c733f84e9 | ||
|
|
71c2cd5065 | ||
|
|
45f9a10bfb | ||
|
|
6ad203f767 | ||
|
|
5bf895eefe | ||
|
|
5a8f5b14c1 | ||
|
|
27ccb37646 | ||
|
|
b4c528f6b5 | ||
|
|
517a255d5a | ||
|
|
d76feeb320 | ||
|
|
e4f23b3fce | ||
|
|
02d8dc22e9 | ||
|
|
2fc3ec7224 | ||
|
|
a243ac7a01 | ||
|
|
6f1ee78623 | ||
|
|
6633370a34 | ||
|
|
a22e29532c | ||
|
|
96c8d1f68b | ||
|
|
4aa3bd56c4 | ||
|
|
7d1cf7975a | ||
|
|
59d08a92d0 | ||
|
|
33c1a002da | ||
|
|
8f04d3418a | ||
|
|
f5b556c6cb | ||
|
|
a18f3bf08a | ||
|
|
41a0a0c64c | ||
|
|
b4c8a317d0 | ||
|
|
30ac075555 | ||
|
|
fcb317ca40 | ||
|
|
9991f79326 | ||
|
|
7cf3f8c7cd | ||
|
|
eeef183cc6 | ||
|
|
96bc02c8cb | ||
|
|
af276a24ed | ||
|
|
a92dbae459 | ||
|
|
b0bfb5e443 | ||
|
|
671e6da104 | ||
|
|
485be533c9 | ||
|
|
53ff4cd651 | ||
|
|
a98c760d93 | ||
|
|
9b0c95e37d | ||
|
|
b419b8f642 | ||
|
|
c32f8d0998 | ||
|
|
c6d253cc25 | ||
|
|
c05cb62cdb | ||
|
|
ddd213dda3 | ||
|
|
8aab844a48 | ||
|
|
8fc1cbfc54 | ||
|
|
82ea330dda | ||
|
|
c63aa3033a | ||
|
|
7837738c17 | ||
|
|
bdb0ad4836 | ||
|
|
0dbf810cfe | ||
|
|
58aa4887ad | ||
|
|
3ff0916f49 | ||
|
|
cc9fb3ff41 | ||
|
|
2c51928b9d | ||
|
|
4451978af6 | ||
|
|
d4193c55ae | ||
|
|
ec3e365b3c | ||
|
|
c245c69389 | ||
|
|
9a9a7c9d7a | ||
|
|
20b516b5fa | ||
|
|
4dcfeca092 | ||
|
|
8ca8c73868 | ||
|
|
1a1c101b9a | ||
|
|
a0a0e2d324 | ||
|
|
d544199eff | ||
|
|
407ebeb699 | ||
|
|
9ba12b3e3f | ||
|
|
97976cbee9 | ||
|
|
cb5c772233 | ||
|
|
2bfbb824bf | ||
|
|
05189651ee | ||
|
|
0c80134523 | ||
|
|
6c686520c4 | ||
|
|
49ae029572 | ||
|
|
10e43203ed | ||
|
|
e25be5a522 | ||
|
|
47ff64a9e9 | ||
|
|
35f3887d79 | ||
|
|
dacfc8b00a | ||
|
|
0840ac7d06 | ||
|
|
f34b96f565 | ||
|
|
808fbed4f3 | ||
|
|
b48800049b | ||
|
|
7e5d4166d8 | ||
|
|
6b724ba8db | ||
|
|
25e1427a84 | ||
|
|
82944bc103 | ||
|
|
1e73b2437d | ||
|
|
574cd304e3 | ||
|
|
2edcd1f5b2 | ||
|
|
e144b26619 | ||
|
|
9ef9220689 | ||
|
|
23b6a8c495 | ||
|
|
4b9552fdf5 | ||
|
|
a249097170 | ||
|
|
6f4b2d7967 | ||
|
|
888d77c96c | ||
|
|
e2ae1ca866 | ||
|
|
7ef9a0f0bf | ||
|
|
885a0882f2 | ||
|
|
2ddeca2976 | ||
|
|
2085012874 | ||
|
|
ebb495ab8b | ||
|
|
905274d6d4 | ||
|
|
120d7edf12 | ||
|
|
12cd06cae5 | ||
|
|
42c26102d8 | ||
|
|
e906564f70 | ||
|
|
b8b2dc3b6d | ||
|
|
d69f76e68a | ||
|
|
ec1ba02cca | ||
|
|
49869d071a | ||
|
|
39275682d9 | ||
|
|
1dbc3a049b | ||
|
|
0a7b2683ff | ||
|
|
9714985855 | ||
|
|
d809b8eee0 | ||
|
|
52e2ed2aa7 | ||
|
|
ab3cc3d2b6 | ||
|
|
0a33324321 | ||
|
|
a948335cb9 | ||
|
|
6d2e5af6bc | ||
|
|
9a7c71394e | ||
|
|
1ebf184e9f | ||
|
|
48b013fa85 | ||
|
|
e96dee478d | ||
|
|
25d2a12a92 | ||
|
|
b0af31a71c | ||
|
|
9fdd766429 | ||
|
|
28ce23350d | ||
|
|
1abe890067 | ||
|
|
75fbedeeac | ||
|
|
a1861a2f84 | ||
|
|
0eecfb52ed | ||
|
|
ce1c28ffb9 | ||
|
|
9667b81c8d | ||
|
|
afac596a7b | ||
|
|
636692ba6a | ||
|
|
203f7e7ea6 | ||
|
|
8aee19b0ef | ||
|
|
49ed31c895 | ||
|
|
526075cd5e | ||
|
|
71340805ec | ||
|
|
654585aa68 | ||
|
|
441d3c1220 | ||
|
|
fc2d7258fb | ||
|
|
8195cedcf4 | ||
|
|
3e98477707 | ||
|
|
7b9479eca9 | ||
|
|
f15b67d386 | ||
|
|
f3141727d7 | ||
|
|
c7e4638736 | ||
|
|
d04981969f | ||
|
|
a148d4f446 | ||
|
|
a9e105f9eb | ||
|
|
c8afce2a3a | ||
|
|
ebf9d11f4d | ||
|
|
7a3bdf93e9 | ||
|
|
35d5c585ec | ||
|
|
45eb4f7201 | ||
|
|
e4c2585e23 | ||
|
|
a05f2fe3fb | ||
|
|
94cdd6829c | ||
|
|
34aa51e98a | ||
|
|
6e93c68a77 | ||
|
|
f16eec2ad2 | ||
|
|
657c1a4626 | ||
|
|
6bf6e943cc | ||
|
|
6252f903ee | ||
|
|
a1e827fb3b | ||
|
|
40218e02e1 | ||
|
|
0891b0d294 | ||
|
|
756c43c087 | ||
|
|
6488e66088 | ||
|
|
31ee3fb12b | ||
|
|
b80d45ad45 | ||
|
|
ea2a3c3b70 | ||
|
|
c0312159eb | ||
|
|
e124f582aa | ||
|
|
54150a8f2a | ||
|
|
385c221ef8 | ||
|
|
7df6e99e28 | ||
|
|
76a5f2f516 | ||
|
|
936c1f2ada | ||
|
|
5787802077 | ||
|
|
c902d25524 | ||
|
|
d152336f42 | ||
|
|
7ab0ae18be | ||
|
|
6189eddecf | ||
|
|
cc7471a9af | ||
|
|
eabaca361c | ||
|
|
141ca2498c | ||
|
|
964db2a779 | ||
|
|
f2867542b1 | ||
|
|
8f6e886821 | ||
|
|
dfca7bc2c4 | ||
|
|
f94b04bb19 | ||
|
|
be1e97c616 | ||
|
|
9790e9f843 | ||
|
|
e339d3857f | ||
|
|
0e4f21842c | ||
|
|
d4ea9a1ba3 | ||
|
|
5300fa929b | ||
|
|
8a5a66a06a | ||
|
|
43ed4427b4 | ||
|
|
22b0d92e6b | ||
|
|
7eb528232e | ||
|
|
33b6b85e72 | ||
|
|
8177119671 | ||
|
|
4b407a9dd2 | ||
|
|
9b904a7b4a | ||
|
|
e0424a1e38 | ||
|
|
00c08e05f5 | ||
|
|
0836d9c5ee | ||
|
|
ff5b6c4eab | ||
|
|
bb5cd49327 | ||
|
|
6f40ef6015 | ||
|
|
c7758506ac | ||
|
|
192ac6959a | ||
|
|
465a4643a8 | ||
|
|
1bf0065d9d | ||
|
|
c14efc843c | ||
|
|
0506dfba10 | ||
|
|
7e4f36e8b4 | ||
|
|
55dc482a61 | ||
|
|
05b9f9cae6 | ||
|
|
c79868de3e | ||
|
|
7eeac060ab | ||
|
|
c8b2b7b279 | ||
|
|
f4b3187494 |
No files matched your search
@@ -1 +1,25 @@
|
||||
*.pyc
|
||||
|
||||
*.xml
|
||||
|
||||
.idea/.name
|
||||
|
||||
*.iml
|
||||
|
||||
*.log
|
||||
|
||||
*.txt
|
||||
|
||||
lbrynet.egg-info/PKG-INFO
|
||||
|
||||
*.egg
|
||||
|
||||
/build
|
||||
|
||||
/dist
|
||||
|
||||
*.so
|
||||
|
||||
*.pem
|
||||
|
||||
*.decTest
|
||||
@@ -1,34 +0,0 @@
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
|
||||
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
|
||||
this is not desired, the testing client can be used to simulate trading points, which is
|
||||
built into LBRYnet.
|
||||
|
||||
on Ubuntu:
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
|
||||
Getting the source
|
||||
------------------
|
||||
|
||||
Don't you already have it?
|
||||
|
||||
Setting up the environment
|
||||
--------------------------
|
||||
|
||||
It's recommended that you use a virtualenv
|
||||
|
||||
sudo apt-get install python-virtualenv
|
||||
cd <source base directory>
|
||||
virtualenv .
|
||||
source bin/activate
|
||||
|
||||
(to deactivate the virtualenv, enter 'deactivate')
|
||||
|
||||
python setup.py install
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see RUNNING
|
||||
@@ -0,0 +1,51 @@
|
||||
#### Installing lbrynet on Linux
|
||||
--------------------------
|
||||
|
||||
The following packages are necessary (the following are their names on Ubuntu):
|
||||
libgmp3-dev build-essential python2.7 python2.7-dev python-pip
|
||||
|
||||
To install them on Ubuntu:
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip
|
||||
|
||||
python setup.py build bdist_egg
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see [RUNNING](RUNNING.md)
|
||||
|
||||
#### On windows:
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
Install the each of the preceding binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Install the rest of the required modules with the standard `pip install module` command
|
||||
|
||||
The one module which requires additional work is [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9).
|
||||
Download the source and compile with MinGW by running `mingw32make.bat`
|
||||
Then install the module by running `python setupmingw32.py install`
|
||||
@@ -0,0 +1,2 @@
|
||||
include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin
|
||||
recursive-include lbrynet/lbrynet_gui *.gif *.ico *.xbm *.conf
|
||||
@@ -1,12 +1,10 @@
|
||||
LBRYnet
|
||||
=======
|
||||
# LBRYnet
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
Overview
|
||||
--------
|
||||
## Overview
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
@@ -20,8 +18,7 @@ help peers find each other. For example, an application for which clients don't
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
Running
|
||||
-------
|
||||
## Running
|
||||
|
||||
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
@@ -31,27 +28,23 @@ must first obtain the 'stream descriptor' and then may open it with his 'lbrynet
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
To install and use this client, see INSTALL and RUNNING
|
||||
To install and use this client, see [INSTALL](INSTALL.md) and [RUNNING](RUNNING.md)
|
||||
|
||||
Installation
|
||||
------------
|
||||
## Installation
|
||||
|
||||
See INSTALL
|
||||
See [INSTALL](INSTALL.md)
|
||||
|
||||
Developers
|
||||
----------
|
||||
## Developers
|
||||
|
||||
Documentation: doc.lbry.io
|
||||
Source code: trac.lbry.io/browser
|
||||
Source code: https://github.com/lbryio/lbry
|
||||
|
||||
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
|
||||
|
||||
Support
|
||||
-------
|
||||
## Support
|
||||
|
||||
Send all support requests to jimmy@lbry.io
|
||||
|
||||
License
|
||||
-------
|
||||
## License
|
||||
|
||||
See LICENSE
|
||||
See [LICENSE](LICENSE)
|
||||
@@ -1,52 +0,0 @@
|
||||
To install LBRYnet and lbrynet-console, see INSTALL
|
||||
|
||||
lbrynet-console is a console application which makes use of the LBRYnet to share files.
|
||||
|
||||
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
|
||||
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
|
||||
others wishing to obtain the file, or can be itself turned into a chunk compatible with
|
||||
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
|
||||
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
|
||||
specified by the hash sums found in the stream descriptor file, decrypts them according to
|
||||
metadata found in the stream, and reconstructs the original file. lbrynet-console features
|
||||
a server so that clients can connect to it and download the chunks and other data gotten
|
||||
from files created locally and files that have been downloaded from LBRYnet.
|
||||
|
||||
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
|
||||
concept which is currently far behind the development of the rest of the application and
|
||||
therefore will not run, and a plugin which attempts to determine which chunks on the
|
||||
network should be downloaded in order for the application to turn a profit. It will run,
|
||||
but its usefulness is extremely limited.
|
||||
|
||||
Passing '--help' to lbrynet-console will cause it to print out a quick help message
|
||||
describing other command line options to the application.
|
||||
|
||||
Once the application has been started, the user is presented with a numbered list of
|
||||
actions which looks something like this:
|
||||
|
||||
...
|
||||
[2] Toggle whether an LBRY File is running
|
||||
[3] Create an LBRY File from file
|
||||
[4] Publish a stream descriptor file to the DHT for an LBRY File
|
||||
...
|
||||
|
||||
To perform an action, type the desired number and then hit enter. For example, if you wish
|
||||
to create an LBRY file from a file as described in the beginning of this document, type 3 and
|
||||
hit enter.
|
||||
|
||||
If the application needs more input in order to for the action to be taken, the application
|
||||
will continue to print prompts for input until it has received what it needs.
|
||||
|
||||
For example, when creating an LBRY file from a file, the application needs to know which file
|
||||
it's supposed to use to create the LBRY file, so the user will be prompted for it:
|
||||
|
||||
File name:
|
||||
|
||||
The user should input the desired file name and hit enter, at which point the application
|
||||
will go about splitting the file and making it available on the network.
|
||||
|
||||
Some actions will produce sub-menus of actions, which work the same way.
|
||||
|
||||
A more detailed user guide is available at doc.lbry.io
|
||||
|
||||
Any issues may be reported to jimmy@lbry.io
|
||||
@@ -0,0 +1,146 @@
|
||||
How to watch It's a Wonderful Life via LBRY
|
||||
|
||||
## Quickest quick guide
|
||||
|
||||
Create a directory called lbry, and go into that directory
|
||||
|
||||
Download the file https://raw.githubusercontent.com/lbryio/lbry-setup/master/lbry_setup.sh and run it in that directory
|
||||
|
||||
Once it's done building, type:
|
||||
|
||||
```
|
||||
lbrynet-console
|
||||
```
|
||||
|
||||
A console application will load, and after a moment you will be presented with a ">" signifying
|
||||
that the console is ready for commands.
|
||||
|
||||
If it's your first time running lbrynet-console, you should be given some credits to test the
|
||||
network. If they don't show up within a minute or two, let us know, and we'll send you some.
|
||||
|
||||
After your credits have shown up, type
|
||||
|
||||
```
|
||||
get wonderfullife
|
||||
```
|
||||
|
||||
into the prompt and hit enter.
|
||||
|
||||
You will be asked if you want to cancel, change options, save, and perhaps stream the file.
|
||||
|
||||
Enter 's' for save and then hit enter.
|
||||
|
||||
The file should start downloading. Enter the command 'status' to check on the status of files
|
||||
that are in the process of being downloaded.
|
||||
|
||||
To stop lbrynet-console, enter the command 'exit'.
|
||||
|
||||
|
||||
## Slightly longer install guide
|
||||
|
||||
### Installing lbrycrd, the full blockchain client
|
||||
|
||||
Note: this process takes upwards of an hour and is not necessary to use lbrynet.
|
||||
|
||||
```
|
||||
git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler
|
||||
./autogen.sh
|
||||
./configure --with-incompatible-bdb --without-gui
|
||||
|
||||
make
|
||||
```
|
||||
|
||||
When make has completed, create the directory where LBRYcrd data will be stored. ~/.lbrycrd is where LBRYcrd will look by default and so is recommended.
|
||||
|
||||
```
|
||||
mkdir ~/.lbrycrd
|
||||
echo 'rpcuser=rpcuser
|
||||
rpcpassword=rpcpassword' > ~/.lbrycrd/lbrycrd.conf
|
||||
# (use a long random password if your computer is on a network anyone else has access to. e.g, pwgen -s 20)
|
||||
cd ..
|
||||
|
||||
```
|
||||
|
||||
### Installing lbrynet from source
|
||||
|
||||
Acquire the LBRYnet source code from https://github.com/lbryio/lbry
|
||||
|
||||
```
|
||||
cd lbry
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
```
|
||||
|
||||
(with virtualenv)
|
||||
|
||||
```
|
||||
python-virtualenv
|
||||
|
||||
virtualenv .
|
||||
|
||||
source bin/activate
|
||||
|
||||
python setup.py install
|
||||
|
||||
```
|
||||
|
||||
to deactivate the virtualenv later:
|
||||
|
||||
```
|
||||
deactivate
|
||||
```
|
||||
|
||||
to reactivate it, go to the directory in which you created it and:
|
||||
|
||||
```
|
||||
source bin/activate
|
||||
```
|
||||
|
||||
(without virtualenv)
|
||||
|
||||
```
|
||||
python setup.py build bdist_egg
|
||||
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
## Slightly longer running guide
|
||||
|
||||
### lbrynet-console can be set to use lbrycrdd instead of the built in lightweight client.
|
||||
|
||||
To run lbrynet-console with lbrycrdd:
|
||||
|
||||
```
|
||||
lbrynet-console </path/to/lbrycrdd>
|
||||
```
|
||||
|
||||
If lbrycrdd is not already running, lbrynet will launch it at that path, and will shut it down
|
||||
when lbrynet exits. If lbrycrdd is already running, lbrynet will not launch it and will not
|
||||
shut it down, but it will connect to it and use it.
|
||||
|
||||
### Running lbrycrdd manually
|
||||
|
||||
From the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon
|
||||
```
|
||||
|
||||
If you want to mine LBC, also use the flag '-gen', so:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon -gen
|
||||
```
|
||||
|
||||
Warning: This will put a heavy load on your CPU
|
||||
|
||||
It will take a few minutes for your client to download the whole block chain.
|
||||
|
||||
To shut lbrycrdd down: from the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrd-cli stop
|
||||
```
|
||||
|
||||
Any questions or problems, email jimmy@lbry.io
|
||||
@@ -0,0 +1,8 @@
|
||||
import logging
|
||||
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
version = (0, 2, 2)
|
||||
__version__ = ".".join([str(x) for x in version])
|
||||
@@ -13,13 +13,31 @@ MAX_RESPONSE_INFO_SIZE = 2**16
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
BLOBFILES_DIR = ".blobfiles"
|
||||
BLOB_SIZE = 2**21
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos
|
||||
MAX_CONNECTIONS_PER_STREAM = 5
|
||||
DEFAULT_MAX_SEARCH_RESULTS = 25
|
||||
DEFAULT_MAX_KEY_FEE = 100.0
|
||||
|
||||
KNOWN_DHT_NODES = [('104.236.42.182', 4000)]
|
||||
|
||||
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
|
||||
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
|
||||
API_INTERFACE = "localhost"
|
||||
API_ADDRESS = "lbryapi"
|
||||
API_PORT = 5279
|
||||
ICON_PATH = "app.icns"
|
||||
APP_NAME = "LBRY"
|
||||
DEFAULT_WALLET = "lbryum"
|
||||
|
||||
API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
|
||||
UI_ADDRESS = "http://" + API_INTERFACE + ":" + str(API_PORT)
|
||||
|
||||
PROTOCOL_PREFIX = "lbry"
|
||||
|
||||
DEFAULT_TIMEOUT = 30
|
||||
@@ -1,14 +1,19 @@
|
||||
import logging
|
||||
import os
|
||||
import leveldb
|
||||
import time
|
||||
import json
|
||||
from twisted.internet import threads, defer, reactor, task
|
||||
import sqlite3
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.Error import NoSuchBlobError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager(DHTHashSupplier):
|
||||
@@ -68,8 +73,8 @@ class DiskBlobManager(BlobManager):
|
||||
def __init__(self, hash_announcer, blob_dir, db_dir):
|
||||
BlobManager.__init__(self, hash_announcer)
|
||||
self.blob_dir = blob_dir
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
self.db_file = os.path.join(db_dir, "blobs.db")
|
||||
self.db_conn = None
|
||||
self.blob_type = BlobFile
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
self.blobs = {}
|
||||
@@ -77,15 +82,19 @@ class DiskBlobManager(BlobManager):
|
||||
self._next_manage_call = None
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
|
||||
str(self.db_file))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping the DiskBlobManager")
|
||||
if self._next_manage_call is not None and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
self.db = None
|
||||
#d = self.db_conn.close()
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
@@ -101,7 +110,7 @@ class DiskBlobManager(BlobManager):
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
d = threads.deferToThread(self._completed_blobs, [blob_hash])
|
||||
d = self._completed_blobs([blob_hash])
|
||||
|
||||
def check_completed(completed_blobs):
|
||||
|
||||
@@ -110,7 +119,7 @@ class DiskBlobManager(BlobManager):
|
||||
|
||||
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
|
||||
blob.verified = True
|
||||
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
inner_d = self._get_blob_length(blob_hash)
|
||||
inner_d.addCallback(set_length)
|
||||
inner_d.addCallback(lambda _: blob)
|
||||
else:
|
||||
@@ -123,18 +132,18 @@ class DiskBlobManager(BlobManager):
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
if next_announce_time is None:
|
||||
next_announce_time = time.time()
|
||||
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
return self._add_completed_blob(blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
return threads.deferToThread(self._completed_blobs, blobs_to_check)
|
||||
return self._completed_blobs(blobs_to_check)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
|
||||
return self._get_blobs_to_announce(next_announce_time)
|
||||
|
||||
def creator_finished(self, blob_creator):
|
||||
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
@@ -155,18 +164,18 @@ class DiskBlobManager(BlobManager):
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def update_all_last_verified_dates(self, timestamp):
|
||||
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
|
||||
return self._update_all_last_verified_dates(timestamp)
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
d = threads.deferToThread(self._get_all_verified_blob_hashes)
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.hash_announcer.immediate_announce)
|
||||
return d
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
return threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
return self._get_blob_length(blob_hash)
|
||||
|
||||
def check_consistency(self):
|
||||
return threads.deferToThread(self._check_consistency)
|
||||
return self._check_consistency()
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -185,19 +194,19 @@ class DiskBlobManager(BlobManager):
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return err
|
||||
|
||||
def delete_from_db(result):
|
||||
b_hs = [r[1] for r in result if r[0] is True]
|
||||
if b_hs:
|
||||
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
|
||||
d = self._delete_blobs_from_db(b_hs)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
log.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
@@ -221,72 +230,127 @@ class DiskBlobManager(BlobManager):
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
|
||||
return self.db_conn.runQuery("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real"
|
||||
")")
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
|
||||
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
if next_announce_time is None:
|
||||
next_announce_time = timestamp
|
||||
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into blobs values (?, ?, ?, ?)",
|
||||
(blob_hash, length, timestamp, next_announce_time))
|
||||
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _completed_blobs(self, blobs_to_check):
|
||||
blobs = []
|
||||
for b in blobs_to_check:
|
||||
if is_valid_blobhash(b):
|
||||
try:
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
|
||||
except KeyError:
|
||||
continue
|
||||
blobs_to_check = filter(is_valid_blobhash, blobs_to_check)
|
||||
|
||||
def get_blobs_in_db(db_transaction):
|
||||
blobs_in_db = [] # [(blob_hash, last_verified_time)]
|
||||
for b in blobs_to_check:
|
||||
result = db_transaction.execute("select last_verified_time from blobs where blob_hash = ?",
|
||||
(b,))
|
||||
row = result.fetchone()
|
||||
if row is not None:
|
||||
blobs_in_db.append((b, row[0]))
|
||||
return blobs_in_db
|
||||
|
||||
def get_valid_blobs(blobs_in_db):
|
||||
|
||||
def check_blob_verified_date(b, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, b)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blobs.append(b)
|
||||
return blobs
|
||||
return True
|
||||
return False
|
||||
|
||||
def return_valid_blobs(results):
|
||||
valid_blobs = []
|
||||
for (b, verified_date), (success, result) in zip(blobs_in_db, results):
|
||||
if success is True and result is True:
|
||||
valid_blobs.append(b)
|
||||
return valid_blobs
|
||||
|
||||
ds = []
|
||||
for b, verified_date in blobs_in_db:
|
||||
ds.append(threads.deferToThread(check_blob_verified_date, b, verified_date))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(return_valid_blobs)
|
||||
return dl
|
||||
|
||||
d = self.db_conn.runInteraction(get_blobs_in_db)
|
||||
d.addCallback(get_valid_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_length(self, blob):
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
return length
|
||||
d = self.db_conn.runQuery("select blob_length from blobs where blob_hash = ?", (blob,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else Failure(NoSuchBlobError(blob)))
|
||||
return d
|
||||
|
||||
#length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
#return length
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_blob_verified_timestamp(self, blob, timestamp):
|
||||
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(blob, timestamp))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blobs_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
blobs = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
|
||||
blobs.append(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
return blobs
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select blob_hash from blobs " +
|
||||
"where next_announce_time < ? and blob_hash is not null",
|
||||
(timestamp,))
|
||||
blobs = [b for b, in r.fetchall()]
|
||||
transaction.execute("update blobs set next_announce_time = ? where next_announce_time < ?",
|
||||
(next_announce_time, timestamp))
|
||||
return blobs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_all_last_verified_dates(self, timestamp):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_date = ?", (timestamp,))
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_blobs_from_db(self, blob_hashes):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash in blob_hashes:
|
||||
batch.Delete(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
|
||||
def delete_blobs(transaction):
|
||||
for b in blob_hashes:
|
||||
transaction.execute("delete from blobs where blob_hash = ?", (b,))
|
||||
|
||||
return self.db_conn.runInteraction(delete_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_consistency(self):
|
||||
batch = leveldb.WriteBatch()
|
||||
|
||||
ALREADY_VERIFIED = 1
|
||||
NEWLY_VERIFIED = 2
|
||||
INVALID = 3
|
||||
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
d = self.db_conn.runQuery("select blob_hash, blob_length, last_verified_time from blobs")
|
||||
|
||||
def check_blob(blob_hash, blob_length, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time < os.path.getctime(file_path):
|
||||
if verified_time >= os.path.getctime(file_path):
|
||||
return ALREADY_VERIFIED
|
||||
else:
|
||||
h = get_lbry_hash_obj()
|
||||
len_so_far = 0
|
||||
f = open(file_path)
|
||||
@@ -296,19 +360,63 @@ class DiskBlobManager(BlobManager):
|
||||
break
|
||||
h.update(data)
|
||||
len_so_far += len(data)
|
||||
if len_so_far == length and h.hexdigest() == blob_hash:
|
||||
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
if len_so_far == blob_length and h.hexdigest() == blob_hash:
|
||||
return NEWLY_VERIFIED
|
||||
return INVALID
|
||||
|
||||
def do_check(blobs):
|
||||
already_verified = []
|
||||
newly_verified = []
|
||||
invalid = []
|
||||
for blob_hash, blob_length, verified_time in blobs:
|
||||
status = check_blob(blob_hash, blob_length, verified_time)
|
||||
if status == ALREADY_VERIFIED:
|
||||
already_verified.append(blob_hash)
|
||||
elif status == NEWLY_VERIFIED:
|
||||
newly_verified.append(blob_hash)
|
||||
else:
|
||||
invalid.append(blob_hash)
|
||||
return already_verified, newly_verified, invalid
|
||||
|
||||
def update_newly_verified(transaction, blobs):
|
||||
for b in blobs:
|
||||
transaction.execute("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(current_time, b))
|
||||
|
||||
def check_blobs(blobs):
|
||||
|
||||
@rerun_if_locked
|
||||
def update_and_return(status_lists):
|
||||
|
||||
already_verified, newly_verified, invalid = status_lists
|
||||
|
||||
d = self.db_conn.runInteraction(update_newly_verified, newly_verified)
|
||||
d.addCallback(lambda _: status_lists)
|
||||
return d
|
||||
|
||||
d = threads.deferToThread(do_check, blobs)
|
||||
|
||||
d.addCallback(update_and_return)
|
||||
return d
|
||||
|
||||
d.addCallback(check_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
blob_hashes = []
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blob_hashes.append(blob_hash)
|
||||
return blob_hashes
|
||||
d = self.db_conn.runQuery("select blob_hash, last_verified_time from blobs")
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash, verified_time in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
|
||||
|
||||
class TempBlobManager(BlobManager):
|
||||
@@ -389,7 +497,7 @@ class TempBlobManager(BlobManager):
|
||||
if blob_hash in self.blobs:
|
||||
if self.blobs[blob_hash].length is not None:
|
||||
return defer.succeed(self.blobs[blob_hash].length)
|
||||
return defer.fail(ValueError("No such blob hash is known"))
|
||||
return defer.fail(NoSuchBlobError(blob_hash))
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
@@ -400,7 +508,7 @@ class TempBlobManager(BlobManager):
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
logging.info("Setting the next manage call in %s", str(self))
|
||||
log.info("Setting the next manage call in %s", str(self))
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
@@ -409,11 +517,11 @@ class TempBlobManager(BlobManager):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
logging.info("Deleted blob %s", blob_hash)
|
||||
log.info("Deleted blob %s", blob_hash)
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return b_h
|
||||
|
||||
@@ -422,7 +530,7 @@ class TempBlobManager(BlobManager):
|
||||
if being_deleted is False:
|
||||
if blob_hash in self.blobs:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
logging.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
log.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
blob = self.blobs[blob_hash]
|
||||
d = blob.delete()
|
||||
|
||||
@@ -432,7 +540,7 @@ class TempBlobManager(BlobManager):
|
||||
ds.append(d)
|
||||
else:
|
||||
remove_from_list(blob_hash)
|
||||
d = defer.fail(Failure(ValueError("No such blob known")))
|
||||
logging.warning("Blob %s cannot be deleted because it is unknown")
|
||||
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
|
||||
log.warning("Blob %s cannot be deleted because it is unknown")
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds)
|
||||
@@ -1,6 +1,21 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
def __init__(self, option_types, long_description, short_description, default):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default = default
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
@@ -26,11 +26,29 @@ class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.name)
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.name)
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
pass
|
||||
@@ -45,4 +63,16 @@ class NoResponseError(MisbehavingPeerError):
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
@@ -13,6 +13,9 @@ from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
implements(interfaces.IConsumer)
|
||||
|
||||
@@ -56,6 +59,9 @@ class HashBlobWriter(object):
|
||||
" %s to %s" % (str(self.len_so_far),
|
||||
str(self.length_getter())))))
|
||||
else:
|
||||
if self.write_handle is None:
|
||||
log.debug("Tried to write to a write_handle that was None.")
|
||||
return
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb(self)
|
||||
@@ -84,7 +90,7 @@ class HashBlob(object):
|
||||
if self.length is None and 0 <= length <= BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
return False
|
||||
|
||||
def get_length(self):
|
||||
@@ -126,8 +132,8 @@ class HashBlob(object):
|
||||
finished_deferred.callback(self)
|
||||
del self.writers[p]
|
||||
return True
|
||||
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
log.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
@@ -199,14 +205,14 @@ class BlobFile(HashBlob):
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
if not peer in self.writers:
|
||||
logging.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
|
||||
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return finished_deferred, writer.write, writer.cancel
|
||||
logging.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
@@ -232,7 +238,7 @@ class BlobFile(HashBlob):
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
log.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
@@ -247,7 +253,7 @@ class BlobFile(HashBlob):
|
||||
|
||||
def _close_writer(self, writer):
|
||||
if writer.write_handle is not None:
|
||||
logging.debug("Closing %s", str(self))
|
||||
log.debug("Closing %s", str(self))
|
||||
name = writer.write_handle.name
|
||||
writer.write_handle.close()
|
||||
threads.deferToThread(os.remove, name)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import leveldb
|
||||
import os
|
||||
import unqlite
|
||||
import time
|
||||
from Crypto.Hash import SHA512
|
||||
from Crypto.PublicKey import RSA
|
||||
@@ -15,6 +15,9 @@ from twisted.python.failure import Failure
|
||||
from lbrynet.core.LBRYcrdWallet import ReservedPoints
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PTCWallet(object):
|
||||
"""This class sends payments to peers and also ensures that expected payments are received.
|
||||
This class is only intended to be used for testing."""
|
||||
@@ -71,7 +74,7 @@ class PTCWallet(object):
|
||||
|
||||
def save_key(success, private_key):
|
||||
if success is True:
|
||||
threads.deferToThread(self.save_private_key, private_key.exportKey())
|
||||
self._save_private_key(private_key.exportKey())
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -95,8 +98,8 @@ class PTCWallet(object):
|
||||
def start_manage():
|
||||
self.manage()
|
||||
return True
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._get_wallet_private_key())
|
||||
d.addCallback(ensure_private_key_exists)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
@@ -160,8 +163,8 @@ class PTCWallet(object):
|
||||
ds.append(d)
|
||||
del self.queued_payments[peer]
|
||||
else:
|
||||
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def get_balance(self):
|
||||
@@ -207,20 +210,25 @@ class PTCWallet(object):
|
||||
if self.peer_pub_keys[peer] in self.received_payments:
|
||||
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
if min_expected_balance > received_balance:
|
||||
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
log.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
def open_db():
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
return threads.deferToThread(open_db)
|
||||
|
||||
def save_private_key(self, private_key):
|
||||
self.db.Put("private_key", private_key)
|
||||
def _save_private_key(self, private_key):
|
||||
def save_key():
|
||||
self.db['private_key'] = private_key
|
||||
return threads.deferToThread(save_key)
|
||||
|
||||
def get_wallet_private_key(self):
|
||||
try:
|
||||
return self.db.Get("private_key")
|
||||
except KeyError:
|
||||
def _get_wallet_private_key(self):
|
||||
def get_key():
|
||||
if 'private_key' in self.db:
|
||||
return self.db['private_key']
|
||||
return None
|
||||
return threads.deferToThread(get_key)
|
||||
|
||||
|
||||
class PointTraderKeyExchanger(object):
|
||||
@@ -256,9 +264,9 @@ class PointTraderKeyExchanger(object):
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
return err
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandlerFactory(object):
|
||||
@@ -301,15 +309,15 @@ class PointTraderKeyQueryHandler(object):
|
||||
try:
|
||||
RSA.importKey(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
logging.warning("Client sent an invalid public key.")
|
||||
log.warning("Client sent an invalid public key.")
|
||||
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
logging.debug("Received the client's public key: %s", str(self.public_key))
|
||||
log.debug("Received the client's public key: %s", str(self.public_key))
|
||||
fields = {'public_key': self.wallet.encoded_public_key}
|
||||
return defer.succeed(fields)
|
||||
if self.public_key is None:
|
||||
logging.warning("Expected a public key, but did not receive one")
|
||||
log.warning("Expected a public key, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
@@ -1,5 +1,6 @@
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
@@ -10,22 +11,20 @@ class DummyRateLimiter(object):
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.ul_delay = 0.00
|
||||
self.dl_delay = 0.00
|
||||
self.next_tick = None
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
@@ -33,12 +32,6 @@ class DummyRateLimiter(object):
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def ul_wait_time(self):
|
||||
return self.ul_delay
|
||||
|
||||
def dl_wait_time(self):
|
||||
return self.dl_delay
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
@@ -58,66 +51,32 @@ class RateLimiter(object):
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.next_tick = None
|
||||
self.next_unthrottle_dl = None
|
||||
self.next_unthrottle_ul = None
|
||||
|
||||
self.next_dl_check = None
|
||||
self.next_ul_check = None
|
||||
|
||||
self.dl_check_interval = 1.0
|
||||
self.ul_check_interval = 1.0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# happens once per second
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.max_dl_bytes is not None:
|
||||
if self.dl_bytes_this_second == 0:
|
||||
self.dl_check_interval = 1.0
|
||||
else:
|
||||
self.dl_check_interval = min(1.0, self.dl_check_interval *
|
||||
self.max_dl_bytes / self.dl_bytes_this_second)
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
if self.max_ul_bytes is not None:
|
||||
if self.ul_bytes_this_second == 0:
|
||||
self.ul_check_interval = 1.0
|
||||
else:
|
||||
self.ul_check_interval = min(1.0, self.ul_check_interval *
|
||||
self.max_ul_bytes / self.ul_bytes_this_second)
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
@@ -129,27 +88,15 @@ class RateLimiter(object):
|
||||
|
||||
def check_dl(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_dl_check = None
|
||||
|
||||
if self.dl_bytes_this_second > self.max_dl_bytes:
|
||||
self.throttle_dl()
|
||||
else:
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
|
||||
if self.max_dl_bytes is not None and self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_ul_check = None
|
||||
|
||||
if self.ul_bytes_this_second > self.max_ul_bytes:
|
||||
self.throttle_ul()
|
||||
else:
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
|
||||
if self.max_ul_bytes is not None and self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
@@ -175,23 +122,17 @@ class RateLimiter(object):
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#deprecated
|
||||
|
||||
def ul_wait_time(self):
|
||||
return 0
|
||||
|
||||
def dl_wait_time(self):
|
||||
return 0
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
|
||||
@@ -13,6 +13,9 @@ from lbrynet.core.PaymentRateManager import BasePaymentRateManager
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYSession(object):
|
||||
"""This class manages all important services common to any application that uses the network:
|
||||
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
|
||||
@@ -92,8 +95,8 @@ class LBRYSession(object):
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
self.external_ip = '127.0.0.1'
|
||||
self.upnp_handler = None
|
||||
self.upnp_redirects_set = False
|
||||
|
||||
self.upnp_redirects = []
|
||||
|
||||
self.wallet = wallet
|
||||
|
||||
@@ -104,7 +107,7 @@ class LBRYSession(object):
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
logging.debug("Setting up the lbry session")
|
||||
log.debug("Setting up the lbry session")
|
||||
|
||||
if self.lbryid is None:
|
||||
self.lbryid = generate_id()
|
||||
@@ -124,7 +127,7 @@ class LBRYSession(object):
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
else:
|
||||
if self.hash_announcer is None and self.peer_port is not None:
|
||||
logging.warning("The server has no way to advertise its available blobs.")
|
||||
log.warning("The server has no way to advertise its available blobs.")
|
||||
self.hash_announcer = DummyHashAnnouncer()
|
||||
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
@@ -135,41 +138,54 @@ class LBRYSession(object):
|
||||
ds = []
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.upnp_redirects_set is True:
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.peer_finder is not None:
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
if self.hash_announcer is not None:
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
logging.debug("In _try_upnp")
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
logging.debug("Not using upnp")
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
self.upnp_handler = u
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0':
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port is not None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
|
||||
else:
|
||||
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
|
||||
if self.dht_node_port is not None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects_set = True
|
||||
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
log.info("Set UPnP redirect for UPD port %d", self.dht_node_port)
|
||||
else:
|
||||
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
@@ -180,7 +196,7 @@ class LBRYSession(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("Starting the dht")
|
||||
log.debug("Starting the dht")
|
||||
|
||||
def match_port(h, p):
|
||||
return h, p
|
||||
@@ -216,7 +232,7 @@ class LBRYSession(object):
|
||||
return dl
|
||||
|
||||
def _setup_other_components(self):
|
||||
logging.debug("Setting up the rest of the components")
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
@@ -227,19 +243,30 @@ class LBRYSession(object):
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
|
||||
|
||||
self.rate_limiter.tick()
|
||||
self.rate_limiter.start()
|
||||
d1 = self.blob_manager.setup()
|
||||
d2 = self.wallet.start()
|
||||
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
|
||||
|
||||
dl.addErrback(lambda err: err.value.subFailure)
|
||||
return dl
|
||||
|
||||
def _unset_upnp(self):
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = self.upnp_handler
|
||||
if self.peer_port is not None:
|
||||
u.deleteportmapping(self.peer_port, 'TCP')
|
||||
if self.dht_node_port is not None:
|
||||
u.deleteportmapping(self.dht_node_port, 'UDP')
|
||||
self.upnp_redirects_set = False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning("UPnP redirect for %s %d was removed by something else.", protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
return threads.deferToThread(threaded_unset_upnp)
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamCreator(object):
|
||||
"""Classes which derive from this class create a 'stream', which can be any
|
||||
collection of associated blobs and associated metadata. These classes
|
||||
@@ -42,7 +45,7 @@ class StreamCreator(object):
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
logging.debug("stop has been called for StreamCreator")
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
current_blob = self.current_blob
|
||||
|
||||
@@ -3,6 +3,10 @@ import json
|
||||
import logging
|
||||
from twisted.internet import threads
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
@@ -81,7 +85,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
logging.debug("Writing the sd file to disk")
|
||||
log.debug("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
@@ -96,13 +100,26 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
logging.debug("Creating the new blob for the stream descriptor")
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
logging.debug("Wrote the data to the new blob")
|
||||
log.debug("Wrote the data to the new blob")
|
||||
return blob_creator.close()
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
@@ -110,9 +127,10 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
def __init__(self):
|
||||
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
|
||||
self._stream_options = {} # {stream_type: IStreamOptions}
|
||||
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
|
||||
|
||||
def add_stream_info_validator(self, stream_type, sd_info_validator):
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""
|
||||
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
@@ -126,9 +144,13 @@ class StreamDescriptorIdentifier(object):
|
||||
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
|
||||
descriptor from being further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions interface. This class's constructor will be
|
||||
passed the sd_info_validator object containing the raw metadata from the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""
|
||||
@@ -146,35 +168,59 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def get_info_and_factories_for_sd_file(self, sd_path):
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_info_and_factories_for_sd_blob(self, sd_blob):
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _return_info_and_factories(self, sd_info):
|
||||
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
factories = self._get_factories(stream_type)
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
d.addCallback(lambda _: (validator, factories))
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -5,10 +5,14 @@ from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
implements(IRequestCreator)
|
||||
|
||||
@@ -68,6 +72,7 @@ class BlobRequester(object):
|
||||
sent_request = True
|
||||
else:
|
||||
d_r.cancel(InsufficientFundsError())
|
||||
d_r.finished_deferred.addErrback(lambda _: True)
|
||||
return defer.fail(InsufficientFundsError())
|
||||
if sent_request is True:
|
||||
if p_r is not None:
|
||||
@@ -84,7 +89,7 @@ class BlobRequester(object):
|
||||
######### internal calls #########
|
||||
|
||||
def _download_succeeded(self, arg, peer, blob):
|
||||
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
log.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
self._update_local_score(peer, 5.0)
|
||||
peer.update_stats('blobs_downloaded', 1)
|
||||
peer.update_score(5.0)
|
||||
@@ -105,8 +110,8 @@ class BlobRequester(object):
|
||||
|
||||
def _handle_download_error(self, err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
logging.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
@@ -123,7 +128,7 @@ class BlobRequester(object):
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
logging.debug("Blob requester peer search response: %s", str(r))
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -137,6 +142,12 @@ class BlobRequester(object):
|
||||
return [p for p in peers if not p in bad_peers]
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
@@ -201,7 +212,7 @@ class BlobRequester(object):
|
||||
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
|
||||
cancel_func, blob_to_download)
|
||||
|
||||
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
log.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
return request
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
@@ -238,11 +249,11 @@ class BlobRequester(object):
|
||||
def _handle_availability(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
logging.debug("Received a response to the availability request")
|
||||
log.debug("Received a response to the availability request")
|
||||
blob_hashes = response_dict[request.response_identifier]
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self._available_blobs[peer].append(blob_hash)
|
||||
if blob_hash in self._unavailable_blobs[peer]:
|
||||
self._unavailable_blobs[peer].remove(blob_hash)
|
||||
@@ -296,12 +307,13 @@ class BlobRequester(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
log.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
self._update_local_score(peer, -10.0)
|
||||
if isinstance(reason, InvalidResponseError):
|
||||
if isinstance(reason, InvalidResponseError) or isinstance(reason, NoResponseError):
|
||||
peer.update_score(-10.0)
|
||||
else:
|
||||
peer.update_score(-2.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
@@ -1,6 +1,6 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import error, defer, reactor
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.python import failure
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
|
||||
@@ -11,6 +11,9 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientProtocol(Protocol):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
|
||||
@@ -39,7 +42,7 @@ class ClientProtocol(Protocol):
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > MAX_RESPONSE_SIZE:
|
||||
logging.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
log.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
@@ -55,7 +58,7 @@ class ClientProtocol(Protocol):
|
||||
else:
|
||||
err = reason
|
||||
#if self._response_deferreds:
|
||||
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
# log.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
@@ -70,7 +73,7 @@ class ClientProtocol(Protocol):
|
||||
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
logging.debug("Adding a request. Request: %s", str(request))
|
||||
log.debug("Adding a request. Request: %s", str(request))
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
@@ -102,12 +105,14 @@ class ClientProtocol(Protocol):
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
log.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
|
||||
log.debug("In _ask_for_request")
|
||||
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
@@ -117,7 +122,7 @@ class ClientProtocol(Protocol):
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
log.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
self.transport.loseConnection()
|
||||
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
@@ -153,14 +158,15 @@ class ClientProtocol(Protocol):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
|
||||
RequestCanceledError):
|
||||
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError):
|
||||
self.transport.loseConnection()
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError): # The connection manager is closing the connection, so
|
||||
# there's no need to do it here.
|
||||
return err
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
log.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds[key]
|
||||
@@ -178,12 +184,27 @@ class ClientProtocol(Protocol):
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
dl.addCallback(lambda _: self._ask_for_request())
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
log.info("The connection is closing due to an error: %s", str(result.getTraceback()))
|
||||
if failed is False:
|
||||
log.debug("Asking for another request.")
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self._ask_for_request)
|
||||
#self._ask_for_request()
|
||||
else:
|
||||
log.debug("Not asking for another request.")
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
logging.debug("The blob has finished downloading")
|
||||
log.debug("The blob has finished downloading")
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
@@ -194,8 +215,8 @@ class ClientProtocol(Protocol):
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
logging.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
log.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
#self.transport.loseConnection()
|
||||
#return True
|
||||
return err
|
||||
|
||||
@@ -7,49 +7,62 @@ from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self.primary_request_creators = primary_request_creators
|
||||
self.secondary_request_creators = secondary_request_creators
|
||||
self.peer_connections = {} # {Peer: {'connection': connection,
|
||||
# 'request_creators': [IRequestCreator if using this connection]}}
|
||||
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self.next_manage_call = None
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
self.stopped = True
|
||||
|
||||
def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = reactor.callLater(0, self._manage)
|
||||
self.stopped = False
|
||||
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = reactor.callLater(0, self._manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
self.stopped = True
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
closing_deferreds = []
|
||||
for peer in self.peer_connections.keys():
|
||||
for peer in self._peer_connections.keys():
|
||||
|
||||
def close_connection(p):
|
||||
logging.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
log.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
|
||||
if self.peer_connections[p]['factory'].p is not None:
|
||||
d = self.peer_connections[p]['factory'].p.cancel_requests()
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def disconnect_peer():
|
||||
self.peer_connections[p]['connection'].disconnect()
|
||||
if p in self.peer_connections:
|
||||
del self.peer_connections[p]
|
||||
d = defer.Deferred()
|
||||
self.connections_closing[p] = d
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
d.addBoth(lambda _: disconnect_peer())
|
||||
@@ -60,26 +73,27 @@ class ConnectionManager(object):
|
||||
|
||||
def get_next_request(self, peer, protocol):
|
||||
|
||||
logging.debug("Trying to get the next request for peer %s", str(peer))
|
||||
log.debug("Trying to get the next request for peer %s", str(peer))
|
||||
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("The peer has already been told to shut down.")
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("The peer has already been told to shut down.")
|
||||
return defer.succeed(False)
|
||||
|
||||
def handle_error(err):
|
||||
if err.check(InsufficientFundsError):
|
||||
self.downloader.insufficient_funds()
|
||||
return False
|
||||
else:
|
||||
return err
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].remove(request_creator)
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].append(request_creator)
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
def check_requests(requests):
|
||||
@@ -89,7 +103,7 @@ class ConnectionManager(object):
|
||||
def get_secondary_requests_if_necessary(have_request):
|
||||
if have_request is True:
|
||||
ds = []
|
||||
for s_r_c in self.secondary_request_creators:
|
||||
for s_r_c in self._secondary_request_creators:
|
||||
d = s_r_c.send_next_request(peer, protocol)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
@@ -100,7 +114,7 @@ class ConnectionManager(object):
|
||||
|
||||
ds = []
|
||||
|
||||
for p_r_c in self.primary_request_creators:
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
@@ -112,11 +126,11 @@ class ConnectionManager(object):
|
||||
return dl
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
if peer in self.peer_connections:
|
||||
del self.peer_connections[peer]
|
||||
if peer in self.connections_closing:
|
||||
d = self.connections_closing[peer]
|
||||
del self.connections_closing[peer]
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
|
||||
def _rank_request_creator_connections(self):
|
||||
@@ -125,30 +139,30 @@ class ConnectionManager(object):
|
||||
connections open that it likes
|
||||
"""
|
||||
def count_peers(request_creator):
|
||||
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
|
||||
return len([p for p in self._peer_connections.itervalues() if request_creator in p.request_creators])
|
||||
|
||||
return sorted(self.primary_request_creators, key=count_peers)
|
||||
return sorted(self._primary_request_creators, key=count_peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if peer is not None:
|
||||
logging.debug("Trying to connect to %s", str(peer))
|
||||
if peer is not None and self.stopped is False:
|
||||
log.debug("Trying to connect to %s", str(peer))
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory)
|
||||
self.peer_connections[peer] = {'connection': connection,
|
||||
'request_creators': self.primary_request_creators[:],
|
||||
'factory': factory}
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
def _manage(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
def get_new_peers(request_creators):
|
||||
logging.debug("Trying to get a new peer to connect to")
|
||||
log.debug("Trying to get a new peer to connect to")
|
||||
if len(request_creators) > 0:
|
||||
logging.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
log.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
d = request_creators[0].get_new_peers()
|
||||
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
|
||||
return d
|
||||
@@ -158,20 +172,20 @@ class ConnectionManager(object):
|
||||
def pick_best_peer(peers):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
logging.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
log.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
if peers is None:
|
||||
return None
|
||||
for peer in peers:
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
if not peer in self._peer_connections:
|
||||
log.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
return peer
|
||||
logging.debug("Couldn't find a good peer to connect to")
|
||||
log.debug("Couldn't find a good peer to connect to")
|
||||
return None
|
||||
|
||||
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
if len(self._peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
d = get_new_peers(ordered_request_creators)
|
||||
d.addCallback(pick_best_peer)
|
||||
d.addCallback(self._connect_to_peer)
|
||||
|
||||
self.next_manage_call = reactor.callLater(1, self._manage)
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
@@ -32,8 +32,9 @@ class DHTPeerFinder(object):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
|
||||
def filter_peers(peer_list):
|
||||
peers = set(peer_list)
|
||||
good_peers = []
|
||||
for host, port in peer_list:
|
||||
for host, port in peers:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
|
||||
@@ -5,6 +5,9 @@ from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
@@ -24,7 +27,7 @@ class DownloadManager(object):
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
logging.debug("Requested the initial blobs from the info finder")
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
@@ -33,7 +36,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_start(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
log.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -49,7 +52,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_stop(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
log.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -63,21 +66,21 @@ class DownloadManager(object):
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
logging.debug("Adding %s to blobs", str(blob_infos))
|
||||
log.debug("Adding %s to blobs", str(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
log.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
|
||||
def error_during_add(err):
|
||||
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
log.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
log.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
|
||||
@@ -5,9 +5,15 @@ from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
implements(interfaces.IMetadataHandler)
|
||||
|
||||
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
logging.debug("Returning the blob info")
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -72,7 +78,7 @@ class SingleProgressManager(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
if self.finished is False:
|
||||
self.finished = True
|
||||
reactor.callLater(0, self.finished_callback, blob)
|
||||
@@ -99,6 +105,9 @@ class StandaloneBlobDownloader(object):
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
@@ -127,7 +136,10 @@ class StandaloneBlobDownloader(object):
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
self.finished_deferred.callback(blob)
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -4,6 +4,9 @@ from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
@@ -61,18 +64,18 @@ class StreamProgressManager(object):
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
logging.debug("delete_blob_after_finished is True")
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
logging.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
logging.debug("delete_blob_after_finished is False")
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
@@ -115,6 +118,8 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
@@ -126,14 +131,18 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
log.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -37,7 +40,7 @@ class BlobAvailabilityHandler(object):
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
logging.debug("Received the client's list of requested blobs")
|
||||
log.debug("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -58,7 +61,7 @@ class BlobRequestHandler(object):
|
||||
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
logging.debug("Received the client's request to send a blob")
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
@@ -75,11 +78,11 @@ class BlobRequestHandler(object):
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
logging.debug("Sending %s to client", str(blob))
|
||||
log.debug("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
return response
|
||||
logging.debug("We can not send %s", str(blob))
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response_fields['error'] = "BLOB_UNAVAILABLE"
|
||||
return response
|
||||
|
||||
@@ -129,13 +132,13 @@ class BlobRequestHandler(object):
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
logging.info("Starting the file upload")
|
||||
log.info("Starting the file upload")
|
||||
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
logging.info("Setting expected payment")
|
||||
log.info("Setting expected payment")
|
||||
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
@@ -151,6 +154,6 @@ class BlobRequestHandler(object):
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
log.info("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
@@ -26,7 +29,7 @@ class ServerProtocol(Protocol):
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
logging.debug("Got a connection")
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
@@ -34,7 +37,7 @@ class ServerProtocol(Protocol):
|
||||
if enabled is True:
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
logging.debug("Setting the request handler")
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
@@ -42,10 +45,10 @@ class ServerProtocol(Protocol):
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
@@ -53,7 +56,7 @@ class ServerProtocol(Protocol):
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
logging.debug("Registering the producer")
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
@@ -61,7 +64,7 @@ class ServerProtocol(Protocol):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
log.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and return request for information about
|
||||
more blobs that are associated with streams"""
|
||||
@@ -52,7 +55,7 @@ class ServerRequestHandler(object):
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk != '':
|
||||
logging.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
log.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
@@ -76,7 +79,7 @@ class ServerRequestHandler(object):
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
logging.debug("Requesting more data from the producer")
|
||||
log.debug("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
@@ -84,8 +87,8 @@ class ServerRequestHandler(object):
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
logging.debug("Received data")
|
||||
logging.debug("%s", str(data))
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
@@ -96,10 +99,10 @@ class ServerRequestHandler(object):
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
else:
|
||||
logging.info("Request buff not a valid json message")
|
||||
logging.info("Request buff: %s", str(self.request_buff))
|
||||
log.info("Request buff not a valid json message")
|
||||
log.info("Request buff: %s", str(self.request_buff))
|
||||
else:
|
||||
logging.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
log.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
@@ -112,7 +115,7 @@ class ServerRequestHandler(object):
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
@@ -122,15 +125,15 @@ class ServerRequestHandler(object):
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
logging.info("Sending a response of length %s", str(len(m)))
|
||||
logging.debug("Response: %s", str(m))
|
||||
log.info("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
logging.debug("Handling a request")
|
||||
logging.debug(str(msg))
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
@@ -140,11 +143,11 @@ class ServerRequestHandler(object):
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
logging.debug("Finished making the response message. Response: %s", str(response))
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
from twisted.internet import task, reactor
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
|
||||
def rerun(err, *args, **kwargs):
|
||||
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
|
||||
return err
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addErrback(rerun, *args, **kwargs)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
@@ -5,6 +5,9 @@ from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
@@ -78,12 +81,12 @@ class CryptStreamBlobMaker(object):
|
||||
return done, num_bytes_to_write
|
||||
|
||||
def close(self):
|
||||
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self._close_buffer()
|
||||
d = self.blob.close()
|
||||
d.addCallback(self._return_info)
|
||||
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
return d
|
||||
|
||||
def _write_buffer(self):
|
||||
|
||||
@@ -12,6 +12,9 @@ from lbrynet.core.StreamCreator import StreamCreator
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(StreamCreator):
|
||||
"""Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
@@ -57,18 +60,18 @@ class CryptStreamCreator(StreamCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _finalize(self):
|
||||
logging.debug("_finalize has been called")
|
||||
log.debug("_finalize has been called")
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob_creator = self.blob_manager.get_blob_creator()
|
||||
logging.debug("Created the finished_deferred")
|
||||
log.debug("Created the finished_deferred")
|
||||
final_blob = self._get_blob_maker(iv, final_blob_creator)
|
||||
logging.debug("Created the final blob")
|
||||
logging.debug("Calling close on final blob")
|
||||
log.debug("Created the final blob")
|
||||
log.debug("Calling close on final blob")
|
||||
d = final_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
self.finished_deferreds.append(d)
|
||||
logging.debug("called close on final blob, returning from make_final_blob")
|
||||
log.debug("called close on final blob, returning from make_final_blob")
|
||||
return d
|
||||
|
||||
def _write(self, data):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
@@ -9,6 +10,9 @@ from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
@@ -71,6 +75,9 @@ class CryptStreamDownloader(object):
|
||||
|
||||
self.points_paid = 0.0
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
return self.start()
|
||||
@@ -79,10 +86,6 @@ class CryptStreamDownloader(object):
|
||||
|
||||
def start(self):
|
||||
|
||||
def set_finished_deferred():
|
||||
self.finished_deferred = defer.Deferred()
|
||||
return self.finished_deferred
|
||||
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
@@ -92,11 +95,13 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
fd = self.finished_deferred
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: set_finished_deferred())
|
||||
d.addCallback(lambda _: fd)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
|
||||
def check_if_stop_succeeded(success):
|
||||
self.stopping = False
|
||||
@@ -112,8 +117,8 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
d = self.download_manager.stop_downloading()
|
||||
self._fire_completed_deferred()
|
||||
d.addCallback(check_if_stop_succeeded)
|
||||
d.addCallback(lambda _: self._fire_completed_deferred(err))
|
||||
return d
|
||||
|
||||
def _start_failed(self):
|
||||
@@ -196,10 +201,15 @@ class CryptStreamDownloader(object):
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self):
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
@@ -209,5 +219,5 @@ class CryptStreamDownloader(object):
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
@@ -0,0 +1,25 @@
|
||||
import logging
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
current = start
|
||||
old_dirs = []
|
||||
while current < end:
|
||||
if current == 0:
|
||||
from lbrynet.db_migrator.migrate0to1 import do_migration
|
||||
old_dirs.append(do_migration(db_dir))
|
||||
current += 1
|
||||
return old_dirs
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
sys.stderr = open("migrator.err.log", 'w')
|
||||
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migration_script()
|
||||
@@ -0,0 +1,307 @@
|
||||
import sqlite3
|
||||
import unqlite
|
||||
import leveldb
|
||||
import shutil
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
known_dbs = ['lbryfile_desc.db', 'lbryfiles.db', 'valuable_blobs.db', 'blobs.db',
|
||||
'lbryfile_blob.db', 'lbryfile_info.db', 'settings.db', 'blind_settings.db',
|
||||
'blind_peers.db', 'blind_info.db', 'lbryfile_info.db', 'lbryfile_manager.db',
|
||||
'live_stream.db', 'stream_info.db', 'stream_blob.db', 'stream_desc.db']
|
||||
|
||||
|
||||
def do_move(from_dir, to_dir):
|
||||
for known_db in known_dbs:
|
||||
known_db_path = os.path.join(from_dir, known_db)
|
||||
if os.path.exists(known_db_path):
|
||||
log.debug("Moving %s to %s",
|
||||
os.path.abspath(known_db_path),
|
||||
os.path.abspath(os.path.join(to_dir, known_db)))
|
||||
shutil.move(known_db_path, os.path.join(to_dir, known_db))
|
||||
else:
|
||||
log.debug("Did not find %s", os.path.abspath(known_db_path))
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
old_dir = os.path.join(db_dir, "_0_to_1_old")
|
||||
new_dir = os.path.join(db_dir, "_0_to_1_new")
|
||||
try:
|
||||
log.info("Moving dbs from the real directory to %s", os.path.abspath(old_dir))
|
||||
os.makedirs(old_dir)
|
||||
do_move(db_dir, old_dir)
|
||||
except:
|
||||
log.error("An error occurred moving the old db files.")
|
||||
raise
|
||||
try:
|
||||
log.info("Creating the new directory in %s", os.path.abspath(new_dir))
|
||||
os.makedirs(new_dir)
|
||||
|
||||
except:
|
||||
log.error("An error occurred creating the new directory.")
|
||||
raise
|
||||
try:
|
||||
log.info("Doing the migration")
|
||||
migrate_blob_db(old_dir, new_dir)
|
||||
migrate_lbryfile_db(old_dir, new_dir)
|
||||
migrate_livestream_db(old_dir, new_dir)
|
||||
migrate_ptc_db(old_dir, new_dir)
|
||||
migrate_lbryfile_manager_db(old_dir, new_dir)
|
||||
migrate_settings_db(old_dir, new_dir)
|
||||
migrate_repeater_db(old_dir, new_dir)
|
||||
log.info("Migration succeeded")
|
||||
except:
|
||||
log.error("An error occurred during the migration. Restoring.")
|
||||
do_move(old_dir, db_dir)
|
||||
raise
|
||||
try:
|
||||
log.info("Moving dbs in the new directory to the real directory")
|
||||
do_move(new_dir, db_dir)
|
||||
db_revision = open(os.path.join(db_dir, 'db_revision'), mode='w+')
|
||||
db_revision.write("1")
|
||||
db_revision.close()
|
||||
os.rmdir(new_dir)
|
||||
except:
|
||||
log.error("An error occurred moving the new db files.")
|
||||
raise
|
||||
return old_dir
|
||||
|
||||
|
||||
def migrate_blob_db(old_db_dir, new_db_dir):
|
||||
old_blob_db_path = os.path.join(old_db_dir, "blobs.db")
|
||||
if not os.path.exists(old_blob_db_path):
|
||||
return True
|
||||
|
||||
old_db = leveldb.LevelDB(old_blob_db_path)
|
||||
new_db_conn = sqlite3.connect(os.path.join(new_db_dir, "blobs.db"))
|
||||
c = new_db_conn.cursor()
|
||||
c.execute("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real"
|
||||
")")
|
||||
new_db_conn.commit()
|
||||
c = new_db_conn.cursor()
|
||||
for blob_hash, blob_info in old_db.RangeIter():
|
||||
blob_length, verified_time, announce_time = json.loads(blob_info)
|
||||
c.execute("insert into blobs values (?, ?, ?, ?)",
|
||||
(blob_hash, blob_length, verified_time, announce_time))
|
||||
new_db_conn.commit()
|
||||
new_db_conn.close()
|
||||
|
||||
|
||||
def migrate_lbryfile_db(old_db_dir, new_db_dir):
|
||||
old_lbryfile_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
if not os.path.exists(old_lbryfile_db_path):
|
||||
return True
|
||||
|
||||
stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_info.db"))
|
||||
stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_blob.db"))
|
||||
stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_desc.db"))
|
||||
|
||||
db_conn = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
c = db_conn.cursor()
|
||||
c.execute("create table if not exists lbry_files (" +
|
||||
" stream_hash text primary key, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" suggested_file_name text" +
|
||||
")")
|
||||
c.execute("create table if not exists lbry_file_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
c.execute("create table if not exists lbry_file_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
db_conn.commit()
|
||||
c = db_conn.cursor()
|
||||
for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
key, name, suggested_file_name = json.loads(stream_info)
|
||||
c.execute("insert into lbry_files values (?, ?, ?, ?)",
|
||||
(stream_hash, key, name, suggested_file_name))
|
||||
db_conn.commit()
|
||||
c = db_conn.cursor()
|
||||
for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
position, iv, length = json.loads(blob_info)
|
||||
c.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
(b_h, s_h, position, iv, length))
|
||||
db_conn.commit()
|
||||
c = db_conn.cursor()
|
||||
for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
c.execute("insert into lbry_file_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
db_conn.commit()
|
||||
db_conn.close()
|
||||
|
||||
|
||||
def migrate_livestream_db(old_db_dir, new_db_dir):
|
||||
old_db_path = os.path.join(old_db_dir, "stream_info.db")
|
||||
if not os.path.exists(old_db_path):
|
||||
return True
|
||||
stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_info.db"))
|
||||
stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_blob.db"))
|
||||
stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_desc.db"))
|
||||
|
||||
db_conn = sqlite3.connect(os.path.join(new_db_dir, "live_stream.db"))
|
||||
|
||||
c = db_conn.cursor()
|
||||
|
||||
c.execute("create table if not exists live_streams (" +
|
||||
" stream_hash text primary key, " +
|
||||
" public_key text, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" next_announce_time real" +
|
||||
")")
|
||||
c.execute("create table if not exists live_stream_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" revision integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" signature text, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
c.execute("create table if not exists live_stream_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
|
||||
db_conn.commit()
|
||||
|
||||
c = db_conn.cursor()
|
||||
for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
public_key, key, name, next_announce_time = json.loads(stream_info)
|
||||
c.execute("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, public_key, key, name, next_announce_time))
|
||||
db_conn.commit()
|
||||
c = db_conn.cursor()
|
||||
for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
position, revision, iv, length, signature = json.loads(blob_info)
|
||||
c.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(b_h, s_h, position, revision, iv, length, signature))
|
||||
db_conn.commit()
|
||||
c = db_conn.cursor()
|
||||
for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
c.execute("insert into live_stream_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
db_conn.commit()
|
||||
db_conn.close()
|
||||
|
||||
|
||||
def migrate_ptc_db(old_db_dir, new_db_dir):
|
||||
old_db_path = os.path.join(old_db_dir, "ptcwallet.db")
|
||||
if not os.path.exists(old_db_path):
|
||||
return True
|
||||
old_db = leveldb.LevelDB(old_db_path)
|
||||
try:
|
||||
p_key = old_db.Get("private_key")
|
||||
new_db = unqlite.UnQLite(os.path.join(new_db_dir, "ptcwallet.db"))
|
||||
new_db['private_key'] = p_key
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def migrate_lbryfile_manager_db(old_db_dir, new_db_dir):
|
||||
old_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
if not os.path.exists(old_db_path):
|
||||
return True
|
||||
old_db = leveldb.LevelDB(old_db_path)
|
||||
new_db = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
c = new_db.cursor()
|
||||
c.execute("create table if not exists lbry_file_options (" +
|
||||
" blob_data_rate real, " +
|
||||
" status text," +
|
||||
" stream_hash text,"
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
new_db.commit()
|
||||
LBRYFILE_STATUS = "t"
|
||||
LBRYFILE_OPTIONS = "o"
|
||||
c = new_db.cursor()
|
||||
for k, v in old_db.RangeIter():
|
||||
key_type, stream_hash = json.loads(k)
|
||||
if key_type == LBRYFILE_STATUS:
|
||||
try:
|
||||
rate = json.loads(old_db.Get(json.dumps((LBRYFILE_OPTIONS, stream_hash))))[0]
|
||||
except KeyError:
|
||||
rate = None
|
||||
c.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
(rate, v, stream_hash))
|
||||
new_db.commit()
|
||||
new_db.close()
|
||||
|
||||
|
||||
def migrate_settings_db(old_db_dir, new_db_dir):
|
||||
old_settings_db_path = os.path.join(old_db_dir, "settings.db")
|
||||
if not os.path.exists(old_settings_db_path):
|
||||
return True
|
||||
old_db = leveldb.LevelDB(old_settings_db_path)
|
||||
new_db = unqlite.UnQLite(os.path.join(new_db_dir, "settings.db"))
|
||||
for k, v in old_db.RangeIter():
|
||||
new_db[k] = v
|
||||
|
||||
|
||||
def migrate_repeater_db(old_db_dir, new_db_dir):
|
||||
old_repeater_db_path = os.path.join(old_db_dir, "valuable_blobs.db")
|
||||
if not os.path.exists(old_repeater_db_path):
|
||||
return True
|
||||
old_db = leveldb.LevelDB(old_repeater_db_path)
|
||||
info_db = sqlite3.connect(os.path.join(new_db_dir, "blind_info.db"))
|
||||
peer_db = sqlite3.connect(os.path.join(new_db_dir, "blind_peers.db"))
|
||||
unql_db = unqlite.UnQLite(os.path.join(new_db_dir, "blind_settings.db"))
|
||||
BLOB_INFO_TYPE = 'b'
|
||||
SETTING_TYPE = 's'
|
||||
PEER_TYPE = 'p'
|
||||
info_c = info_db.cursor()
|
||||
info_c.execute("create table if not exists valuable_blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" reference text, " +
|
||||
" peer_host text, " +
|
||||
" peer_port integer, " +
|
||||
" peer_score text" +
|
||||
")")
|
||||
info_db.commit()
|
||||
peer_c = peer_db.cursor()
|
||||
peer_c.execute("create table if not exists approved_peers (" +
|
||||
" ip_address text, " +
|
||||
" port integer" +
|
||||
")")
|
||||
peer_db.commit()
|
||||
info_c = info_db.cursor()
|
||||
peer_c = peer_db.cursor()
|
||||
for k, v in old_db.RangeIter():
|
||||
key_type, key_rest = json.loads(k)
|
||||
if key_type == PEER_TYPE:
|
||||
host, port = key_rest
|
||||
peer_c.execute("insert into approved_peers values (?, ?)",
|
||||
(host, port))
|
||||
elif key_type == SETTING_TYPE:
|
||||
unql_db[key_rest] = v
|
||||
elif key_type == BLOB_INFO_TYPE:
|
||||
blob_hash = key_rest
|
||||
length, reference, peer_host, peer_port, peer_score = json.loads(v)
|
||||
info_c.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, reference, peer_host, peer_port, peer_score))
|
||||
info_db.commit()
|
||||
peer_db.commit()
|
||||
info_db.close()
|
||||
peer_db.close()
|
||||
@@ -99,7 +99,10 @@ class Bencode(Encoding):
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError, 'Cannot decode empty string'
|
||||
return self._decodeRecursive(data)[0]
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError, e.message
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
import hashlib, random, struct, time, math, binascii
|
||||
import argparse
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, error
|
||||
import constants
|
||||
import routingtable
|
||||
import datastore
|
||||
@@ -22,6 +22,9 @@ from hashwatcher import HashWatcher
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
|
||||
@@ -139,7 +142,13 @@ class Node(object):
|
||||
"""
|
||||
# Prepare the underlying Kademlia protocol
|
||||
if self.port is not None:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol) #IGNORE:E1101
|
||||
try:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running." % str(e))
|
||||
#IGNORE:E1101
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
if knownNodeAddresses != None:
|
||||
bootstrapContacts = []
|
||||
@@ -231,12 +240,12 @@ class Node(object):
|
||||
known_nodes = {}
|
||||
|
||||
def log_error(err, n):
|
||||
logging.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
logging.error(binascii.hexlify(err.getErrorMessage()))
|
||||
logging.error(err.getTraceback())
|
||||
log.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
log.error(binascii.hexlify(err.getErrorMessage()))
|
||||
log.error(err.getTraceback())
|
||||
|
||||
def log_success(res):
|
||||
logging.debug("Response to store request: %s", str(res))
|
||||
log.debug("Response to store request: %s", str(res))
|
||||
return res
|
||||
|
||||
def announce_to_peer(responseTuple):
|
||||
@@ -332,7 +341,7 @@ class Node(object):
|
||||
# originalPublisherID = self.id
|
||||
#
|
||||
# def log_error(err):
|
||||
# logging.error(err.getErrorMessage())
|
||||
# log.error(err.getErrorMessage())
|
||||
#
|
||||
# # Prepare a callback for doing "STORE" RPC calls
|
||||
# def executeStoreRPCs(nodes):
|
||||
@@ -507,7 +516,7 @@ class Node(object):
|
||||
else:
|
||||
raise TypeError, 'No NodeID given. Therefore we can\'t store this node'
|
||||
|
||||
if self_store is True and self.externalIP is not None:
|
||||
if self_store is True and self.externalIP:
|
||||
contact = Contact(self.id, self.externalIP, self.port, None, None)
|
||||
compact_ip = contact.compact_ip()
|
||||
elif '_rpcNodeContact' in kwargs:
|
||||
@@ -756,7 +765,7 @@ class Node(object):
|
||||
searchIteration()
|
||||
|
||||
def log_error(err):
|
||||
logging.error(err.getErrorMessage())
|
||||
log.error(err.getErrorMessage())
|
||||
|
||||
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
|
||||
def searchIteration():
|
||||
|
||||
@@ -6,6 +6,9 @@ import sys
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def print_usage():
|
||||
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
|
||||
|
||||
@@ -13,15 +16,15 @@ def print_usage():
|
||||
def join_network(udp_port, known_nodes):
|
||||
lbryid = generate_id()
|
||||
|
||||
logging.info('Creating Node...')
|
||||
log.info('Creating Node...')
|
||||
node = Node(udpPort=udp_port, lbryid=lbryid)
|
||||
|
||||
logging.info('Joining network...')
|
||||
log.info('Joining network...')
|
||||
d = node.joinNetwork(known_nodes)
|
||||
|
||||
def log_network_size():
|
||||
logging.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
logging.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
log.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
log.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
|
||||
d.addCallback(lambda _: log_network_size())
|
||||
|
||||
@@ -36,7 +39,7 @@ def get_hosts(node, h):
|
||||
print "Hosts returned from the DHT: "
|
||||
print hosts
|
||||
|
||||
logging.info("Looking up %s", h)
|
||||
log.info("Looking up %s", h)
|
||||
d = node.getPeersForBlob(h)
|
||||
d.addCallback(print_hosts)
|
||||
return d
|
||||
@@ -48,9 +51,9 @@ def announce_hash(node, h):
|
||||
def log_results(results):
|
||||
for success, result in results:
|
||||
if success:
|
||||
logging.info("Succeeded: %s", str(result))
|
||||
log.info("Succeeded: %s", str(result))
|
||||
else:
|
||||
logging.info("Failed: %s", str(result.getErrorMessage()))
|
||||
log.info("Failed: %s", str(result.getErrorMessage()))
|
||||
|
||||
d.addCallback(log_results)
|
||||
return d
|
||||
@@ -80,11 +83,11 @@ def run_dht_script(dht_func):
|
||||
d.addCallback(run_dht_func)
|
||||
|
||||
def log_err(err):
|
||||
logging.error("An error occurred: %s", err.getTraceback())
|
||||
log.error("An error occurred: %s", err.getTraceback())
|
||||
return err
|
||||
|
||||
def shut_down():
|
||||
logging.info("Shutting down")
|
||||
log.info("Shutting down")
|
||||
reactor.stop()
|
||||
|
||||
d.addErrback(log_err)
|
||||
|
||||
@@ -445,10 +445,7 @@ class IQueryHandlerFactory(Interface):
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
class IStreamDownloaderOptions(Interface):
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Return the list of options that can be used to modify IStreamDownloader behavior
|
||||
@@ -459,8 +456,28 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: [(option_description, default)]
|
||||
@rtype: [(string, string)]
|
||||
@return: [DownloadOption]
|
||||
@rtype: [DownloadOption]
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
|
||||
def can_download(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Decide whether the downloaders created by this factory can download the stream described by sd_validator
|
||||
|
||||
@param sd_validator: object containing stream metadata
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: True if the downloaders can download the stream, False otherwise
|
||||
@rtype: bool
|
||||
"""
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
@@ -470,10 +487,10 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param sd_validator: object containing stream metadata which will be given to the IStreamDownloader
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param options: a list of strings that will be used by the IStreamDownloaderFactory to
|
||||
@param options: a list of values that will be used by the IStreamDownloaderFactory to
|
||||
construct the IStreamDownloader. the options are in the same order as they were given
|
||||
by get_downloader_options.
|
||||
@type options: [string]
|
||||
@type options: [Object]
|
||||
|
||||
@param payment_rate_manager: the PaymentRateManager which the IStreamDownloader should use.
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
@@ -504,7 +521,7 @@ class IStreamDownloader(Interface):
|
||||
@rtype: Deferred which fires with anything
|
||||
"""
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
"""
|
||||
this function informs the stream downloader that funds are too low to finish downloading.
|
||||
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
import sqlite3
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBLBRYFileMetadataManager(object):
|
||||
@@ -16,163 +21,226 @@ class DBLBRYFileMetadataManager(object):
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, file_name, key, suggested_file_name)
|
||||
d = self._store_stream(stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
log.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_desc.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool("sqlite3", (os.path.join(self.db_dir, "lbryfile_info.db")),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists lbry_files (" +
|
||||
" stream_hash text primary key, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" suggested_file_name text" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from lbry_files where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((key, name, suggested_file_name)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into lbry_files values (?, ?, ?, ?)",
|
||||
(stream_hash, key, name, suggested_file_name))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
d = self.db_conn.runQuery("select key, stream_name, suggested_file_name from lbry_files where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
d = self.db_conn.runQuery("select position from lbry_file_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, iv, length = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, iv, length from lbry_file_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.iv,
|
||||
blob_info.length)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.iv, blob_info.length))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_file_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
log.info("Saving sd blob hash %s to stream hash %s", str(sd_blob_hash), str(stream_hash))
|
||||
d = self.db_conn.runQuery("insert into lbry_file_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
def ignore_duplicate(err):
|
||||
err.trap(sqlite3.IntegrityError)
|
||||
log.info("sd blob hash already known")
|
||||
|
||||
d.addErrback(ignore_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
log.info("Looking up sd blob hashes for stream hash %s", str(stream_hash))
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from lbry_file_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempLBRYFileMetadataManager(object):
|
||||
|
||||
@@ -2,15 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
|
||||
import os
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LBRYFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
@@ -26,7 +31,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
blob_num = blob['blob_num']
|
||||
iv = blob['iv']
|
||||
crypt_blobs.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, hex_stream_name, key,
|
||||
suggested_file_name, crypt_blobs)
|
||||
|
||||
@@ -75,12 +80,53 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
return d
|
||||
|
||||
|
||||
def publish_sd_blob(stream_info_manager, blob_manager, stream_hash):
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
|
||||
def add_sd_blob_to_stream(sd_blob_hash):
|
||||
d = stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
d.addCallback(lambda _: sd_blob_hash)
|
||||
return d
|
||||
|
||||
d.addCallback(add_sd_blob_to_stream)
|
||||
return d
|
||||
|
||||
|
||||
def create_plain_sd(stream_info_manager, stream_hash, file_name, overwrite_existing=False):
|
||||
|
||||
def _get_file_name():
|
||||
actual_file_name = file_name
|
||||
if os.path.exists(actual_file_name):
|
||||
ext_num = 1
|
||||
while os.path.exists(actual_file_name + "_" + str(ext_num)):
|
||||
ext_num += 1
|
||||
actual_file_name = actual_file_name + "_" + str(ext_num)
|
||||
return actual_file_name
|
||||
|
||||
if overwrite_existing is False:
|
||||
d = threads.deferToThread(_get_file_name())
|
||||
else:
|
||||
d = defer.succeed(file_name)
|
||||
|
||||
def do_create(file_name):
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name)
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
return d
|
||||
|
||||
d.addCallback(do_create)
|
||||
return d
|
||||
|
||||
|
||||
class LBRYFileStreamDescriptorValidator(object):
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
try:
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
key = self.raw_info['key']
|
||||
@@ -88,11 +134,10 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
stream_hash = self.raw_info['stream_hash']
|
||||
blobs = self.raw_info['blobs']
|
||||
except KeyError as e:
|
||||
raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
for c in hex_suggested_file_name:
|
||||
if c not in '0123456789abcdef':
|
||||
raise ValueError("Invalid stream descriptor: "
|
||||
"suggested file name is not a hex-encoded string")
|
||||
raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string")
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name)
|
||||
h.update(key)
|
||||
@@ -118,10 +163,11 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
for blob in blobs:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
if blobs[-1]['length'] != 0:
|
||||
raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
h.update(blobs_hashsum.digest())
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
log.debug("It is validated")
|
||||
return defer.succeed(True)
|
||||
|
||||
def info_to_show(self):
|
||||
@@ -130,9 +176,15 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(size_so_far)))
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
@@ -3,14 +3,20 @@ import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.client.LBRYFileMetadataHandler import LBRYFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.python.procutils import which
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileDownloader(CryptStreamDownloader):
|
||||
@@ -40,9 +46,34 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
def delete_data(self):
|
||||
d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def get_blob_hashes(blob_infos):
|
||||
return [b[0] for b in blob_infos if b[0] is not None]
|
||||
|
||||
d1.addCallback(get_blob_hashes)
|
||||
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def combine_blob_hashes(results):
|
||||
blob_hashes = []
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
blob_hashes.extend(result)
|
||||
return blob_hashes
|
||||
|
||||
def delete_blobs(blob_hashes):
|
||||
self.blob_manager.delete_blobs(blob_hashes)
|
||||
return True
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
dl.addCallback(combine_blob_hashes)
|
||||
dl.addCallback(delete_blobs)
|
||||
return dl
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self))
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
@@ -60,6 +91,15 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b[3] for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
@@ -94,37 +134,30 @@ class LBRYFileDownloaderFactory(object):
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.wallet = wallet
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager, **kwargs):
|
||||
if options[0] is not None:
|
||||
payment_rate_manager.float(options[0])
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
payment_rate_manager.min_blob_data_payment_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager, sd_validator.raw_info,
|
||||
upload_allowed)
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info, upload_allowed)
|
||||
d = downloader.set_stream_info()
|
||||
d.addCallback(lambda _: downloader)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
@@ -139,8 +172,15 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
self.download_directory = download_directory
|
||||
self.file_name = file_name
|
||||
self.file_written_to = None
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
if self.file_written_to is not None:
|
||||
return str(self.file_written_to)
|
||||
else:
|
||||
return str(self.file_name)
|
||||
|
||||
def set_stream_info(self):
|
||||
d = LBRYFileDownloader.set_stream_info(self)
|
||||
|
||||
@@ -154,14 +194,14 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
d.addCallback(lambda _: set_file_name())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = LBRYFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
@@ -175,7 +215,14 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
file_name + "_" + str(ext_num))):
|
||||
ext_num += 1
|
||||
file_name = file_name + "_" + str(ext_num)
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
try:
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
self.file_written_to = os.path.join(self.download_directory, file_name)
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError("Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % str(os.path.join(self.download_directory,
|
||||
file_name)))
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
@@ -212,7 +259,8 @@ class LBRYFileSaverFactory(LBRYFileDownloaderFactory):
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet,
|
||||
self.download_directory, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
|
||||
|
||||
@@ -224,14 +272,14 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
self.process = None
|
||||
self.process_log = None
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = LBRYFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def start_process():
|
||||
@@ -276,9 +324,21 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
|
||||
|
||||
class LBRYFileOpenerFactory(LBRYFileDownloaderFactory):
|
||||
def can_download(self, sd_validator):
|
||||
if which('vlc'):
|
||||
return True
|
||||
elif os.name == "nt":
|
||||
paths = [r'C:\Program Files\VideoLAN\VLC\vlc.exe',
|
||||
r'C:\Program Files (x86)\VideoLAN\VLC\vlc.exe']
|
||||
for p in paths:
|
||||
if os.path.exists(p):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Stream"
|
||||
@@ -4,6 +4,9 @@ from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
|
||||
@@ -31,6 +34,6 @@ class LBRYFileMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
@@ -0,0 +1,55 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType, LBRYFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
sd_identifier.add_stream_type(LBRYFileStreamType, LBRYFileStreamDescriptorValidator, LBRYFileOptions())
|
||||
|
||||
|
||||
class LBRYFileOptions(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.min_blob_data_payment_rate
|
||||
|
||||
rate_choices = []
|
||||
rate_choices.append(DownloadOptionChoice(prm.min_blob_data_payment_rate,
|
||||
"No change - %s" % get_default_data_rate_description(),
|
||||
"No change - %s" % get_default_data_rate_description()))
|
||||
if prm.min_blob_data_payment_rate is not None:
|
||||
rate_choices.append(DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
|
||||
rate_choices.append(DownloadOptionChoice(float,
|
||||
"Enter rate in LBC/MB",
|
||||
"Enter rate in LBC/MB"))
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
rate_choices,
|
||||
"Rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(bool,
|
||||
None,
|
||||
None,
|
||||
bool_options_description=("Allow", "Disallow")),
|
||||
],
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -14,6 +14,9 @@ from twisted.protocols.basic import FileSender
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
"""
|
||||
A CryptStreamCreator which adds itself and its additional metadata to an LBRYFileManager
|
||||
@@ -30,10 +33,10 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
self.blob_infos = []
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
self.blob_infos.append(blob_info)
|
||||
|
||||
def _save_lbry_file_info(self):
|
||||
def _save_stream_info(self):
|
||||
stream_info_manager = self.lbry_file_manager.stream_info_manager
|
||||
d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
|
||||
binascii.hexlify(self.key),
|
||||
@@ -43,8 +46,6 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def setup(self):
|
||||
d = CryptStreamCreator.setup(self)
|
||||
d.addCallback(lambda _: self.stream_hash)
|
||||
|
||||
return d
|
||||
|
||||
def _get_blobs_hashsum(self):
|
||||
@@ -76,10 +77,7 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def _finished(self):
|
||||
self._make_stream_hash()
|
||||
d = self._save_lbry_file_info()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.change_lbry_file_status(
|
||||
self.stream_hash, ManagedLBRYFileDownloader.STATUS_FINISHED
|
||||
))
|
||||
d = self._save_stream_info()
|
||||
return d
|
||||
|
||||
|
||||
@@ -128,11 +126,11 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
"""
|
||||
|
||||
def stop_file(creator):
|
||||
logging.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
log.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
return creator.stop()
|
||||
|
||||
def make_stream_desc_file(stream_hash):
|
||||
logging.debug("creating the stream descriptor file")
|
||||
log.debug("creating the stream descriptor file")
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
|
||||
|
||||
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaver, LBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileStatusReport import LBRYFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
@@ -18,19 +18,18 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, upload_allowed,
|
||||
file_name=None):
|
||||
LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, download_directory,
|
||||
upload_allowed)
|
||||
upload_allowed, file_name)
|
||||
self.rowid = rowid
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.file_name = file_name
|
||||
self.file_handle = None
|
||||
self.saving_status = False
|
||||
|
||||
def restore(self):
|
||||
d = self.lbry_file_manager.get_lbry_file_status(self.stream_hash)
|
||||
d = self.lbry_file_manager.get_lbry_file_status(self)
|
||||
|
||||
def restore_status(status):
|
||||
if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
|
||||
@@ -44,12 +43,12 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
d.addCallback(restore_status)
|
||||
return d
|
||||
|
||||
def stop(self, change_status=True):
|
||||
def stop(self, err=None, change_status=True):
|
||||
|
||||
def set_saving_status_done():
|
||||
self.saving_status = False
|
||||
|
||||
d = LBRYFileDownloader.stop(self) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
d = LBRYFileDownloader.stop(self, err=err) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
if change_status is True:
|
||||
self.saving_status = True
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
@@ -105,7 +104,7 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
s = ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
s = ManagedLBRYFileDownloader.STATUS_RUNNING
|
||||
return self.lbry_file_manager.change_lbry_file_status(self.stream_hash, s)
|
||||
return self.lbry_file_manager.change_lbry_file_status(self, s)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
@@ -117,33 +116,31 @@ class ManagedLBRYFileDownloaderFactory(object):
|
||||
def __init__(self, lbry_file_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None):
|
||||
data_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, sd_validator.raw_info)
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.lbry_file_manager.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash,
|
||||
metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(lambda stream_hash: self.lbry_file_manager.add_lbry_file(stream_hash,
|
||||
payment_rate_manager,
|
||||
data_rate,
|
||||
upload_allowed))
|
||||
upload_allowed,
|
||||
download_directory=download_directory))
|
||||
return d
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save the file to disk"
|
||||
@@ -3,84 +3,84 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
|
||||
import leveldb
|
||||
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamDescriptorValidator
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
from twisted.internet.task import LoopingCall
|
||||
from twisted.enterprise import adbapi
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from twisted.internet import threads, defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileManager(object):
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
|
||||
"""
|
||||
SETTING = "s"
|
||||
LBRYFILE_STATUS = "t"
|
||||
LBRYFILE_OPTIONS = "o"
|
||||
|
||||
def __init__(self, session, stream_info_manager, sd_identifier):
|
||||
def __init__(self, session, stream_info_manager, sd_identifier, delete_data=False, download_directory=None):
|
||||
self.session = session
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.db = None
|
||||
self.download_directory = os.getcwd()
|
||||
self.sql_db = None
|
||||
# self.delete_data = delete_data
|
||||
# self.check_exists_loop = LoopingCall(self.check_files_exist)
|
||||
if download_directory:
|
||||
self.download_directory = download_directory
|
||||
else:
|
||||
self.download_directory = os.getcwd()
|
||||
log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
# self.check_exists_loop.start(10)
|
||||
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._add_to_sd_identifier())
|
||||
d.addCallback(lambda _: self._start_lbry_files())
|
||||
return d
|
||||
|
||||
def get_all_lbry_file_stream_hashes_and_options(self):
|
||||
d = threads.deferToThread(self._get_all_lbry_file_stream_hashes)
|
||||
# def check_files_exist(self):
|
||||
# def _disp(deleted_files):
|
||||
# if deleted_files[0][0]:
|
||||
# for file in bad_files:
|
||||
# log.info("[" + str(datetime.now()) + "] Detected " + file.file_name + " was deleted, removing from file manager")
|
||||
#
|
||||
# def _delete_stream_data(lbry_file):
|
||||
# s_h = lbry_file.stream_hash
|
||||
# d = self.get_count_for_stream_hash(s_h)
|
||||
# # TODO: could possibly be a timing issue here
|
||||
# d.addCallback(lambda c: self.stream_info_manager.delete_stream(s_h) if c == 0 else True)
|
||||
# return d
|
||||
#
|
||||
# bad_files = [lbry_file for lbry_file in self.lbry_files
|
||||
# if lbry_file.completed == True and
|
||||
# os.path.isfile(os.path.join(self.download_directory, lbry_file.file_name)) == False]
|
||||
# d = defer.DeferredList([self.delete_lbry_file(lbry_file) for lbry_file in bad_files], consumeErrors=True)
|
||||
# d.addCallback(lambda files: _disp(files) if len(files) else defer.succeed(None))
|
||||
#
|
||||
# if self.delete_data:
|
||||
# d2 = defer.DeferredList([_delete_stream_data(lbry_file) for lbry_file in bad_files], consumeErrors=True)
|
||||
|
||||
def get_options(stream_hashes):
|
||||
ds = []
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self._get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def get_options_for_stream_hash(stream_hash):
|
||||
d = self.get_lbry_file_options(stream_hash)
|
||||
d.addCallback(lambda options: (stream_hash, options))
|
||||
return d
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)
|
||||
|
||||
for stream_hash in stream_hashes:
|
||||
ds.append(get_options_for_stream_hash(stream_hash))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(lambda results: [r[1] for r in results if r[0]])
|
||||
return dl
|
||||
|
||||
d.addCallback(get_options)
|
||||
return d
|
||||
|
||||
def get_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_status, stream_hash)
|
||||
|
||||
def save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
return threads.deferToThread(self._save_lbry_file_options, stream_hash, blob_data_rate)
|
||||
|
||||
def get_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_options, stream_hash)
|
||||
|
||||
def delete_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_options, stream_hash)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, stream_hash, new_rate):
|
||||
return threads.deferToThread(self._set_lbry_file_payment_rate, stream_hash, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, stream_hash, status):
|
||||
logging.debug("Changing status of %s to %s", stream_hash, status)
|
||||
return threads.deferToThread(self._change_file_status, stream_hash, status)
|
||||
|
||||
def delete_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_status, stream_hash)
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self._change_file_status(lbry_file.rowid, status)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -98,52 +98,61 @@ class LBRYFileManager(object):
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedLBRYFileDownloaderFactory(self)
|
||||
self.sd_identifier.add_stream_info_validator(LBRYFileStreamType, LBRYFileStreamDescriptorValidator)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
|
||||
|
||||
def _start_lbry_files(self):
|
||||
|
||||
def set_options_and_restore(stream_hash, options):
|
||||
def set_options_and_restore(rowid, stream_hash, options):
|
||||
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
d = self.add_lbry_file(stream_hash, payment_rate_manager, blob_data_rate=options[0])
|
||||
d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate=options)
|
||||
d.addCallback(lambda downloader: downloader.restore())
|
||||
return d
|
||||
|
||||
def log_error(err):
|
||||
logging.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
|
||||
def start_lbry_files(stream_hashes_and_options):
|
||||
for stream_hash, options in stream_hashes_and_options:
|
||||
d = set_options_and_restore(stream_hash, options)
|
||||
def start_lbry_files(lbry_files_and_options):
|
||||
for rowid, stream_hash, options in lbry_files_and_options:
|
||||
d = set_options_and_restore(rowid, stream_hash, options)
|
||||
d.addErrback(log_error)
|
||||
return True
|
||||
|
||||
d = self.get_all_lbry_file_stream_hashes_and_options()
|
||||
d = self._get_all_lbry_files()
|
||||
d.addCallback(start_lbry_files)
|
||||
return d
|
||||
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True):
|
||||
def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
|
||||
download_directory=None):
|
||||
if not download_directory:
|
||||
download_directory = self.download_directory
|
||||
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
|
||||
lbry_file_downloader = ManagedLBRYFileDownloader(stream_hash, self.session.peer_finder,
|
||||
self.session.rate_limiter, self.session.blob_manager,
|
||||
lbry_file_downloader = ManagedLBRYFileDownloader(rowid, stream_hash,
|
||||
self.session.peer_finder,
|
||||
self.session.rate_limiter,
|
||||
self.session.blob_manager,
|
||||
self.stream_info_manager, self,
|
||||
payment_rate_manager, self.session.wallet,
|
||||
self.download_directory,
|
||||
download_directory,
|
||||
upload_allowed)
|
||||
self.lbry_files.append(lbry_file_downloader)
|
||||
d = self.save_lbry_file_options(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda _: lbry_file_downloader.set_stream_info())
|
||||
d = lbry_file_downloader.set_stream_info()
|
||||
d.addCallback(lambda _: lbry_file_downloader)
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, stream_hash):
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True, download_directory=None):
|
||||
d = self._save_lbry_file(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate, upload_allowed, download_directory))
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, lbry_file):
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
if l == lbry_file:
|
||||
lbry_file = l
|
||||
break
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
@@ -162,26 +171,20 @@ class LBRYFileManager(object):
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
d.addCallback(lambda _: remove_from_list())
|
||||
d.addCallback(lambda _: self.delete_lbry_file_options(stream_hash))
|
||||
d.addCallback(lambda _: self.delete_lbry_file_status(stream_hash))
|
||||
d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
|
||||
return d
|
||||
|
||||
def toggle_lbry_file_running(self, stream_hash):
|
||||
def toggle_lbry_file_running(self, lbry_file):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
if l == lbry_file:
|
||||
return l.toggle_running()
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
|
||||
def get_stream_hash_from_name(self, lbry_file_name):
|
||||
for l in self.lbry_files:
|
||||
if l.file_name == lbry_file_name:
|
||||
return l.stream_hash
|
||||
return None
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def stop(self):
|
||||
# self.check_exists_loop.stop()
|
||||
|
||||
ds = []
|
||||
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
@@ -206,50 +209,62 @@ class LBRYFileManager(object):
|
||||
dl.addCallback(lambda _: close_db())
|
||||
return dl
|
||||
|
||||
def get_count_for_stream_hash(self, stream_hash):
|
||||
return self._get_count_for_stream_hash(stream_hash)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.session.db_dir, "lbryfiles.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
|
||||
check_same_thread=False)
|
||||
return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
|
||||
" blob_data_rate real, " +
|
||||
" status text," +
|
||||
" stream_hash text,"
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
def _save_payment_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db.Put(json.dumps((self.SETTING, rate_type)), json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(json.dumps((self.SETTING, rate_type)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _save_lbry_file(self, stream_hash, data_payment_rate):
|
||||
def do_save(db_transaction):
|
||||
db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
(data_payment_rate, ManagedLBRYFileDownloader.STATUS_STOPPED,
|
||||
stream_hash))
|
||||
return db_transaction.lastrowid
|
||||
return self.sql_db.runInteraction(do_save)
|
||||
|
||||
def _save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((blob_data_rate,)),
|
||||
sync=True)
|
||||
@rerun_if_locked
|
||||
def _delete_lbry_file_options(self, rowid):
|
||||
return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
|
||||
def _get_lbry_file_options(self, stream_hash):
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((self.LBRYFILE_OPTIONS, stream_hash))))
|
||||
except KeyError:
|
||||
return None, None
|
||||
@rerun_if_locked
|
||||
def _set_lbry_file_payment_rate(self, rowid, new_rate):
|
||||
return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
|
||||
(new_rate, rowid))
|
||||
|
||||
def _delete_lbry_file_options(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_all_lbry_files(self):
|
||||
d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
|
||||
return d
|
||||
|
||||
def _set_lbry_file_payment_rate(self, stream_hash, new_rate):
|
||||
@rerun_if_locked
|
||||
def _change_file_status(self, rowid, new_status):
|
||||
return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
|
||||
(new_status, rowid))
|
||||
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((new_rate, )), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_lbry_file_status(self, rowid):
|
||||
d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else ManagedLBRYFileDownloader.STATUS_STOPPED)
|
||||
return d
|
||||
|
||||
def _get_all_lbry_file_stream_hashes(self):
|
||||
hashes = []
|
||||
for k, v in self.db.RangeIter():
|
||||
key_type, stream_hash = json.loads(k)
|
||||
if key_type == self.LBRYFILE_STATUS:
|
||||
hashes.append(stream_hash)
|
||||
return hashes
|
||||
|
||||
def _change_file_status(self, stream_hash, new_status):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_STATUS, stream_hash)), new_status, sync=True)
|
||||
|
||||
def _get_lbry_file_status(self, stream_hash):
|
||||
try:
|
||||
return self.db.Get(json.dumps((self.LBRYFILE_STATUS, stream_hash)))
|
||||
except KeyError:
|
||||
return ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
|
||||
def _delete_lbry_file_status(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_STATUS, stream_hash)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_count_for_stream_hash(self, stream_hash):
|
||||
return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
@@ -1,20 +1,20 @@
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info, LiveStreamType, LBRYLiveStreamDescriptorValidator
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.lbrylive.LiveBlob import LiveStreamBlobMaker
|
||||
from lbrynet.lbrylive.PaymentRateManager import BaseLiveStreamPaymentRateManager
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_pass_phrase
|
||||
from Crypto import Random
|
||||
import binascii
|
||||
import logging
|
||||
from lbrynet.conf import CRYPTSD_FILE_EXTENSION
|
||||
from lbrynet.conf import MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.lbrylive.client.LiveStreamDownloader import FullLiveStreamDownloaderFactory
|
||||
from twisted.internet import interfaces, defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamCreator(CryptStreamCreator):
|
||||
def __init__(self, blob_manager, stream_info_manager, name=None, key=None, iv_generator=None,
|
||||
delete_after_num=None, secret_pass_phrase=None):
|
||||
@@ -33,8 +33,8 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
return d
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("In blob_finished")
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("In blob_finished")
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
sig_hash = get_lbry_hash_obj()
|
||||
sig_hash.update(self.stream_hash)
|
||||
if blob_info.length != 0:
|
||||
@@ -51,11 +51,11 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
d = self.stream_info_manager.add_blobs_to_stream(self.stream_hash, [blob_info])
|
||||
|
||||
def log_add_error(err):
|
||||
logging.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
log.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_add_error)
|
||||
logging.debug("returning from blob_finished")
|
||||
log.debug("returning from blob_finished")
|
||||
return d
|
||||
|
||||
def setup(self):
|
||||
@@ -173,17 +173,4 @@ class StdinStreamProducer(object):
|
||||
self.consumer.write(data)
|
||||
|
||||
def childConnectionLost(self, fd, reason):
|
||||
self.stopProducing()
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(session, stream_info_manager, sd_identifier):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
BaseLiveStreamPaymentRateManager(
|
||||
MIN_BLOB_INFO_PAYMENT_RATE
|
||||
))
|
||||
sd_identifier.add_stream_info_validator(LiveStreamType, LBRYLiveStreamDescriptorValidator)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
self.stopProducing()
|
||||
@@ -1,11 +1,16 @@
|
||||
import time
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
from twisted.enterprise import adbapi
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
import sqlite3
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
@@ -14,26 +19,22 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
def __init__(self, db_dir, hash_announcer):
|
||||
DHTHashSupplier.__init__(self, hash_announcer)
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, pub_key, file_name, key, blobs):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
d = self._store_stream(stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
|
||||
def save_blobs():
|
||||
return self.add_blobs_to_stream(stream_hash, blobs)
|
||||
@@ -48,169 +49,229 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
log.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_streams_to_announce, next_announce_time)
|
||||
return self._get_streams_to_announce(next_announce_time)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_desc.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "live_stream.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists live_streams (" +
|
||||
" stream_hash text primary key, " +
|
||||
" public_key text, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" next_announce_time real" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" revision integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" signature text, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from live_streams where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, public_key, name, key, next_announce_time=None):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, public_key, key, name, next_announce_time))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
d = self.db_conn.runQuery("select public_key, key, stream_name from live_streams where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_streams_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
stream_hashes = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for stream_hash, stream_info in self.stream_info_db.RangeIter():
|
||||
public_key, key, name, announce_time = json.loads(stream_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)))
|
||||
stream_hashes.append(stream_hash)
|
||||
self.stream_info_db.Write(batch, sync=True)
|
||||
return stream_hashes
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select stream_hash from live_streams where" +
|
||||
" (next_announce_time is null or next_announce_time < ?) " +
|
||||
" and stream_hash is not null", (timestamp, ))
|
||||
s_hs = [s_h for s_h, in r.fetchall()]
|
||||
transaction.execute("update live_streams set next_announce_time = ? where " +
|
||||
" (next_announce_time is null or next_announce_time < ?)",
|
||||
(next_announce_time, timestamp))
|
||||
return s_hs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
d = self.db_conn.runQuery("select position from live_stream_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, revision, iv, length, signature = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, revision, iv, length, signature))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, revision, iv, length, signature from live_stream_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.revision,
|
||||
blob_info.iv,
|
||||
blob_info.length,
|
||||
blob_info.signature)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.revision, blob_info.iv, blob_info.length,
|
||||
blob_info.signature))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
d = self.db_conn.runQuery("select stream_hash from live_stream_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
return self.db_conn.runQuery("insert into live_stream_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from live_stream_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempLiveStreamMetadataManager(DHTHashSupplier):
|
||||
|
||||
@@ -2,17 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.interfaces import IStreamDescriptorValidator
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LiveStreamType = "lbrylive"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
public_key = sd_info['public_key']
|
||||
key = sd_info['key']
|
||||
@@ -30,7 +33,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
iv = blob['iv']
|
||||
signature = blob['signature']
|
||||
crypt_blobs.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, public_key, hex_stream_name,
|
||||
key, crypt_blobs)
|
||||
|
||||
@@ -88,7 +91,7 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
public_key = self.raw_info['public_key']
|
||||
key = self.raw_info['key']
|
||||
@@ -98,7 +101,7 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
h.update(public_key)
|
||||
h.update(key)
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
blobs = self.raw_info['blobs']
|
||||
|
||||
def check_blob_signatures():
|
||||
@@ -121,11 +124,14 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
hashsum.update(iv)
|
||||
hashsum.update(str(length))
|
||||
if not verify_signature(hashsum.digest(), signature, public_key):
|
||||
raise ValueError("Invalid signature in stream descriptor")
|
||||
raise InvalidStreamDescriptorError("Invalid signature in stream descriptor")
|
||||
|
||||
return threads.deferToThread(check_blob_signatures)
|
||||
|
||||
def info_to_show(self):
|
||||
info = []
|
||||
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
return None
|
||||
@@ -1,5 +1,5 @@
|
||||
import binascii
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from zope.interface import implements
|
||||
from lbrynet.lbrylive.client.LiveStreamMetadataHandler import LiveStreamMetadataHandler
|
||||
@@ -9,6 +9,7 @@ from lbrynet.lbrylive.StreamDescriptor import save_sd_info
|
||||
from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
|
||||
from twisted.internet import defer, threads # , process
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType
|
||||
|
||||
|
||||
class LiveStreamDownloader(CryptStreamDownloader):
|
||||
@@ -85,9 +86,9 @@ class FullLiveStreamDownloader(LiveStreamDownloader):
|
||||
d.addCallback(lambda _: set_file_name_if_unset())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
d = self._close_file()
|
||||
d.addBoth(lambda _: LiveStreamDownloader.stop(self))
|
||||
d.addBoth(lambda _: LiveStreamDownloader.stop(self, err))
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
@@ -138,34 +139,24 @@ class FullLiveStreamDownloaderFactory(object):
|
||||
self.wallet = wallet
|
||||
self.default_payment_rate_manager = default_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for metadata (None means use application default)",
|
||||
"metadata payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
def make_downloader(self, metadata, options, payment_rate_manager):
|
||||
# TODO: check options for payment rate manager parameters
|
||||
payment_rate_manager = LiveStreamPaymentRateManager(self.default_payment_rate_manager,
|
||||
payment_rate_manager)
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter,
|
||||
@@ -177,4 +168,15 @@ class FullLiveStreamDownloaderFactory(object):
|
||||
return d
|
||||
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
return d
|
||||
|
||||
|
||||
def add_full_live_stream_downloader_to_sd_identifier(session, stream_info_manager, sd_identifier,
|
||||
base_live_stream_payment_rate_manager):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
base_live_stream_payment_rate_manager)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
@@ -9,7 +9,10 @@ from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from lbrynet.interfaces import IRequestCreator, IMetadataHandler
|
||||
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
|
||||
from lbrynet.core.Error import NoResponseError
|
||||
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamMetadataHandler(object):
|
||||
@@ -79,7 +82,7 @@ class LiveStreamMetadataHandler(object):
|
||||
r = None
|
||||
if self._finished_discovery() is False:
|
||||
r = self.stream_hash
|
||||
logging.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
log.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -101,7 +104,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
|
||||
@@ -134,7 +137,7 @@ class LiveStreamMetadataHandler(object):
|
||||
further_blobs_request['count'] = count
|
||||
else:
|
||||
further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST
|
||||
logging.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
r_dict = {'further_blobs': further_blobs_request}
|
||||
response_identifier = 'further_blobs'
|
||||
request = ClientPaidRequest(r_dict, response_identifier, further_blobs_request['count'])
|
||||
@@ -142,7 +145,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return None
|
||||
|
||||
def _get_discovery_params(self):
|
||||
logging.debug("In _get_discovery_params")
|
||||
log.debug("In _get_discovery_params")
|
||||
stream_position = self.download_manager.stream_position()
|
||||
blobs = self.download_manager.blobs
|
||||
if blobs:
|
||||
@@ -153,7 +156,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if final_blob_num is not None:
|
||||
last_blob_num = final_blob_num
|
||||
if self.download_whole is False:
|
||||
logging.debug("download_whole is False")
|
||||
log.debug("download_whole is False")
|
||||
if final_blob_num is not None:
|
||||
for i in xrange(stream_position, final_blob_num + 1):
|
||||
if not i in blobs:
|
||||
@@ -171,7 +174,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, 'end', -1 * self.max_before_skip_ahead
|
||||
else:
|
||||
return self.stream_hash, None, 'end', -1 * self.max_before_skip_ahead
|
||||
logging.debug("download_whole is True")
|
||||
log.debug("download_whole is True")
|
||||
beginning = None
|
||||
end = None
|
||||
for i in xrange(stream_position, last_blob_num + 1):
|
||||
@@ -187,17 +190,17 @@ class LiveStreamMetadataHandler(object):
|
||||
break
|
||||
if beginning is None:
|
||||
if final_blob_num is not None:
|
||||
logging.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
log.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
return None
|
||||
else:
|
||||
logging.debug("Discovery is not finished. final blob num is unknown.")
|
||||
log.debug("Discovery is not finished. final blob num is unknown.")
|
||||
if last_blob_num != -1:
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, None, None
|
||||
else:
|
||||
return self.stream_hash, 'beginning', None, None
|
||||
else:
|
||||
logging.info("Discovery is not finished. Not all blobs are known.")
|
||||
log.info("Discovery is not finished. Not all blobs are known.")
|
||||
return self.stream_hash, beginning, end, None
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
@@ -244,7 +247,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
logging.info("Rate offer has been rejected by %s", str(peer))
|
||||
log.info("Rate offer has been rejected by %s", str(peer))
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
@@ -263,8 +266,8 @@ class LiveStreamMetadataHandler(object):
|
||||
if not 'blob_infos' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_infos'")
|
||||
raw_blob_infos = response['blob_infos']
|
||||
logging.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
logging.debug("blobs: %s", str(raw_blob_infos))
|
||||
log.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
log.debug("blobs: %s", str(raw_blob_infos))
|
||||
for raw_blob_info in raw_blob_infos:
|
||||
length = raw_blob_info['length']
|
||||
if length != 0:
|
||||
@@ -276,10 +279,10 @@ class LiveStreamMetadataHandler(object):
|
||||
iv = raw_blob_info['iv']
|
||||
signature = raw_blob_info['signature']
|
||||
blob_info = LiveBlobInfo(blob_hash, num, length, iv, revision, signature)
|
||||
logging.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
log.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
if self._verify_blob(blob_info):
|
||||
if blob_hash is None:
|
||||
logging.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
log.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
self._final_blob_num = num - 1
|
||||
else:
|
||||
blob_infos.append(blob_info)
|
||||
@@ -289,7 +292,7 @@ class LiveStreamMetadataHandler(object):
|
||||
|
||||
def add_blobs_to_download_manager():
|
||||
blob_nums = [b.blob_num for b in blob_infos]
|
||||
logging.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
log.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
self.download_manager.add_blobs_to_download(blob_infos)
|
||||
|
||||
d.addCallback(lambda _: add_blobs_to_download_manager())
|
||||
@@ -308,12 +311,12 @@ class LiveStreamMetadataHandler(object):
|
||||
return d
|
||||
|
||||
def _verify_blob(self, blob):
|
||||
logging.debug("Got an unverified blob to check:")
|
||||
logging.debug("blob_hash: %s", blob.blob_hash)
|
||||
logging.debug("blob_num: %s", str(blob.blob_num))
|
||||
logging.debug("revision: %s", str(blob.revision))
|
||||
logging.debug("iv: %s", blob.iv)
|
||||
logging.debug("length: %s", str(blob.length))
|
||||
log.debug("Got an unverified blob to check:")
|
||||
log.debug("blob_hash: %s", blob.blob_hash)
|
||||
log.debug("blob_num: %s", str(blob.blob_num))
|
||||
log.debug("revision: %s", str(blob.revision))
|
||||
log.debug("iv: %s", blob.iv)
|
||||
log.debug("length: %s", str(blob.length))
|
||||
hashsum = get_lbry_hash_obj()
|
||||
hashsum.update(self.stream_hash)
|
||||
if blob.length != 0:
|
||||
@@ -322,12 +325,12 @@ class LiveStreamMetadataHandler(object):
|
||||
hashsum.update(str(blob.revision))
|
||||
hashsum.update(blob.iv)
|
||||
hashsum.update(str(blob.length))
|
||||
logging.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
log.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
if verify_signature(hashsum.digest(), blob.signature, self.stream_pub_key):
|
||||
logging.debug("Blob info is valid")
|
||||
log.debug("Blob info is valid")
|
||||
return True
|
||||
else:
|
||||
logging.debug("The blob info is invalid")
|
||||
log.debug("The blob info is invalid")
|
||||
return False
|
||||
|
||||
def _request_failed(self, reason, peer):
|
||||
@@ -335,8 +338,9 @@ class LiveStreamMetadataHandler(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
self._update_local_score(peer, -5.0)
|
||||
peer.update_score(-10.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
@@ -0,0 +1,73 @@
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType, LBRYLiveStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(sd_identifier, base_live_stream_payment_rate_manager):
|
||||
sd_identifier.add_stream_type(LiveStreamType, LBRYLiveStreamDescriptorValidator,
|
||||
LiveStreamOptions(base_live_stream_payment_rate_manager))
|
||||
|
||||
|
||||
class LiveStreamOptions(object):
|
||||
def __init__(self, base_live_stream_payment_rate_manager):
|
||||
self.base_live_stream_prm = base_live_stream_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.min_blob_data_payment_rate
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for metadata",
|
||||
"metadata payment rate",
|
||||
None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(True,
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"Allow reuploading"),
|
||||
DownloadOptionChoice(False,
|
||||
"Disallow reuploading data downloaded for this file",
|
||||
"Disallow reuploading")
|
||||
],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -3,6 +3,9 @@ from lbrynet.core.client.StreamProgressManager import StreamProgressManager
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False,
|
||||
download_whole=True, max_before_skip_ahead=5):
|
||||
@@ -52,9 +55,9 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
return
|
||||
|
||||
blobs = self.download_manager.blobs
|
||||
logging.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
log.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
if blobs:
|
||||
logging.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
log.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
if self.outputting_d is None:
|
||||
self.outputting_d = defer.Deferred()
|
||||
|
||||
@@ -71,15 +74,15 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
reactor.callLater(0, self._output_loop)
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(current_blob_num))
|
||||
log.info("Outputting blob %s", str(current_blob_num))
|
||||
self.provided_blob_nums.append(current_blob_num)
|
||||
d = self.download_manager.handle_blob(current_blob_num)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
elif blobs and max(blobs.iterkeys()) > self.last_blob_outputted + self.max_before_skip_ahead - 1:
|
||||
self.last_blob_outputted += 1
|
||||
logging.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
log.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
self._finished_with_blob(current_blob_num)
|
||||
reactor.callLater(0, self._output_loop)
|
||||
else:
|
||||
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfoQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -54,7 +57,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
further_blobs_request = queries[self.query_identifiers[1]]
|
||||
logging.debug("Received the client's request for additional blob information")
|
||||
log.debug("Received the client's request for additional blob information")
|
||||
|
||||
if self.blob_info_payment_rate is None:
|
||||
response['further_blobs'] = {'error': 'RATE_UNSET'}
|
||||
@@ -62,7 +65,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
def count_and_charge(blob_infos):
|
||||
if len(blob_infos) != 0:
|
||||
logging.info("Responding with %s infos", str(len(blob_infos)))
|
||||
log.info("Responding with %s infos", str(len(blob_infos)))
|
||||
expected_payment = 1.0 * len(blob_infos) * self.blob_info_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_crypt_blob_infos', len(blob_infos))
|
||||
|
||||
@@ -1,60 +1,176 @@
|
||||
from twisted.protocols import basic
|
||||
from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConsoleControl(basic.LineReceiver):
|
||||
from os import linesep as delimiter
|
||||
|
||||
def __init__(self, control_handlers):
|
||||
self.control_handlers = {}
|
||||
self.categories = {}
|
||||
categories = set([category for category, handler in control_handlers])
|
||||
prompt_number = 0
|
||||
for category in categories:
|
||||
self.categories[prompt_number] = category
|
||||
for handler in [handler for cat, handler in control_handlers if cat == category]:
|
||||
self.control_handlers[prompt_number] = handler
|
||||
prompt_number += 1
|
||||
def __init__(self):
|
||||
self.connected = False
|
||||
self.buffer = []
|
||||
self.command_handlers = {}
|
||||
self.current_handler = None
|
||||
|
||||
def start(self, command_handlers):
|
||||
self.command_handlers = {h.command: h for h in command_handlers}
|
||||
self.current_handler = None
|
||||
self.send_initial_prompt()
|
||||
return defer.succeed(True)
|
||||
|
||||
def connectionMade(self):
|
||||
self.connected = True
|
||||
if self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def send_initial_prompt(self):
|
||||
self.sendLine("")
|
||||
self.sendLine("You should have received 1000 LBC the first time you ran\n"
|
||||
"this program. If you did not, let us know! But first give\n"
|
||||
"them a moment to show up. The block time is currently 30\n"
|
||||
"seconds so they should show up within a couple of minutes.\n\n"
|
||||
"Welcome to lbrynet-console!")
|
||||
self.sendLine("")
|
||||
self.sendLine("Enter a command. Try 'get wonderfullife' or 'help' to see more options.")
|
||||
self.show_prompt()
|
||||
|
||||
def send(self, s):
|
||||
self.transport.write(s)
|
||||
|
||||
def write(self, s):
|
||||
if self.connected is False:
|
||||
self.buffer.append(s)
|
||||
else:
|
||||
self.send(s)
|
||||
|
||||
def flush(self):
|
||||
if self.connected is True and self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def show_prompt(self):
|
||||
self.send("> ")
|
||||
|
||||
def show_quick_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
showed_help_all = False
|
||||
sorted_handlers = sorted(self.command_handlers.items(), key=lambda x: x[0])
|
||||
sorted_handlers = sorted(sorted_handlers, key=lambda x: x[1].priority, reverse=True)
|
||||
for command, handler in sorted_handlers:
|
||||
if handler.priority > 0:
|
||||
if showed_help_all is False and handler.priority < 10:
|
||||
self.sendLine("help-all - Show the full list of available commands")
|
||||
showed_help_all = True
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def show_full_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
for command, handler in sorted(self.command_handlers.items(), key=lambda x: x[0]):
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def handler_done(self):
|
||||
self.current_handler = None
|
||||
self.show_prompt()
|
||||
|
||||
def handler_failed(self, err):
|
||||
self.current_handler = None
|
||||
self.sendLine("An error occurred:")
|
||||
self.sendLine(err.getTraceback())
|
||||
self.show_prompt()
|
||||
|
||||
def lineReceived(self, line):
|
||||
|
||||
def show_response(response):
|
||||
if response is not None:
|
||||
self.sendLine(response)
|
||||
|
||||
def show_error(err):
|
||||
self.sendLine(err.getTraceback())
|
||||
|
||||
if not self.command_handlers:
|
||||
return
|
||||
if self.current_handler is None:
|
||||
words = line.split()
|
||||
if len(words) == 0:
|
||||
self.show_prompt()
|
||||
return
|
||||
command, args = words[0], words[1:]
|
||||
if command == "help":
|
||||
if len(args) == 0:
|
||||
self.show_quick_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] in self.command_handlers:
|
||||
self.sendLine(self.command_handlers[args[0]].full_help)
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] == "help-all":
|
||||
self.sendLine("Show the full list of available commands!")
|
||||
self.show_prompt()
|
||||
return
|
||||
self.sendLine("Can't help you with '%s'. Sorry!" % args[0])
|
||||
self.show_prompt()
|
||||
return
|
||||
elif command == "help-all":
|
||||
self.show_full_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if command in self.command_handlers:
|
||||
command_handler = self.command_handlers[command]
|
||||
else:
|
||||
candidates = [k for k in self.command_handlers.keys() if k.startswith(command)]
|
||||
if len(candidates) == 0:
|
||||
self.sendLine("Unknown command. Type 'help' for a list of commands.")
|
||||
self.show_prompt()
|
||||
return
|
||||
if len(candidates) >= 2:
|
||||
l = "Ambiguous command. Matches: "
|
||||
for candidate in candidates:
|
||||
l += candidate
|
||||
l += ", "
|
||||
l = l[:-2]
|
||||
l += l
|
||||
self.sendLine(l)
|
||||
self.show_prompt()
|
||||
return
|
||||
else:
|
||||
command_handler = self.command_handlers[candidates[0]]
|
||||
try:
|
||||
num = int(line)
|
||||
except ValueError:
|
||||
num = None
|
||||
if num in self.control_handlers:
|
||||
self.current_handler = self.control_handlers[num].get_handler()
|
||||
line = None
|
||||
if self.current_handler is not None:
|
||||
try:
|
||||
r = self.current_handler.handle_line(line)
|
||||
done, ds = r[0], [d for d in r[1:] if d is not None]
|
||||
except Exception as e:
|
||||
done = True
|
||||
ds = [defer.fail(e)]
|
||||
if done is True:
|
||||
self.current_handler = command_handler.get_handler(self)
|
||||
except:
|
||||
self.current_handler = None
|
||||
map(lambda d: d.addCallbacks(show_response, show_error), ds)
|
||||
if self.current_handler is None:
|
||||
self.show_prompt()
|
||||
|
||||
def show_prompt(self):
|
||||
self.sendLine("Options:")
|
||||
for num, handler in self.control_handlers.iteritems():
|
||||
if num in self.categories:
|
||||
self.sendLine("")
|
||||
self.sendLine(self.categories[num])
|
||||
self.sendLine("")
|
||||
self.sendLine("[" + str(num) + "] " + handler.get_prompt_description())
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
try:
|
||||
self.current_handler.start(*args)
|
||||
except TypeError:
|
||||
self.current_handler = None
|
||||
self.sendLine("Invalid arguments. Type 'help <command>' for the argument list.")
|
||||
import traceback
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
except:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
self.current_handler.finished_deferred.addCallbacks(lambda _: self.handler_done(),
|
||||
self.handler_failed)
|
||||
else:
|
||||
try:
|
||||
self.current_handler.handle_line(line)
|
||||
except Exception as e:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
@@ -1,13 +1,22 @@
|
||||
import logging
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
import os.path
|
||||
import argparse
|
||||
import requests
|
||||
import locale
|
||||
import sys
|
||||
import webbrowser
|
||||
|
||||
if sys.platform == "darwin":
|
||||
from appdirs import user_data_dir
|
||||
from yapsy.PluginManager import PluginManager
|
||||
from twisted.internet import defer, threads, stdio, task
|
||||
from twisted.internet import defer, threads, stdio, task, error
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
|
||||
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
|
||||
from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE # , MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, API_CONNECTION_STRING # , MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
@@ -15,11 +24,12 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileOpenerFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager
|
||||
#from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import LBRYFileStatusFactory, DeleteLBRYFileChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ToggleLBRYFileRunningChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory
|
||||
@@ -28,17 +38,23 @@ from lbrynet.lbrynet_console.ControlHandlers import ShowPublishedSDHashesChooser
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CreatePlainStreamDescriptorChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowLBRYFileStreamHashChooserFactory, AddStreamFromHashFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import AddStreamFromSDFactory, AddStreamFromLBRYcrdNameFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory, GetNewWalletAddressFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowServerStatusFactory, ModifyServerSettingsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory
|
||||
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory, StatusFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory, PublishFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import BlockchainStatusFactory
|
||||
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet, LBRYumWallet
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
alert = logging.getLogger("lbryalert." + __name__)
|
||||
|
||||
|
||||
class LBRYConsole():
|
||||
"""A class which can upload and download file streams to and from the network"""
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes, control_class, wallet_type, lbrycrd_rpc_port,
|
||||
use_upnp, conf_dir, data_dir):
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes, fake_wallet,
|
||||
lbrycrd_conf, lbrycrd_dir, use_upnp, data_dir, created_data_dir,
|
||||
lbrycrdd_path):
|
||||
"""
|
||||
@param peer_port: the network port on which to listen for peers
|
||||
|
||||
@@ -49,34 +65,49 @@ class LBRYConsole():
|
||||
self.peer_port = peer_port
|
||||
self.dht_node_port = dht_node_port
|
||||
self.known_dht_nodes = known_dht_nodes
|
||||
self.wallet_type = wallet_type
|
||||
self.wallet_rpc_port = lbrycrd_rpc_port
|
||||
self.fake_wallet = fake_wallet
|
||||
self.lbrycrd_conf = lbrycrd_conf
|
||||
self.lbrycrd_dir = lbrycrd_dir
|
||||
if not self.lbrycrd_dir:
|
||||
if sys.platform == "darwin":
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), "Library/Application Support/lbrycrd")
|
||||
else:
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
|
||||
if not self.lbrycrd_conf:
|
||||
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
|
||||
self.lbrycrdd_path = lbrycrdd_path
|
||||
self.use_upnp = use_upnp
|
||||
self.lbry_server_port = None
|
||||
self.control_class = control_class
|
||||
self.session = None
|
||||
self.lbry_file_metadata_manager = None
|
||||
self.lbry_file_manager = None
|
||||
self.conf_dir = conf_dir
|
||||
self.data_dir = data_dir
|
||||
self.db_dir = data_dir
|
||||
self.current_db_revision = 1
|
||||
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
|
||||
self.created_data_dir = created_data_dir
|
||||
self.plugin_manager = PluginManager()
|
||||
self.plugin_manager.setPluginPlaces([
|
||||
os.path.join(self.conf_dir, "plugins"),
|
||||
os.path.join(self.db_dir, "plugins"),
|
||||
os.path.join(os.path.dirname(__file__), "plugins"),
|
||||
])
|
||||
self.control_handlers = []
|
||||
self.command_handlers = []
|
||||
self.query_handlers = {}
|
||||
|
||||
self.settings = LBRYSettings(self.conf_dir)
|
||||
self.settings = LBRYSettings(self.db_dir)
|
||||
self.blob_request_payment_rate_manager = None
|
||||
self.lbryid = None
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.plugin_objects = []
|
||||
self.db_migration_revisions = None
|
||||
|
||||
def start(self):
|
||||
"""Initialize the session and restore everything to its saved state"""
|
||||
d = threads.deferToThread(self._create_directory)
|
||||
d = self._setup_controller()
|
||||
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
|
||||
d.addCallback(lambda _: self._check_db_migration())
|
||||
d.addCallback(lambda _: self._get_settings())
|
||||
d.addCallback(lambda _: self._get_session())
|
||||
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
|
||||
d.addCallback(lambda _: self._setup_lbry_file_manager())
|
||||
d.addCallback(lambda _: self._setup_lbry_file_opener())
|
||||
d.addCallback(lambda _: self._setup_control_handlers())
|
||||
@@ -84,17 +115,30 @@ class LBRYConsole():
|
||||
d.addCallback(lambda _: self._load_plugins())
|
||||
d.addCallback(lambda _: self._setup_server())
|
||||
d.addCallback(lambda _: self._start_controller())
|
||||
d.addErrback(self._show_start_error)
|
||||
return d
|
||||
|
||||
def _show_start_error(self, error):
|
||||
print error.getTraceback()
|
||||
log.error("An error occurred during start up: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def _show_shutdown_error(self, error):
|
||||
print error.getErrorMessage()
|
||||
log.error("An error occurred during shutdown: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop the session, all currently running streams, and stop the server"""
|
||||
d = self.session.shut_down()
|
||||
d.addCallback(lambda _: self._shut_down())
|
||||
d = self._shut_down()
|
||||
if self.session is not None:
|
||||
d.addCallback(lambda _: self.session.shut_down())
|
||||
d.addErrback(self._show_shutdown_error)
|
||||
return d
|
||||
|
||||
def add_control_handlers(self, control_handlers):
|
||||
for control_handler in control_handlers:
|
||||
self.control_handlers.append(control_handler)
|
||||
self.command_handlers.append(control_handler)
|
||||
|
||||
def add_query_handlers(self, query_handlers):
|
||||
|
||||
@@ -111,13 +155,40 @@ class LBRYConsole():
|
||||
dl.addCallback(_set_query_handlers)
|
||||
return dl
|
||||
|
||||
def _create_directory(self):
|
||||
if not os.path.exists(self.conf_dir):
|
||||
os.makedirs(self.conf_dir)
|
||||
logging.debug("Created the configuration directory: %s", str(self.conf_dir))
|
||||
if not os.path.exists(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
logging.debug("Created the data directory: %s", str(self.data_dir))
|
||||
def _setup_data_directory(self):
|
||||
alert.info("Loading databases...")
|
||||
if self.created_data_dir:
|
||||
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
|
||||
db_revision.write(str(self.current_db_revision))
|
||||
db_revision.close()
|
||||
log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision")))
|
||||
if not os.path.exists(self.blobfile_dir):
|
||||
os.mkdir(self.blobfile_dir)
|
||||
log.debug("Created the blobfile directory: %s", str(self.blobfile_dir))
|
||||
|
||||
def _check_db_migration(self):
|
||||
old_revision = 0
|
||||
db_revision_file = os.path.join(self.db_dir, "db_revision")
|
||||
if os.path.exists(db_revision_file):
|
||||
old_revision = int(open(db_revision_file).read().strip())
|
||||
if old_revision < self.current_db_revision:
|
||||
from lbrynet.db_migrator import dbmigrator
|
||||
print "Upgrading your databases..."
|
||||
d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision)
|
||||
|
||||
def print_success(old_dirs):
|
||||
success_string = "Finished upgrading the databases. It is now safe to delete the"
|
||||
success_string += " following directories, if you feel like it. It won't make any"
|
||||
success_string += " difference.\nAnyway here they are: "
|
||||
for i, old_dir in enumerate(old_dirs):
|
||||
success_string += old_dir
|
||||
if i + 1 < len(old_dir):
|
||||
success_string += ", "
|
||||
print success_string
|
||||
|
||||
d.addCallback(print_success)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_settings(self):
|
||||
d = self.settings.start()
|
||||
@@ -137,28 +208,96 @@ class LBRYConsole():
|
||||
return d
|
||||
|
||||
def _get_session(self):
|
||||
d = self.settings.get_default_data_payment_rate()
|
||||
def get_default_data_rate():
|
||||
d = self.settings.get_default_data_payment_rate()
|
||||
d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE})
|
||||
return d
|
||||
|
||||
def create_session(default_data_payment_rate):
|
||||
if default_data_payment_rate is None:
|
||||
default_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
|
||||
if self.wallet_type == "lbrycrd":
|
||||
wallet = LBRYcrdWallet("rpcuser", "rpcpassword", "127.0.0.1", self.wallet_rpc_port)
|
||||
def get_wallet():
|
||||
if self.fake_wallet:
|
||||
d = defer.succeed(PTCWallet(self.db_dir))
|
||||
elif self.lbrycrdd_path is not None:
|
||||
d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir,
|
||||
wallet_conf=self.lbrycrd_conf,
|
||||
lbrycrdd_path=self.lbrycrdd_path))
|
||||
else:
|
||||
wallet = PTCWallet(self.conf_dir)
|
||||
self.session = LBRYSession(default_data_payment_rate, db_dir=self.conf_dir, lbryid=self.lbryid,
|
||||
blob_dir=self.data_dir, dht_node_port=self.dht_node_port,
|
||||
d = defer.succeed(LBRYumWallet(self.db_dir))
|
||||
d.addCallback(lambda wallet: {"wallet": wallet})
|
||||
return d
|
||||
|
||||
d1 = get_default_data_rate()
|
||||
d2 = get_wallet()
|
||||
|
||||
def combine_results(results):
|
||||
r = {}
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
r.update(result)
|
||||
return r
|
||||
|
||||
def create_session(results):
|
||||
|
||||
alert.info("Databases loaded.")
|
||||
|
||||
self.session = LBRYSession(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
|
||||
blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
|
||||
known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port,
|
||||
use_upnp=self.use_upnp, wallet=wallet)
|
||||
use_upnp=self.use_upnp, wallet=results['wallet'])
|
||||
|
||||
d.addCallback(create_session)
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
d.addCallback(lambda _: self.session.setup())
|
||||
dl.addCallback(combine_results)
|
||||
|
||||
dl.addCallback(create_session)
|
||||
|
||||
dl.addCallback(lambda _: self.session.setup())
|
||||
|
||||
dl.addCallback(lambda _: self.check_first_run())
|
||||
|
||||
dl.addCallback(self._show_first_run_result)
|
||||
|
||||
return dl
|
||||
|
||||
def check_first_run(self):
|
||||
d = self.session.wallet.is_first_run()
|
||||
d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else 0.0)
|
||||
return d
|
||||
|
||||
def _do_first_run(self):
|
||||
d = self.session.wallet.get_new_address()
|
||||
|
||||
def send_request(url, data):
|
||||
r = requests.post(url, json=data)
|
||||
if r.status_code == 200:
|
||||
return r.json()['credits_sent']
|
||||
return 0.0
|
||||
|
||||
def log_error(err):
|
||||
log.warning("unable to request free credits. %s", err.getErrorMessage())
|
||||
return 0.0
|
||||
|
||||
def request_credits(address):
|
||||
url = "http://credreq.lbry.io/requestcredits"
|
||||
data = {"address": address}
|
||||
d = threads.deferToThread(send_request, url, data)
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
d.addCallback(request_credits)
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _show_first_run_result(credits_received):
|
||||
if credits_received != 0.0:
|
||||
points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),),
|
||||
grouping=True)
|
||||
alert.info("\n\nThank you for testing the alpha version of LBRY!\n\n"
|
||||
"You have been given %s for free because we love you.\n"
|
||||
"Please give them a few minutes to show up while you\n"
|
||||
"catch up with our blockchain.\n", points_string)
|
||||
|
||||
def _setup_lbry_file_manager(self):
|
||||
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.conf_dir)
|
||||
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
|
||||
d = self.lbry_file_metadata_manager.setup()
|
||||
|
||||
def set_lbry_file_manager():
|
||||
@@ -179,59 +318,46 @@ class LBRYConsole():
|
||||
|
||||
def _setup_control_handlers(self):
|
||||
handlers = [
|
||||
('General',
|
||||
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node)),
|
||||
('General',
|
||||
GetWalletBalancesFactory(self.session.wallet)),
|
||||
('General',
|
||||
ModifyApplicationDefaultsFactory(self)),
|
||||
('General',
|
||||
ShutDownFactory(self)),
|
||||
('General',
|
||||
PeerStatsAndSettingsChooserFactory(self.session.peer_manager)),
|
||||
('lbryfile',
|
||||
LBRYFileStatusFactory(self.lbry_file_manager)),
|
||||
('Stream Downloading',
|
||||
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager)),
|
||||
('lbryfile',
|
||||
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
CreateLBRYFileFactory(self.session, self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.session.blob_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager)),
|
||||
('Stream Downloading',
|
||||
AddStreamFromHashFactory(self.sd_identifier, self.session))
|
||||
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node),
|
||||
GetWalletBalancesFactory(self.session.wallet),
|
||||
ModifyApplicationDefaultsFactory(self),
|
||||
ShutDownFactory(self),
|
||||
PeerStatsAndSettingsChooserFactory(self.session.peer_manager),
|
||||
LBRYFileStatusFactory(self.lbry_file_manager),
|
||||
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager,
|
||||
self.session.wallet),
|
||||
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
|
||||
self.lbry_file_manager),
|
||||
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager),
|
||||
CreateLBRYFileFactory(self.session, self.lbry_file_manager),
|
||||
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.session.blob_manager),
|
||||
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.lbry_file_manager),
|
||||
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager),
|
||||
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager),
|
||||
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager),
|
||||
AddStreamFromHashFactory(self.sd_identifier, self.session, self.session.wallet),
|
||||
StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager,
|
||||
self.session.blob_manager, self.session.wallet if not self.fake_wallet else None),
|
||||
ImmediateAnnounceAllBlobsFactory(self.session.blob_manager)
|
||||
]
|
||||
self.add_control_handlers(handlers)
|
||||
if self.wallet_type == 'lbrycrd':
|
||||
if not self.fake_wallet:
|
||||
lbrycrd_handlers = [
|
||||
('Stream Downloading',
|
||||
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
|
||||
self.session.wallet)),
|
||||
('General',
|
||||
ClaimNameFactory(self.session.wallet)),
|
||||
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
|
||||
self.session.wallet),
|
||||
ClaimNameFactory(self.session.wallet, self.lbry_file_manager,
|
||||
self.session.blob_manager),
|
||||
GetNewWalletAddressFactory(self.session.wallet),
|
||||
PublishFactory(self.session, self.lbry_file_manager, self.session.wallet),
|
||||
BlockchainStatusFactory(self.session.wallet)
|
||||
]
|
||||
self.add_control_handlers(lbrycrd_handlers)
|
||||
if self.peer_port is not None:
|
||||
server_handlers = [
|
||||
('Server',
|
||||
ShowServerStatusFactory(self)),
|
||||
('Server',
|
||||
ModifyServerSettingsFactory(self)),
|
||||
ShowServerStatusFactory(self),
|
||||
ModifyServerSettingsFactory(self),
|
||||
]
|
||||
self.add_control_handlers(server_handlers)
|
||||
|
||||
@@ -265,12 +391,19 @@ class LBRYConsole():
|
||||
def setup_plugins():
|
||||
ds = []
|
||||
for plugin in self.plugin_manager.getAllPlugins():
|
||||
self.plugin_objects.append(plugin.plugin_object)
|
||||
ds.append(plugin.plugin_object.setup(self))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
d.addCallback(lambda _: setup_plugins())
|
||||
return d
|
||||
|
||||
def _stop_plugins(self):
|
||||
ds = []
|
||||
for plugin_object in self.plugin_objects:
|
||||
ds.append(defer.maybeDeferred(plugin_object.stop))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _setup_server(self):
|
||||
|
||||
def restore_running_status(running):
|
||||
@@ -290,7 +423,12 @@ class LBRYConsole():
|
||||
self.query_handlers,
|
||||
self.session.peer_manager)
|
||||
from twisted.internet import reactor
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
try:
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.peer_port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer.", str(e))
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop_server(self):
|
||||
@@ -300,26 +438,35 @@ class LBRYConsole():
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def _start_controller(self):
|
||||
self.control_class(self.control_handlers)
|
||||
def _setup_controller(self):
|
||||
self.controller = ConsoleControl()
|
||||
stdio.StandardIO(self.controller)
|
||||
logger = logging.getLogger()
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
alert_handler = logging.StreamHandler(self.controller)
|
||||
alert_handler.setFormatter(formatter)
|
||||
alert_handler.addFilter(logging.Filter("lbryalert"))
|
||||
alert_handler.setLevel(logging.DEBUG)
|
||||
logger.addHandler(alert_handler)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _start_controller(self):
|
||||
return self.controller.start(self.command_handlers)
|
||||
|
||||
def _shut_down(self):
|
||||
self.plugin_manager = None
|
||||
d1 = self.lbry_file_metadata_manager.stop()
|
||||
d1.addCallback(lambda _: self.lbry_file_manager.stop())
|
||||
d2 = self.stop_server()
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
ds = []
|
||||
if self.lbry_file_metadata_manager is not None:
|
||||
d = self.lbry_file_metadata_manager.stop()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.stop())
|
||||
ds.append(d)
|
||||
ds.append(self.stop_server())
|
||||
ds.append(self._stop_plugins())
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
|
||||
class StdIOControl():
|
||||
def __init__(self, control_handlers):
|
||||
stdio.StandardIO(ConsoleControl(control_handlers))
|
||||
|
||||
|
||||
def launch_lbry_console():
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
parser = argparse.ArgumentParser(description="Launch a lbrynet console")
|
||||
@@ -335,12 +482,9 @@ def launch_lbry_console():
|
||||
parser.add_argument("--dht_node_port",
|
||||
help="The port on which the console will listen for DHT connections.",
|
||||
type=int, default=4444)
|
||||
parser.add_argument("--wallet_type",
|
||||
help="Either 'lbrycrd' or 'ptc'.",
|
||||
type=str, default="lbrycrd")
|
||||
parser.add_argument("--lbrycrd_wallet_rpc_port",
|
||||
help="The rpc port on which the LBRYcrd wallet is listening",
|
||||
type=int, default=8332)
|
||||
parser.add_argument("--fake_wallet",
|
||||
help="Testing purposes only. Use a non-blockchain wallet.",
|
||||
action="store_true")
|
||||
parser.add_argument("--no_dht_bootstrap",
|
||||
help="Don't try to connect to the DHT",
|
||||
action="store_true")
|
||||
@@ -352,16 +496,22 @@ def launch_lbry_console():
|
||||
help="The port of a known DHT node, to be used to bootstrap into the DHT. Must "
|
||||
"be used with --dht_bootstrap_host",
|
||||
type=int, default=4000)
|
||||
parser.add_argument("--use_upnp",
|
||||
help="Try to use UPnP to enable incoming connections through the firewall",
|
||||
parser.add_argument("--disable_upnp",
|
||||
help="Don't try to use UPnP to enable incoming connections through the firewall",
|
||||
action="store_true")
|
||||
parser.add_argument("--conf_dir",
|
||||
help=("The full path to the directory in which to store configuration "
|
||||
"options and user added plugins. Default: ~/.lbrynet"),
|
||||
type=str)
|
||||
parser.add_argument("--data_dir",
|
||||
help=("The full path to the directory in which to store data chunks "
|
||||
"downloaded from lbrynet. Default: <conf_dir>/blobfiles"),
|
||||
help=("The full path to the directory in which lbrynet data and metadata will be stored. "
|
||||
"Default: ~/.lbrynet on linux, ~/Library/Application Support/lbrynet on OS X"),
|
||||
type=str)
|
||||
parser.add_argument("--lbrycrdd_path",
|
||||
help="The path to lbrycrdd, which will be launched if it isn't running. If"
|
||||
"this option is chosen, lbrycrdd will be used as the interface to the"
|
||||
"blockchain. By default, a lightweight interface is used.")
|
||||
parser.add_argument("--lbrycrd_wallet_dir",
|
||||
help="The directory in which lbrycrd data will stored. Used if lbrycrdd is "
|
||||
"launched by this application.")
|
||||
parser.add_argument("--lbrycrd_wallet_conf",
|
||||
help="The configuration file for the LBRYcrd wallet. Default: ~/.lbrycrd/lbrycrd.conf",
|
||||
type=str)
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -381,27 +531,49 @@ def launch_lbry_console():
|
||||
else:
|
||||
dht_node_port = args.dht_node_port
|
||||
|
||||
if not args.conf_dir:
|
||||
conf_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
conf_dir = args.conf_dir
|
||||
if not os.path.exists(conf_dir):
|
||||
os.mkdir(conf_dir)
|
||||
created_data_dir = False
|
||||
if not args.data_dir:
|
||||
data_dir = os.path.join(conf_dir, "blobfiles")
|
||||
if sys.platform == "darwin":
|
||||
data_dir = user_data_dir("LBRY")
|
||||
else:
|
||||
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
data_dir = args.data_dir
|
||||
if not os.path.exists(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
created_data_dir = True
|
||||
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, filename=os.path.join(conf_dir, "console.log"),
|
||||
format=log_format)
|
||||
daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
try:
|
||||
daemon.is_running()
|
||||
log.info("Attempt to start lbrynet-console while lbrynet-daemon is running")
|
||||
print "lbrynet-daemon is running, you must turn it off before using lbrynet-console"
|
||||
print "If you're running the app, quit before starting lbrynet-console"
|
||||
print "If you're running lbrynet-daemon in a terminal, run 'stop-lbrynet-daemon' to turn it off"
|
||||
|
||||
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, StdIOControl, wallet_type=args.wallet_type,
|
||||
lbrycrd_rpc_port=args.lbrycrd_wallet_rpc_port, use_upnp=args.use_upnp,
|
||||
conf_dir=conf_dir, data_dir=data_dir)
|
||||
webbrowser.open("http://localhost:5279")
|
||||
|
||||
d = task.deferLater(reactor, 0, console.start)
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
|
||||
reactor.run()
|
||||
except:
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
formatter = logging.Formatter(log_format)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
file_handler = logging.FileHandler(os.path.join(data_dir, "console.log"))
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(logging.Filter("lbrynet"))
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
|
||||
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, fake_wallet=args.fake_wallet,
|
||||
lbrycrd_conf=args.lbrycrd_wallet_conf, lbrycrd_dir=args.lbrycrd_wallet_dir,
|
||||
use_upnp=not args.disable_upnp, data_dir=data_dir,
|
||||
created_data_dir=created_data_dir, lbrycrdd_path=args.lbrycrdd_path)
|
||||
|
||||
d = task.deferLater(reactor, 0, console.start)
|
||||
d.addErrback(lambda _: reactor.stop())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
|
||||
reactor.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
launch_lbry_console()
|
||||
@@ -7,4 +7,7 @@ class LBRYPlugin(IPlugin):
|
||||
IPlugin.__init__(self)
|
||||
|
||||
def setup(self, lbry_console):
|
||||
raise NotImplementedError
|
||||
|
||||
def stop(self):
|
||||
raise NotImplementedError
|
||||
@@ -1,40 +1,44 @@
|
||||
import binascii
|
||||
import json
|
||||
import leveldb
|
||||
import unqlite
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYSettings(object):
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
|
||||
def start(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def _open_db(self):
|
||||
logging.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "settings.db"))
|
||||
log.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "settings.db"))
|
||||
return defer.succeed(True)
|
||||
|
||||
def save_lbryid(self, lbryid):
|
||||
|
||||
def save_lbryid():
|
||||
self.db.Put("lbryid", binascii.hexlify(lbryid), sync=True)
|
||||
self.db['lbryid'] = binascii.hexlify(lbryid)
|
||||
|
||||
return threads.deferToThread(save_lbryid)
|
||||
|
||||
def get_lbryid(self):
|
||||
|
||||
def get_lbryid():
|
||||
try:
|
||||
return binascii.unhexlify(self.db.Get("lbryid"))
|
||||
except KeyError:
|
||||
if 'lbryid' in self.db:
|
||||
return binascii.unhexlify(self.db['lbryid'])
|
||||
else:
|
||||
return None
|
||||
|
||||
return threads.deferToThread(get_lbryid)
|
||||
@@ -42,9 +46,9 @@ class LBRYSettings(object):
|
||||
def get_server_running_status(self):
|
||||
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get("server_running"))
|
||||
except KeyError:
|
||||
if 'server_running' in self.db:
|
||||
return json.loads(self.db['server_running'])
|
||||
else:
|
||||
return True
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
@@ -52,7 +56,7 @@ class LBRYSettings(object):
|
||||
def save_server_running_status(self, running):
|
||||
|
||||
def save_status():
|
||||
self.db.Put("server_running", json.dumps(running), sync=True)
|
||||
self.db['server_running'] = json.dumps(running)
|
||||
|
||||
return threads.deferToThread(save_status)
|
||||
|
||||
@@ -77,9 +81,9 @@ class LBRYSettings(object):
|
||||
def _get_payment_rate(self, rate_type):
|
||||
|
||||
def get_rate():
|
||||
try:
|
||||
return json.loads(self.db.Get(rate_type))
|
||||
except KeyError:
|
||||
if rate_type in self.db:
|
||||
return json.loads(self.db[rate_type])
|
||||
else:
|
||||
return None
|
||||
|
||||
return threads.deferToThread(get_rate)
|
||||
@@ -88,18 +92,18 @@ class LBRYSettings(object):
|
||||
|
||||
def save_rate():
|
||||
if rate is not None:
|
||||
self.db.Put(rate_type, json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(rate_type, sync=True)
|
||||
self.db[rate_type] = json.dumps(rate)
|
||||
elif rate_type in self.db:
|
||||
del self.db[rate_type]
|
||||
|
||||
return threads.deferToThread(save_rate)
|
||||
|
||||
def get_query_handler_status(self, query_identifier):
|
||||
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps(('q_h', query_identifier))))
|
||||
except KeyError:
|
||||
if json.dumps(('q_h', query_identifier)) in self.db:
|
||||
return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
|
||||
else:
|
||||
return True
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
@@ -112,5 +116,5 @@ class LBRYSettings(object):
|
||||
|
||||
def _set_query_handler_status(self, query_identifier, status):
|
||||
def set_status():
|
||||
self.db.Put(json.dumps(('q_h', query_identifier)), json.dumps(status), sync=True)
|
||||
return threads.deferToThread(set_status)
|
||||
self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
|
||||
return threads.deferToThread(set_status)
|
||||
@@ -10,5 +10,15 @@ class IControlHandlerFactory(Interface):
|
||||
|
||||
|
||||
class IControlHandler(Interface):
|
||||
def handle_line(self, line):
|
||||
pass
|
||||
|
||||
|
||||
class ICommandHandlerFactory(Interface):
|
||||
def get_handler(self):
|
||||
pass
|
||||
|
||||
|
||||
class ICommandHandler(Interface):
|
||||
def handle_line(self, line):
|
||||
pass
|
||||
@@ -1,25 +1,43 @@
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.internet import defer
|
||||
from ValuableBlobInfo import ValuableBlobInfo
|
||||
from db_keys import BLOB_INFO_TYPE
|
||||
import json
|
||||
import leveldb
|
||||
import os
|
||||
import sqlite3
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
class BlindInfoManager(object):
|
||||
|
||||
def __init__(self, db, peer_manager):
|
||||
self.db = db
|
||||
def __init__(self, db_dir, peer_manager):
|
||||
self.db_dir = db_dir
|
||||
self.db_conn = None
|
||||
self.peer_manager = peer_manager
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_info.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
def set_up_table(transaction):
|
||||
transaction.execute("create table if not exists valuable_blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" reference text, " +
|
||||
" peer_host text, " +
|
||||
" peer_port integer, " +
|
||||
" peer_score text" +
|
||||
")")
|
||||
return self.db_conn.runInteraction(set_up_table)
|
||||
|
||||
def stop(self):
|
||||
self.db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_blob_infos(self):
|
||||
d = threads.deferToThread(self._get_all_blob_infos)
|
||||
d = self._get_all_blob_infos()
|
||||
|
||||
def make_blob_infos(blob_data):
|
||||
blob_infos = []
|
||||
@@ -42,21 +60,19 @@ class BlindInfoManager(object):
|
||||
peer_port = blob_info.peer.port
|
||||
peer_score = blob_info.peer_score
|
||||
blobs.append((blob_hash, length, reference, peer_host, peer_port, peer_score))
|
||||
return threads.deferToThread(self._save_blob_infos, blobs)
|
||||
return self._save_blob_infos(blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_blob_infos(self):
|
||||
blob_infos = []
|
||||
for key, blob_info in self.db.RangeIter():
|
||||
key_type, blob_hash = json.loads(key)
|
||||
if key_type == BLOB_INFO_TYPE:
|
||||
blob_infos.append([blob_hash] + json.loads(blob_info))
|
||||
return blob_infos
|
||||
return self.db_conn.runQuery("select * from valuable_blobs")
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_blob_infos(self, blobs):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob in blobs:
|
||||
try:
|
||||
self.db.Get(json.dumps((BLOB_INFO_TYPE, blob[0])))
|
||||
except KeyError:
|
||||
batch.Put(json.dumps((BLOB_INFO_TYPE, blob[0])), json.dumps(blob[1:]))
|
||||
self.db.Write(batch, sync=True)
|
||||
def save_infos(transaction):
|
||||
for blob in blobs:
|
||||
try:
|
||||
transaction.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
|
||||
blob)
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
return self.db_conn.runInteraction(save_infos)
|
||||
@@ -2,7 +2,7 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IMetadataHandler, IRequestCreator
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest
|
||||
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
|
||||
from lbrynet.core.Error import NoResponseError
|
||||
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
|
||||
from ValuableBlobInfo import ValuableBlobInfo
|
||||
import datetime
|
||||
import logging
|
||||
@@ -12,6 +12,9 @@ from twisted.python.failure import Failure
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlindMetadataHandler(object):
|
||||
implements(IMetadataHandler, IRequestCreator)
|
||||
|
||||
@@ -195,7 +198,7 @@ class BlindMetadataHandler(object):
|
||||
if not 'valuable_blob_hashes' in response:
|
||||
return InvalidResponseError("Missing the required field 'valuable_blob_hashes'")
|
||||
hashes = response['valuable_blob_hashes']
|
||||
logging.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
|
||||
log.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
|
||||
expire_time = datetime.datetime.now() + datetime.timedelta(minutes=10)
|
||||
reference = None
|
||||
unique_hashes = set()
|
||||
@@ -230,8 +233,8 @@ class BlindMetadataHandler(object):
|
||||
if not 'blob_lengths' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_lengths'")
|
||||
raw_blob_lengths = response['blob_lengths']
|
||||
logging.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
|
||||
logging.debug("blobs: %s", str(raw_blob_lengths))
|
||||
log.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
|
||||
log.debug("blobs: %s", str(raw_blob_lengths))
|
||||
infos = []
|
||||
unique_hashes = set()
|
||||
for blob_hash, length in raw_blob_lengths:
|
||||
@@ -287,11 +290,12 @@ class BlindMetadataHandler(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
|
||||
str(request_type), str(reason.getErrorMessage()))
|
||||
log.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
|
||||
str(request_type), str(reason.getErrorMessage()))
|
||||
self._update_local_score(peer, -10.0)
|
||||
peer.update_score(-5.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
|
||||
def _search_for_peers(self):
|
||||
|
||||
@@ -109,5 +109,5 @@ class BlindRepeater(object):
|
||||
def _update_status(self, stopped=True):
|
||||
self.stopped = stopped
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop()
|
||||
@@ -1,111 +1,121 @@
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ControlHandler, ControlHandlerFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import RecursiveControlHandler, ModifyPaymentRate
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CommandHandler, CommandHandlerFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import RecursiveCommandHandler, ModifyPaymentRate
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
class StartRepeater(ControlHandler):
|
||||
class StartRepeater(CommandHandler):
|
||||
prompt_description = "Start the blind repeater"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Start repeater should not be passed any arguments"
|
||||
def start(self):
|
||||
#assert line is None, "Start repeater should not be passed any arguments"
|
||||
d = self.settings.save_repeater_status(running=True)
|
||||
d.addCallback(lambda _: self.repeater.start())
|
||||
d.addCallback(lambda _: "Started the repeater")
|
||||
return True, d
|
||||
d.addCallback(lambda _: self.console.sendLine("Started the repeater"))
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
|
||||
class StartRepeaterFactory(ControlHandlerFactory):
|
||||
class StartRepeaterFactory(CommandHandlerFactory):
|
||||
control_handler_class = StartRepeater
|
||||
|
||||
|
||||
class StopRepeater(ControlHandler):
|
||||
class StopRepeater(CommandHandler):
|
||||
prompt_description = "Stop the blind repeater"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Stop repeater should not be passed any arguments"
|
||||
def start(self):
|
||||
#assert line is None, "Stop repeater should not be passed any arguments"
|
||||
d = self.settings.save_repeater_status(running=False)
|
||||
d.addCallback(lambda _: self.repeater.stop())
|
||||
d.addCallback(lambda _: "Stopped the repeater")
|
||||
return True, d
|
||||
d.addCallback(lambda _: self.console.sendLine("Stopped the repeater"))
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
|
||||
class StopRepeaterFactory(ControlHandlerFactory):
|
||||
class StopRepeaterFactory(CommandHandlerFactory):
|
||||
control_handler_class = StopRepeater
|
||||
|
||||
|
||||
class UpdateMaxSpace(ControlHandler):
|
||||
class UpdateMaxSpace(CommandHandler):
|
||||
prompt_description = "Set the maximum space to be used by the blind repeater"
|
||||
line_prompt = "Maximum space (in bytes):"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def start(self):
|
||||
self.console.sendLine(self.line_prompt)
|
||||
|
||||
def handle_line(self, line):
|
||||
if line is None:
|
||||
return False, defer.succeed(self.line_prompt)
|
||||
return True, self._set_max_space(line)
|
||||
d = self._set_max_space(line)
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _set_max_space(self, line):
|
||||
max_space = int(line)
|
||||
d = self.settings.save_max_space(max_space)
|
||||
d.addCallback(lambda _: self.repeater.set_max_space(max_space))
|
||||
d.addCallback(lambda _: "Set the maximum space to " + str(max_space) + " bytes")
|
||||
d.addCallback(lambda _: self.console.sendLine("Set the maximum space to " + str(max_space) + " bytes"))
|
||||
return d
|
||||
|
||||
|
||||
class UpdateMaxSpaceFactory(ControlHandlerFactory):
|
||||
class UpdateMaxSpaceFactory(CommandHandlerFactory):
|
||||
control_handler_class = UpdateMaxSpace
|
||||
|
||||
|
||||
class AddApprovedPeer(ControlHandler):
|
||||
class AddApprovedPeer(CommandHandler):
|
||||
prompt_description = "Add a peer to the approved list of peers to check for valuable blob hashes"
|
||||
host_prompt = "Peer host in dotted quad (e.g. 127.0.0.1)"
|
||||
port_prompt = "Peer port (e.g. 4444)"
|
||||
|
||||
def __init__(self, repeater, peer_manager, settings):
|
||||
def __init__(self, console, repeater, peer_manager, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.peer_manager = peer_manager
|
||||
self.settings = settings
|
||||
self.host_to_add = None
|
||||
|
||||
def start(self):
|
||||
self.console.sendLine(self.host_prompt)
|
||||
|
||||
def handle_line(self, line):
|
||||
if line is None:
|
||||
return False, defer.succeed(self.host_prompt)
|
||||
elif self.host_to_add is None:
|
||||
#if line is None:
|
||||
# return False, defer.succeed(self.host_prompt)
|
||||
if self.host_to_add is None:
|
||||
self.host_to_add = line
|
||||
return False, defer.succeed(self.port_prompt)
|
||||
self.console.sendLine(self.port_prompt)
|
||||
else:
|
||||
self.host_to_add, host = None, self.host_to_add
|
||||
return True, self._add_peer(host, line)
|
||||
d = self._add_peer(host, line)
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _add_peer(self, host, port):
|
||||
peer = self.peer_manager.get_peer(host, int(port))
|
||||
d = self.settings.save_approved_peer(host, int(port))
|
||||
d.addCallback(lambda _: self.repeater.add_approved_peer(peer))
|
||||
d.addCallback(lambda _: "Successfully added peer")
|
||||
d.addCallback(lambda _: self.console.sendLine("Successfully added peer"))
|
||||
return d
|
||||
|
||||
|
||||
class AddApprovedPeerFactory(ControlHandlerFactory):
|
||||
class AddApprovedPeerFactory(CommandHandlerFactory):
|
||||
control_handler_class = AddApprovedPeer
|
||||
|
||||
|
||||
class ApprovedPeerChooser(RecursiveControlHandler):
|
||||
class ApprovedPeerChooser(RecursiveCommandHandler):
|
||||
|
||||
def __init__(self, repeater, factory_class, *args, **kwargs):
|
||||
def __init__(self, console, repeater, factory_class, *args, **kwargs):
|
||||
self.repeater = repeater
|
||||
self.factory_class = factory_class
|
||||
self.args = args
|
||||
RecursiveControlHandler.__init__(self, **kwargs)
|
||||
RecursiveCommandHandler.__init__(self, console, **kwargs)
|
||||
|
||||
def _get_control_handler_factories(self):
|
||||
control_handler_factories = []
|
||||
@@ -114,7 +124,7 @@ class ApprovedPeerChooser(RecursiveControlHandler):
|
||||
return control_handler_factories
|
||||
|
||||
|
||||
class ApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class ApprovedPeerChooserFactory(CommandHandlerFactory):
|
||||
def get_prompt_description(self):
|
||||
peer = self.args[0]
|
||||
return str(peer)
|
||||
@@ -123,30 +133,32 @@ class ApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class DeleteApprovedPeerChooser(ApprovedPeerChooser):
|
||||
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ApprovedPeerChooser.__init__(self, repeater, DeleteApprovedPeerFactory, repeater, settings,
|
||||
exit_after_one_done=True)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ApprovedPeerChooser.__init__(self, console, repeater, DeleteApprovedPeerFactory, repeater,
|
||||
settings, exit_after_one_done=True)
|
||||
|
||||
|
||||
class DeleteApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class DeleteApprovedPeerChooserFactory(CommandHandlerFactory):
|
||||
control_handler_class = DeleteApprovedPeerChooser
|
||||
|
||||
|
||||
class DeleteApprovedPeer(ControlHandler):
|
||||
class DeleteApprovedPeer(CommandHandler):
|
||||
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
|
||||
|
||||
def __init__(self, peer, repeater, settings):
|
||||
def __init__(self, console, peer, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
self.peer_to_remove = peer
|
||||
|
||||
def handle_line(self, line):
|
||||
return True, self._remove_peer()
|
||||
def start(self):
|
||||
d = self._remove_peer()
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _remove_peer(self):
|
||||
d = self.settings.remove_approved_peer(self.peer_to_remove.host, int(self.peer_to_remove.port))
|
||||
d.addCallback(lambda _: self.repeater.remove_approved_peer(self.peer_to_remove))
|
||||
d.addCallback(lambda _: "Successfully removed peer")
|
||||
d.addCallback(lambda _: self.console.sendLine("Successfully removed peer"))
|
||||
return d
|
||||
|
||||
|
||||
@@ -154,38 +166,43 @@ class DeleteApprovedPeerFactory(ApprovedPeerChooserFactory):
|
||||
control_handler_class = DeleteApprovedPeer
|
||||
|
||||
|
||||
class ShowApprovedPeers(ControlHandler):
|
||||
class ShowApprovedPeers(CommandHandler):
|
||||
prompt_description = "Show the list of peers approved to be checked for valuable blob hashes"
|
||||
|
||||
def __init__(self, repeater):
|
||||
def __init__(self, console, repeater):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Show approved peers should not be passed any arguments"
|
||||
return True, self._show_peers()
|
||||
def start(self):
|
||||
#assert line is None, "Show approved peers should not be passed any arguments"
|
||||
d = self._show_peers()
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _show_peers(self):
|
||||
peer_string = "Approved peers:\n"
|
||||
for peer in self.repeater.approved_peers:
|
||||
peer_string += str(peer) + "\n"
|
||||
return defer.succeed(peer_string)
|
||||
self.console.sendLine(peer_string)
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class ShowApprovedPeersFactory(ControlHandlerFactory):
|
||||
class ShowApprovedPeersFactory(CommandHandlerFactory):
|
||||
control_handler_class = ShowApprovedPeers
|
||||
|
||||
|
||||
class RepeaterStatus(ControlHandler):
|
||||
class RepeaterStatus(CommandHandler):
|
||||
prompt_description = "Show the repeater's status"
|
||||
|
||||
def __init__(self, repeater):
|
||||
def __init__(self, console, repeater):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Show repeater status should not be passed any arguments"
|
||||
return True, defer.maybeDeferred(self._get_status)
|
||||
def start(self):
|
||||
#assert line is None, "Show repeater status should not be passed any arguments"
|
||||
self._show_status()
|
||||
self.finished_deferred.callback(None)
|
||||
|
||||
def _get_status(self):
|
||||
def _show_status(self):
|
||||
status_string = "Repeater status: " + self.repeater.status() + "\n"
|
||||
|
||||
if self.repeater.stopped is False:
|
||||
@@ -197,25 +214,25 @@ class RepeaterStatus(ControlHandler):
|
||||
|
||||
status_string += "Maximum space: " + str(max_space) + " bytes\n"
|
||||
status_string += "Space used: " + str(space_used) + " bytes\n"
|
||||
return defer.succeed(status_string)
|
||||
self.console.sendLine(status_string)
|
||||
|
||||
|
||||
class RepeaterStatusFactory(ControlHandlerFactory):
|
||||
class RepeaterStatusFactory(CommandHandlerFactory):
|
||||
control_handler_class = RepeaterStatus
|
||||
|
||||
|
||||
class ModifyDataPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater data payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self._prompt_choices['unset'] = (self._unset, "Use the application default data rate")
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
def _unset(self):
|
||||
self._set_rate(None)
|
||||
return True, defer.succeed("Using the application default data rate")
|
||||
return defer.succeed("Using the application default data rate")
|
||||
|
||||
def _set_rate(self, rate):
|
||||
|
||||
@@ -237,15 +254,15 @@ class ModifyDataPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyDataPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyDataPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyDataPaymentRate
|
||||
|
||||
|
||||
class ModifyInfoPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater valuable info payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
@@ -264,15 +281,15 @@ class ModifyInfoPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyInfoPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyInfoPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyInfoPaymentRate
|
||||
|
||||
|
||||
class ModifyHashPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater valuable hash payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
@@ -291,18 +308,18 @@ class ModifyHashPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyHashPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyHashPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyHashPaymentRate
|
||||
|
||||
|
||||
class ModifyRepeaterOptions(RecursiveControlHandler):
|
||||
class ModifyRepeaterOptions(RecursiveCommandHandler):
|
||||
prompt_description = "Modify Blind Repeater options"
|
||||
|
||||
def __init__(self, repeater, lbry_session, settings):
|
||||
def __init__(self, console, repeater, lbry_session, settings):
|
||||
self.repeater = repeater
|
||||
self.lbry_session = lbry_session
|
||||
self.settings = settings
|
||||
RecursiveControlHandler.__init__(self)
|
||||
RecursiveCommandHandler.__init__(self, console)
|
||||
|
||||
def _get_control_handler_factories(self):
|
||||
return [ModifyDataPaymentRateFactory(self.repeater, self.settings),
|
||||
@@ -314,5 +331,5 @@ class ModifyRepeaterOptions(RecursiveControlHandler):
|
||||
]
|
||||
|
||||
|
||||
class ModifyRepeaterOptionsFactory(ControlHandlerFactory):
|
||||
class ModifyRepeaterOptionsFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyRepeaterOptions
|
||||
@@ -1,67 +1,79 @@
|
||||
from db_keys import SETTING_TYPE, PEER_TYPE
|
||||
from twisted.internet import threads
|
||||
from twisted.internet import threads, defer
|
||||
import json
|
||||
|
||||
import unqlite
|
||||
import os
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
class BlindRepeaterSettings(object):
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.unq_db = None
|
||||
self.sql_db = None
|
||||
|
||||
def setup(self):
|
||||
self.unq_db = unqlite.UnQLite(os.path.join(self.db_dir, "blind_settings.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.sql_db = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_peers.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
return self.sql_db.runQuery("create table if not exists approved_peers (" +
|
||||
" ip_address text, " +
|
||||
" port integer" +
|
||||
")")
|
||||
|
||||
def stop(self):
|
||||
self.unq_db = None
|
||||
self.sql_db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def save_repeater_status(self, running):
|
||||
def save_status():
|
||||
self.db.Put(json.dumps((SETTING_TYPE, "running")), json.dumps(running), sync=True)
|
||||
self.unq_db["running"] = json.dumps(running)
|
||||
|
||||
return threads.deferToThread(save_status)
|
||||
|
||||
def get_repeater_saved_status(self):
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, "running"))))
|
||||
except KeyError:
|
||||
if "running" in self.unq_db:
|
||||
return json.loads(self.unq_db['running'])
|
||||
else:
|
||||
return False
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
|
||||
def save_max_space(self, max_space):
|
||||
def save_space():
|
||||
self.db.Put(json.dumps((SETTING_TYPE, "max_space")), str(max_space), sync=True)
|
||||
self.unq_db['max_space'] = json.dumps(max_space)
|
||||
|
||||
return threads.deferToThread(save_space)
|
||||
|
||||
def get_saved_max_space(self):
|
||||
def get_space():
|
||||
try:
|
||||
return int(self.db.Get(json.dumps((SETTING_TYPE, "max_space"))))
|
||||
except KeyError:
|
||||
if 'max_space' in self.unq_db:
|
||||
return json.loads(self.unq_db['max_space'])
|
||||
else:
|
||||
return 0
|
||||
|
||||
return threads.deferToThread(get_space)
|
||||
|
||||
@rerun_if_locked
|
||||
def save_approved_peer(self, host, port):
|
||||
def add_peer():
|
||||
peer_string = json.dumps((PEER_TYPE, (host, port)))
|
||||
self.db.Put(peer_string, "", sync=True)
|
||||
|
||||
return threads.deferToThread(add_peer)
|
||||
return self.sql_db.runQuery("insert into approved_peers values (?, ?)",
|
||||
(host, port))
|
||||
|
||||
@rerun_if_locked
|
||||
def remove_approved_peer(self, host, port):
|
||||
def remove_peer():
|
||||
peer_string = json.dumps((PEER_TYPE, (host, port)))
|
||||
self.db.Delete(peer_string, sync=True)
|
||||
|
||||
return threads.deferToThread(remove_peer)
|
||||
return self.sql_db.runQuery("delete from approved_peers where ip_address = ? and port = ?",
|
||||
(host, port))
|
||||
|
||||
@rerun_if_locked
|
||||
def get_approved_peers(self):
|
||||
def get_peers():
|
||||
peers = []
|
||||
for k, v in self.db.RangeIter():
|
||||
key_type, peer_info = json.loads(k)
|
||||
if key_type == PEER_TYPE:
|
||||
peers.append(peer_info)
|
||||
return peers
|
||||
|
||||
return threads.deferToThread(get_peers)
|
||||
return self.sql_db.runQuery("select * from approved_peers")
|
||||
|
||||
def get_data_payment_rate(self):
|
||||
return threads.deferToThread(self._get_rate, "data_payment_rate")
|
||||
@@ -82,13 +94,13 @@ class BlindRepeaterSettings(object):
|
||||
return threads.deferToThread(self._save_rate, "valuable_hash_rate", rate)
|
||||
|
||||
def _get_rate(self, rate_type):
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, rate_type))))
|
||||
except KeyError:
|
||||
if rate_type in self.unq_db:
|
||||
return json.loads(self.unq_db[rate_type])
|
||||
else:
|
||||
return None
|
||||
|
||||
def _save_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db.Put(json.dumps((SETTING_TYPE, rate_type)), json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(json.dumps((SETTING_TYPE, rate_type)), sync=True)
|
||||
self.unq_db[rate_type] = json.dumps(rate)
|
||||
elif rate_type in self.unq_db:
|
||||
del self.unq_db[rate_type]
|
||||
@@ -4,6 +4,9 @@ from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValuableQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
@@ -98,8 +101,8 @@ class ValuableBlobHashQueryHandler(ValuableQueryHandler):
|
||||
for blob_hash, count in valuable_hashes:
|
||||
hashes_and_scores.append((blob_hash, 1.0 * count / 10.0))
|
||||
if len(hashes_and_scores) != 0:
|
||||
logging.info("Responding to a valuable blob hashes request with %s blob hashes: %s",
|
||||
str(len(hashes_and_scores)))
|
||||
log.info("Responding to a valuable blob hashes request with %s blob hashes: %s",
|
||||
str(len(hashes_and_scores)))
|
||||
expected_payment = 1.0 * len(hashes_and_scores) * self.valuable_blob_hash_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_valuable_blob_hashes', len(hashes_and_scores))
|
||||
@@ -190,7 +193,7 @@ class ValuableBlobLengthQueryHandler(ValuableQueryHandler):
|
||||
if success is True:
|
||||
lengths.append(response_pair)
|
||||
if len(lengths) > 0:
|
||||
logging.info("Responding with %s blob lengths: %s", str(len(lengths)))
|
||||
log.info("Responding with %s blob lengths: %s", str(len(lengths)))
|
||||
expected_payment = 1.0 * len(lengths) * self.blob_length_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_valuable_blob_infos', len(lengths))
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import leveldb
|
||||
import os
|
||||
from lbrynet.lbrynet_console import LBRYPlugin
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet import defer
|
||||
from lbrynet.conf import MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE, MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE
|
||||
from BlindRepeater import BlindRepeater
|
||||
from BlindInfoManager import BlindInfoManager
|
||||
@@ -19,6 +17,7 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
|
||||
def __init__(self):
|
||||
LBRYPlugin.LBRYPlugin.__init__(self)
|
||||
self.enabled = False
|
||||
self.blind_info_manager = None
|
||||
self.valuable_blob_length_query_handler = None
|
||||
self.valuable_blob_hash_query_handler = None
|
||||
@@ -26,14 +25,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
self.control_handlers = None
|
||||
self.payment_rate_manager = None
|
||||
self.settings = None
|
||||
self.db = None
|
||||
|
||||
def setup(self, lbry_console):
|
||||
if not self.enabled:
|
||||
return defer.succeed(True)
|
||||
lbry_session = lbry_console.session
|
||||
d = threads.deferToThread(self._setup_db, lbry_session.db_dir)
|
||||
d.addCallback(lambda _: self._setup_settings())
|
||||
d = self._setup_settings(lbry_session.db_dir)
|
||||
d.addCallback(lambda _: self._get_payment_rate_manager(lbry_session.base_payment_rate_manager))
|
||||
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager))
|
||||
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager, lbry_session.db_dir))
|
||||
d.addCallback(lambda _: self._setup_blind_repeater(lbry_session))
|
||||
d.addCallback(lambda _: self._setup_valuable_blob_query_handler(lbry_session))
|
||||
d.addCallback(lambda _: self._create_control_handlers(lbry_session))
|
||||
@@ -41,11 +40,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
d.addCallback(lambda _: self._add_to_lbry_console(lbry_console))
|
||||
return d
|
||||
|
||||
def _setup_db(self, db_dir):
|
||||
self.db = leveldb.LevelDB(os.path.join(db_dir, "valuable_blobs.db"))
|
||||
def stop(self):
|
||||
if self.settings is not None:
|
||||
return self.settings.stop()
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_settings(self):
|
||||
self.settings = BlindRepeaterSettings(self.db)
|
||||
def _setup_settings(self, db_dir):
|
||||
self.settings = BlindRepeaterSettings(db_dir)
|
||||
return self.settings.setup()
|
||||
|
||||
def _get_payment_rate_manager(self, default_payment_rate_manager):
|
||||
d1 = self.settings.get_data_payment_rate()
|
||||
@@ -67,8 +69,8 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
dl.addCallback(get_payment_rate_manager)
|
||||
return dl
|
||||
|
||||
def _setup_blind_info_manager(self, peer_manager):
|
||||
self.blind_info_manager = BlindInfoManager(self.db, peer_manager)
|
||||
def _setup_blind_info_manager(self, peer_manager, db_dir):
|
||||
self.blind_info_manager = BlindInfoManager(db_dir, peer_manager)
|
||||
return self.blind_info_manager.setup()
|
||||
|
||||
def _setup_valuable_blob_query_handler(self, lbry_session):
|
||||
@@ -125,4 +127,4 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
def _add_to_lbry_console(self, lbry_console):
|
||||
lbry_console.add_control_handlers(self.control_handlers)
|
||||
lbry_console.add_query_handlers([self.valuable_blob_length_query_handler,
|
||||
self.valuable_blob_hash_query_handler])
|
||||
self.valuable_blob_hash_query_handler])
|
||||
@@ -1,3 +0,0 @@
|
||||
BLOB_INFO_TYPE = 'b'
|
||||
SETTING_TYPE = 's'
|
||||
PEER_TYPE = 'p'
|
||||
@@ -0,0 +1,108 @@
|
||||
import rumps
|
||||
import xmlrpclib
|
||||
import os
|
||||
import webbrowser
|
||||
import subprocess
|
||||
import argparse
|
||||
|
||||
|
||||
class DaemonStatusBarApp(rumps.App):
|
||||
def __init__(self):
|
||||
icon_path = 'app.icns'
|
||||
if os.path.isfile(icon_path):
|
||||
rumps.App.__init__(self, name="LBRY", icon=icon_path, quit_button=None,
|
||||
menu=["Open", "Preferences", "View balance", "Quit"])
|
||||
else:
|
||||
rumps.App.__init__(self, name="LBRY", title="LBRY", quit_button=None,
|
||||
menu=["Open", "Preferences", "View balance", "Quit"])
|
||||
|
||||
@rumps.timer(1)
|
||||
def alert_daemon_start(self):
|
||||
daemon = xmlrpclib.ServerProxy("http://localhost:7080/")
|
||||
try:
|
||||
start_msg = daemon.is_running()
|
||||
if isinstance(start_msg, str):
|
||||
rumps.notification(title='LBRY', subtitle='', message=str(start_msg), sound=True)
|
||||
update_info = daemon.check_for_new_version()
|
||||
update_msg = ""
|
||||
for p in update_info:
|
||||
if not p[0]:
|
||||
update_msg += p[1] + "\n"
|
||||
if update_msg:
|
||||
update_msg += "\n Try running the installer again to fix this"
|
||||
rumps.notification(title='LBRY', subtitle='', message=update_msg, sound=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
@rumps.clicked('Open')
|
||||
def get_ui(self):
|
||||
daemon = xmlrpclib.ServerProxy("http://localhost:7080/")
|
||||
try:
|
||||
daemon.is_running()
|
||||
webbrowser.get('safari').open("lbry://lbry")
|
||||
except:
|
||||
try:
|
||||
rumps.notification(title='LBRY', subtitle='', message="Couldn't connect to lbrynet daemon", sound=True)
|
||||
except:
|
||||
rumps.alert(title='LBRY', message="Couldn't connect to lbrynet daemon")
|
||||
|
||||
@rumps.clicked("Preferences")
|
||||
def prefs(self):
|
||||
daemon = xmlrpclib.ServerProxy("http://localhost:7080/")
|
||||
try:
|
||||
daemon.is_running()
|
||||
webbrowser.get('safari').open("lbry://settings")
|
||||
except:
|
||||
rumps.notification(title='LBRY', subtitle='', message="Couldn't connect to lbrynet daemon", sound=True)
|
||||
|
||||
@rumps.clicked("View balance")
|
||||
def disp_balance(self):
|
||||
daemon = xmlrpclib.ServerProxy("http://localhost:7080/")
|
||||
try:
|
||||
balance = daemon.get_balance()
|
||||
r = round(float(balance), 2)
|
||||
try:
|
||||
rumps.notification(title='LBRY', subtitle='', message=str("Your balance is %.2f LBC" % r), sound=False)
|
||||
except:
|
||||
rumps.alert(title='LBRY', message=str("Your balance is %.2f LBC" % r))
|
||||
|
||||
except:
|
||||
try:
|
||||
rumps.notification(title='LBRY', subtitle='', message="Couldn't connect to lbrynet daemon", sound=True)
|
||||
except:
|
||||
rumps.alert(title='LBRY', message="Couldn't connect to lbrynet daemon")
|
||||
|
||||
@rumps.clicked('Quit')
|
||||
def clean_quit(self):
|
||||
daemon = xmlrpclib.ServerProxy("http://localhost:7080/")
|
||||
try:
|
||||
daemon.stop()
|
||||
except:
|
||||
pass
|
||||
rumps.quit_application()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet status bar application")
|
||||
parser.add_argument("--startdaemon",
|
||||
help="true or false, default true",
|
||||
type=str,
|
||||
default="true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if str(args.startdaemon).lower() == "true":
|
||||
daemon = xmlrpclib.ServerProxy('http://localhost:7080')
|
||||
try:
|
||||
daemon.is_running()
|
||||
except:
|
||||
subprocess.Popen("screen -dmS lbrynet bash -c "
|
||||
"'PYTHONPATH=$PYTHONPATH:`cat /Users/${USER}/Library/Application\ Support/lbrynet/.python_path`; "
|
||||
"PATH=$PATH:`cat /Users/${USER}/Library/Application\ Support/lbrynet/.lbry_bin_path`; "
|
||||
"lbrynet-daemon --update=False'", shell=True)
|
||||
|
||||
status_app = DaemonStatusBarApp()
|
||||
status_app.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
import os
|
||||
import json
|
||||
import webbrowser
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from time import sleep
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
|
||||
UI_ADDRESS = "http://localhost:5279"
|
||||
|
||||
|
||||
class Timeout(Exception):
|
||||
def __init__(self, value):
|
||||
self.parameter = value
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.parameter)
|
||||
|
||||
|
||||
class LBRYURIHandler(object):
|
||||
def __init__(self):
|
||||
self.started_daemon = False
|
||||
self.start_timeout = 0
|
||||
self.daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
|
||||
def check_status(self):
|
||||
status = None
|
||||
try:
|
||||
status = self.daemon.is_running()
|
||||
if self.start_timeout < 30 and not status:
|
||||
sleep(1)
|
||||
self.start_timeout += 1
|
||||
self.check_status()
|
||||
elif status:
|
||||
return True
|
||||
else:
|
||||
raise Timeout("LBRY daemon is running, but connection timed out")
|
||||
except:
|
||||
if self.start_timeout < 30:
|
||||
sleep(1)
|
||||
self.start_timeout += 1
|
||||
self.check_status()
|
||||
else:
|
||||
raise Timeout("Timed out trying to start LBRY daemon")
|
||||
|
||||
def handle_osx(self, lbry_name):
|
||||
lbry_process = [d for d in subprocess.Popen(['ps','aux'], stdout=subprocess.PIPE).stdout.readlines()
|
||||
if 'LBRY.app' in d and 'LBRYURIHandler' not in d]
|
||||
try:
|
||||
status = self.daemon.is_running()
|
||||
except:
|
||||
status = None
|
||||
|
||||
if lbry_process or status:
|
||||
self.check_status()
|
||||
started = False
|
||||
else:
|
||||
os.system("open /Applications/LBRY.app")
|
||||
self.check_status()
|
||||
started = True
|
||||
|
||||
if lbry_name == "lbry" or lbry_name == "" and not started:
|
||||
webbrowser.open(UI_ADDRESS)
|
||||
else:
|
||||
webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
|
||||
|
||||
def handle_linux(self, lbry_name):
|
||||
try:
|
||||
is_running = self.daemon.is_running()
|
||||
if not is_running:
|
||||
sys.exit(0)
|
||||
except:
|
||||
sys.exit(0)
|
||||
|
||||
if lbry_name == "lbry":
|
||||
webbrowser.open(UI_ADDRESS)
|
||||
else:
|
||||
webbrowser.open(UI_ADDRESS + "/view?name=" + lbry_name)
|
||||
|
||||
|
||||
def main(args):
|
||||
if len(args) != 1:
|
||||
args = ['lbry://lbry']
|
||||
|
||||
name = args[0][7:]
|
||||
if sys.platform == "darwin":
|
||||
LBRYURIHandler().handle_osx(lbry_name=name)
|
||||
else:
|
||||
LBRYURIHandler().handle_linux(lbry_name=name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,180 @@
|
||||
import argparse
|
||||
import logging
|
||||
import logging.handlers
|
||||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
import webbrowser
|
||||
import sys
|
||||
import socket
|
||||
|
||||
from StringIO import StringIO
|
||||
from zipfile import ZipFile
|
||||
from urllib import urlopen
|
||||
from datetime import datetime
|
||||
from appdirs import user_data_dir
|
||||
from twisted.web import server, static
|
||||
from twisted.internet import reactor, defer
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender
|
||||
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, DEFAULT_WALLET, UI_ADDRESS
|
||||
|
||||
if sys.platform != "darwin":
|
||||
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
log_dir = user_data_dir("LBRY")
|
||||
|
||||
if not os.path.isdir(log_dir):
|
||||
os.mkdir(log_dir)
|
||||
|
||||
LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log')
|
||||
log = logging.getLogger(__name__)
|
||||
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, backupCount=5)
|
||||
log.addHandler(handler)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
REMOTE_SERVER = "www.google.com"
|
||||
|
||||
|
||||
def test_internet_connection():
|
||||
try:
|
||||
host = socket.gethostbyname(REMOTE_SERVER)
|
||||
s = socket.create_connection((host, 80), 2)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def stop():
|
||||
def _disp_shutdown():
|
||||
log.info("Shutting down lbrynet-daemon from command line")
|
||||
|
||||
def _disp_not_running():
|
||||
log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
|
||||
|
||||
d = defer.Deferred(None)
|
||||
d.addCallback(lambda _: JSONRPCProxy.from_url(API_CONNECTION_STRING).stop())
|
||||
d.addCallbacks(lambda _: _disp_shutdown(), lambda _: _disp_not_running())
|
||||
d.callback(None)
|
||||
|
||||
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument("--wallet",
|
||||
help="lbrycrd or lbryum, default lbryum",
|
||||
type=str,
|
||||
default=DEFAULT_WALLET)
|
||||
parser.add_argument("--ui",
|
||||
help="path to custom UI folder",
|
||||
default="")
|
||||
parser.add_argument("--branch",
|
||||
help="Branch of lbry-web-ui repo to use, defaults on HEAD",
|
||||
default="HEAD")
|
||||
parser.add_argument('--no-launch', dest='launchui', action="store_false")
|
||||
parser.set_defaults(launchui=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
|
||||
log.info("lbrynet-daemon is already running")
|
||||
if args.launchui:
|
||||
webbrowser.open(UI_ADDRESS)
|
||||
return
|
||||
except:
|
||||
pass
|
||||
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
print "Starting lbrynet-daemon from command line"
|
||||
print "To view activity, view the log file here: " + LOG_FILENAME
|
||||
print "Web UI is available at http://%s:%i" %(API_INTERFACE, API_PORT)
|
||||
print "JSONRPC API is available at " + API_CONNECTION_STRING
|
||||
print "To quit press ctrl-c or call 'stop' via the API"
|
||||
|
||||
if args.branch == "HEAD":
|
||||
GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch
|
||||
DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip"
|
||||
else:
|
||||
log.info("Using UI branch: " + args.branch)
|
||||
GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % args.branch
|
||||
DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % args.branch
|
||||
|
||||
def getui(ui_dir=None):
|
||||
if ui_dir:
|
||||
if os.path.isdir(ui_dir):
|
||||
log.info("Using user specified UI directory: " + str(ui_dir))
|
||||
ui_version_info = "user-specified"
|
||||
return defer.succeed([ui_dir, ui_version_info])
|
||||
else:
|
||||
log.info("User specified UI directory doesn't exist: " + str(ui_dir))
|
||||
|
||||
def download_ui(dest_dir, ui_version):
|
||||
url = urlopen(DIST_URL)
|
||||
z = ZipFile(StringIO(url.read()))
|
||||
names = [i for i in z.namelist() if '.DS_Store' not in i and '__MACOSX' not in i]
|
||||
z.extractall(dest_dir, members=names)
|
||||
return defer.succeed([dest_dir, ui_version])
|
||||
|
||||
data_dir = user_data_dir("LBRY")
|
||||
version_dir = os.path.join(data_dir, "ui_version_history")
|
||||
|
||||
git_version = subprocess.check_output(GIT_CMD_STRING, shell=True)
|
||||
if not git_version:
|
||||
log.info("You should have been notified to install xcode command line tools, once it's installed you can start LBRY")
|
||||
print "You should have been notified to install xcode command line tools, once it's installed you can start LBRY"
|
||||
sys.exit(0)
|
||||
|
||||
ui_version_info = git_version
|
||||
|
||||
if not os.path.isdir(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
|
||||
if not os.path.isdir(os.path.join(data_dir, "ui_version_history")):
|
||||
os.mkdir(version_dir)
|
||||
|
||||
if not os.path.isfile(os.path.join(version_dir, git_version)):
|
||||
f = open(os.path.join(version_dir, git_version), "w")
|
||||
version_message = "[" + str(datetime.now()) + "] Updating UI --> " + git_version
|
||||
f.write(version_message)
|
||||
f.close()
|
||||
log.info(version_message)
|
||||
|
||||
if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
|
||||
shutil.rmtree(os.path.join(data_dir, "lbry-web-ui"))
|
||||
else:
|
||||
version_message = "[" + str(datetime.now()) + "] UI version " + git_version + " up to date"
|
||||
log.info(version_message)
|
||||
|
||||
if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")):
|
||||
return defer.succeed([os.path.join(data_dir, "lbry-web-ui"), ui_version_info])
|
||||
else:
|
||||
return download_ui(os.path.join(data_dir, "lbry-web-ui"), ui_version_info)
|
||||
|
||||
def setupserver(ui_dir, ui_version):
|
||||
root = LBRYindex(ui_dir)
|
||||
root.putChild("css", static.File(os.path.join(ui_dir, "css")))
|
||||
root.putChild("font", static.File(os.path.join(ui_dir, "font")))
|
||||
root.putChild("img", static.File(os.path.join(ui_dir, "img")))
|
||||
root.putChild("js", static.File(os.path.join(ui_dir, "js")))
|
||||
root.putChild("view", LBRYFileRender())
|
||||
return defer.succeed([root, ui_version])
|
||||
|
||||
def setupapi(root, wallet, ui_version):
|
||||
daemon = LBRYDaemon(ui_version, wallet_type=wallet)
|
||||
root.putChild(API_ADDRESS, daemon)
|
||||
reactor.listenTCP(API_PORT, server.Site(root), interface=API_INTERFACE)
|
||||
return daemon.setup()
|
||||
|
||||
if test_internet_connection():
|
||||
d = getui(args.ui)
|
||||
d.addCallback(lambda r: setupserver(r[0], r[1]))
|
||||
d.addCallback(lambda r: setupapi(r[0], args.wallet, r[1]))
|
||||
if args.launchui:
|
||||
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
|
||||
reactor.run()
|
||||
print "\nClosing lbrynet-daemon"
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
print "Not connected to internet, unable to start"
|
||||
return
|
||||
@@ -0,0 +1,239 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from datetime import datetime
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.core.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
|
||||
from lbrynet.conf import DEFAULT_TIMEOUT
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream(object):
|
||||
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, pay_key=True, data_rate=0.5,
|
||||
timeout=DEFAULT_TIMEOUT, download_directory=None):
|
||||
self.wallet = wallet
|
||||
self.resolved_name = None
|
||||
self.description = None
|
||||
self.key_fee = None
|
||||
self.key_fee_address = None
|
||||
self.data_rate = data_rate
|
||||
self.pay_key = pay_key
|
||||
self.name = None
|
||||
self.session = session
|
||||
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.stream_hash = None
|
||||
self.max_key_fee = max_key_fee
|
||||
self.stream_info = None
|
||||
self.stream_info_manager = None
|
||||
self.d = defer.Deferred(None)
|
||||
self.timeout = timeout
|
||||
self.timeout_counter = 0
|
||||
self.download_directory = download_directory
|
||||
self.download_path = None
|
||||
self.checker = LoopingCall(self.check_status)
|
||||
|
||||
|
||||
def check_status(self):
|
||||
self.timeout_counter += 1
|
||||
|
||||
if self.download_path and os.path.isfile(self.download_path):
|
||||
self.checker.stop()
|
||||
return defer.succeed(True)
|
||||
|
||||
elif self.timeout_counter >= self.timeout:
|
||||
log.info("Timeout downloading " + str(self.stream_info))
|
||||
self.checker.stop()
|
||||
self.d.cancel()
|
||||
|
||||
def start(self, stream_info):
|
||||
self.stream_info = stream_info
|
||||
if 'stream_hash' in self.stream_info.keys():
|
||||
self.description = self.stream_info['description']
|
||||
if 'key_fee' in self.stream_info.keys():
|
||||
self.key_fee = float(self.stream_info['key_fee'])
|
||||
if 'key_fee_address' in self.stream_info.keys():
|
||||
self.key_fee_address = self.stream_info['key_fee_address']
|
||||
else:
|
||||
self.key_fee_address = None
|
||||
else:
|
||||
self.key_fee = None
|
||||
self.key_fee_address = None
|
||||
|
||||
self.stream_hash = self.stream_info['stream_hash']
|
||||
if isinstance(self.stream_hash, dict):
|
||||
self.stream_hash = self.stream_hash['sd_hash']
|
||||
|
||||
else:
|
||||
log.error("InvalidStreamInfoError in autofetcher: ", stream_info)
|
||||
raise InvalidStreamInfoError(self.stream_info)
|
||||
|
||||
if self.key_fee > self.max_key_fee:
|
||||
if self.pay_key:
|
||||
log.info("Key fee (" + str(self.key_fee) + ") above limit of " + str(
|
||||
self.max_key_fee) + ", didn't download lbry://" + str(self.resolved_name))
|
||||
return defer.fail(None)
|
||||
else:
|
||||
pass
|
||||
|
||||
self.checker.start(1)
|
||||
|
||||
self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
|
||||
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
|
||||
self.d.addCallback(lambda metadata: (next(factory for factory in metadata.factories if isinstance(factory, ManagedLBRYFileDownloaderFactory)), metadata))
|
||||
self.d.addCallback(lambda (factory, metadata): factory.make_downloader(metadata,
|
||||
[self.data_rate, True],
|
||||
self.payment_rate_manager,
|
||||
download_directory=self.download_directory))
|
||||
self.d.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
self.d.addErrback(lambda err: log.error("An exception occurred attempting to load the stream descriptor: %s", err.getTraceback()))
|
||||
self.d.addCallback(self._start_download)
|
||||
self.d.callback(None)
|
||||
|
||||
return self.d
|
||||
|
||||
def _start_download(self, downloader):
|
||||
def _pay_key_fee():
|
||||
if self.key_fee is not None and self.key_fee_address is not None:
|
||||
reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee)
|
||||
if reserved_points is None:
|
||||
return defer.fail(InsufficientFundsError())
|
||||
log.info("Key fee: " + str(self.key_fee) + " | " + str(self.key_fee_address))
|
||||
return self.wallet.send_points_to_address(reserved_points, self.key_fee)
|
||||
return defer.succeed(None)
|
||||
|
||||
if self.pay_key:
|
||||
d = _pay_key_fee()
|
||||
else:
|
||||
d = defer.Deferred()
|
||||
|
||||
downloader.start()
|
||||
|
||||
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
|
||||
d.addCallback(lambda _: log.info("Downloading " + str(self.stream_hash) + " --> " + str(self.download_path)))
|
||||
|
||||
return d
|
||||
|
||||
|
||||
class FetcherDaemon(object):
|
||||
def __init__(self, session, lbry_file_manager, lbry_file_metadata_manager, wallet, sd_identifier, autofetcher_conf,
|
||||
verbose=False):
|
||||
self.autofetcher_conf = autofetcher_conf
|
||||
self.max_key_fee = 0.0
|
||||
self.sd_identifier = sd_identifier
|
||||
self.wallet = wallet
|
||||
self.session = session
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.lbry_metadata_manager = lbry_file_metadata_manager
|
||||
self.seen = []
|
||||
self.lastbestblock = None
|
||||
self.search = None
|
||||
self.first_run = True
|
||||
self.is_running = False
|
||||
self.verbose = verbose
|
||||
self._get_autofetcher_conf()
|
||||
|
||||
def start(self):
|
||||
if not self.is_running:
|
||||
self.is_running = True
|
||||
self.search = LoopingCall(self._looped_search)
|
||||
self.search.start(1)
|
||||
log.info("Starting autofetcher")
|
||||
else:
|
||||
log.info("Autofetcher is already running")
|
||||
|
||||
def stop(self):
|
||||
if self.is_running:
|
||||
self.search.stop()
|
||||
self.is_running = False
|
||||
else:
|
||||
log.info("Autofetcher isn't running, there's nothing to stop")
|
||||
|
||||
def check_if_running(self):
|
||||
if self.is_running:
|
||||
msg = "Autofetcher is running\n"
|
||||
msg += "Last block hash: " + str(self.lastbestblock)
|
||||
else:
|
||||
msg = "Autofetcher is not running"
|
||||
return msg
|
||||
|
||||
def _get_names(self):
|
||||
d = self.wallet.get_best_blockhash()
|
||||
d.addCallback(lambda blockhash: get_new_streams(blockhash) if blockhash != self.lastbestblock else [])
|
||||
|
||||
def get_new_streams(blockhash):
|
||||
self.lastbestblock = blockhash
|
||||
d = self.wallet.get_block(blockhash)
|
||||
d.addCallback(lambda block: get_new_streams_in_txes(block['tx'], blockhash))
|
||||
return d
|
||||
|
||||
def get_new_streams_in_txes(txids, blockhash):
|
||||
ds = []
|
||||
for t in txids:
|
||||
d = self.wallet.get_claims_from_tx(t)
|
||||
d.addCallback(get_new_streams_in_tx, t, blockhash)
|
||||
ds.append(d)
|
||||
d = defer.DeferredList(ds, consumeErrors=True)
|
||||
d.addCallback(lambda result: [r[1] for r in result if r[0]])
|
||||
d.addCallback(lambda stream_lists: [stream for streams in stream_lists for stream in streams])
|
||||
return d
|
||||
|
||||
def get_new_streams_in_tx(claims, t, blockhash):
|
||||
rtn = []
|
||||
if claims:
|
||||
for claim in claims:
|
||||
if claim not in self.seen:
|
||||
msg = "[" + str(datetime.now()) + "] New claim | lbry://" + str(claim['name']) + \
|
||||
" | stream hash: " + str(json.loads(claim['value'])['stream_hash'])
|
||||
log.info(msg)
|
||||
if self.verbose:
|
||||
print msg
|
||||
rtn.append((claim['name'], t))
|
||||
self.seen.append(claim)
|
||||
else:
|
||||
if self.verbose:
|
||||
print "[" + str(datetime.now()) + "] No claims in block", blockhash
|
||||
return rtn
|
||||
|
||||
d.addCallback(lambda streams: defer.DeferredList(
|
||||
[self.wallet.get_stream_info_from_txid(name, t) for name, t in streams]))
|
||||
return d
|
||||
|
||||
def _download_claims(self, claims):
|
||||
if claims:
|
||||
for claim in claims:
|
||||
stream = GetStream(self.sd_identifier, self.session, self.wallet, self.lbry_file_manager,
|
||||
self.max_key_fee, pay_key=False)
|
||||
stream.start(claim[1])
|
||||
|
||||
return defer.succeed(None)
|
||||
|
||||
def _looped_search(self):
|
||||
d = self._get_names()
|
||||
d.addCallback(self._download_claims)
|
||||
return d
|
||||
|
||||
def _get_autofetcher_conf(self):
|
||||
settings = {"maxkey": "0.0"}
|
||||
if os.path.exists(self.autofetcher_conf):
|
||||
conf = open(self.autofetcher_conf)
|
||||
for l in conf:
|
||||
if l.startswith("maxkey="):
|
||||
settings["maxkey"] = float(l[7:].rstrip('\n'))
|
||||
conf.close()
|
||||
else:
|
||||
conf = open(self.autofetcher_conf, "w")
|
||||
conf.write("maxkey=10.0")
|
||||
conf.close()
|
||||
settings["maxkey"] = 10.0
|
||||
log.info("No autofetcher conf file found, making one with max key fee of 10.0")
|
||||
|
||||
self.max_key_fee = settings["maxkey"]
|
||||
@@ -0,0 +1,133 @@
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from twisted.internet import threads, defer
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Publisher(object):
|
||||
def __init__(self, session, lbry_file_manager, wallet):
|
||||
self.session = session
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.wallet = wallet
|
||||
self.received_file_name = False
|
||||
self.file_path = None
|
||||
self.file_name = None
|
||||
self.thumbnail = None
|
||||
self.title = None
|
||||
self.publish_name = None
|
||||
self.bid_amount = None
|
||||
self.key_fee = None
|
||||
self.key_fee_address = None
|
||||
self.key_fee_address_chosen = False
|
||||
self.description = None
|
||||
self.verified = False
|
||||
self.lbry_file = None
|
||||
self.sd_hash = None
|
||||
self.tx_hash = None
|
||||
self.content_license = None
|
||||
self.author = None
|
||||
self.sources = None
|
||||
|
||||
def start(self, name, file_path, bid, title=None, description=None, thumbnail=None,
|
||||
key_fee=None, key_fee_address=None, content_license=None, author=None, sources=None):
|
||||
|
||||
def _show_result():
|
||||
message = "[" + str(datetime.now()) + "] Published " + self.file_name + " --> lbry://" + \
|
||||
str(self.publish_name) + " with txid: " + str(self.tx_hash)
|
||||
log.info(message)
|
||||
return defer.succeed(self.tx_hash)
|
||||
|
||||
self.publish_name = name
|
||||
self.file_path = file_path
|
||||
self.bid_amount = bid
|
||||
self.title = title
|
||||
self.description = description
|
||||
self.thumbnail = thumbnail
|
||||
self.key_fee = key_fee
|
||||
self.key_fee_address = key_fee_address
|
||||
self.content_license = content_license
|
||||
self.author = author
|
||||
self.sources = sources
|
||||
|
||||
d = self._check_file_path(self.file_path)
|
||||
d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
|
||||
self.file_name, open(self.file_path)))
|
||||
d.addCallback(self.add_to_lbry_files)
|
||||
d.addCallback(lambda _: self._create_sd_blob())
|
||||
d.addCallback(lambda _: self._claim_name())
|
||||
d.addCallbacks(lambda _: _show_result(), self._show_publish_error)
|
||||
|
||||
return d
|
||||
|
||||
def _check_file_path(self, file_path):
|
||||
def check_file_threaded():
|
||||
f = open(file_path)
|
||||
f.close()
|
||||
self.file_name = os.path.basename(self.file_path)
|
||||
return True
|
||||
return threads.deferToThread(check_file_threaded)
|
||||
|
||||
def _get_new_address(self):
|
||||
d = self.wallet.get_new_address()
|
||||
|
||||
def set_address(address):
|
||||
self.key_fee_address = address
|
||||
return True
|
||||
|
||||
d.addCallback(set_address)
|
||||
return d
|
||||
|
||||
def set_status(self, lbry_file_downloader):
|
||||
self.lbry_file = lbry_file_downloader
|
||||
d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
|
||||
d.addCallback(lambda _: lbry_file_downloader.restore())
|
||||
return d
|
||||
|
||||
def add_to_lbry_files(self, stream_hash):
|
||||
prm = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
|
||||
d.addCallback(self.set_status)
|
||||
return d
|
||||
|
||||
def _create_sd_blob(self):
|
||||
d = publish_sd_blob(self.lbry_file_manager.stream_info_manager, self.session.blob_manager,
|
||||
self.lbry_file.stream_hash)
|
||||
|
||||
def set_sd_hash(sd_hash):
|
||||
self.sd_hash = sd_hash
|
||||
|
||||
d.addCallback(set_sd_hash)
|
||||
return d
|
||||
|
||||
def _claim_name(self):
|
||||
d = self.wallet.claim_name(self.publish_name, self.sd_hash, self.bid_amount,
|
||||
description=self.description, key_fee=self.key_fee,
|
||||
key_fee_address=self.key_fee_address, thumbnail=self.thumbnail,
|
||||
content_license=self.content_license, author=self.author,
|
||||
sources=self.sources)
|
||||
|
||||
def set_tx_hash(tx_hash):
|
||||
self.tx_hash = tx_hash
|
||||
|
||||
d.addCallback(set_tx_hash)
|
||||
return d
|
||||
|
||||
def _show_publish_error(self, err):
|
||||
log.info(err.getTraceback())
|
||||
message = "An error occurred publishing %s to %s. Error: %s."
|
||||
if err.check(InsufficientFundsError):
|
||||
error_message = "Insufficient funds"
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
error_message = err.getErrorMessage()
|
||||
log.error(error_message)
|
||||
log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())
|
||||
return d
|
||||
@@ -1,874 +0,0 @@
|
||||
import Tkinter as tk
|
||||
import ttk
|
||||
import tkFont
|
||||
import tkMessageBox
|
||||
import logging
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
|
||||
from twisted.internet import tksupport, reactor, defer, task, threads
|
||||
import sys
|
||||
import os
|
||||
import locale
|
||||
import binascii
|
||||
from Crypto import Random
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.lbryfile.LBRYFileMetadataManager import TempLBRYFileMetadataManager
|
||||
from lbrynet.core import StreamDescriptor
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType, LBRYFileStreamDescriptorValidator
|
||||
import requests
|
||||
|
||||
|
||||
class LBRYDownloader(object):
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.known_dht_nodes = [('104.236.42.182', 4000)]
|
||||
self.conf_dir = os.path.join(os.path.expanduser("~"), ".lbrydownloader")
|
||||
self.data_dir = os.path.join(self.conf_dir, "blobfiles")
|
||||
self.wallet_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
|
||||
self.wallet_conf = os.path.join(self.wallet_dir, "lbrycrd.conf")
|
||||
self.first_run = False
|
||||
if os.name == "nt":
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
self.download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
else:
|
||||
self.download_directory = os.getcwd()
|
||||
self.wallet_user = None
|
||||
self.wallet_password = None
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.wallet_rpc_port = 8332
|
||||
self.download_deferreds = []
|
||||
self.stream_frames = []
|
||||
|
||||
def start(self):
|
||||
d = threads.deferToThread(self._create_directory)
|
||||
d.addCallback(lambda _: self._get_session())
|
||||
d.addCallback(lambda _: self._setup_stream_info_manager())
|
||||
d.addCallback(lambda _: self._setup_stream_identifier())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
dl = defer.DeferredList(self.download_deferreds)
|
||||
for stream_frame in self.stream_frames:
|
||||
stream_frame.cancel_func()
|
||||
if self.session is not None:
|
||||
dl.addBoth(lambda _: self.session.shut_down())
|
||||
return dl
|
||||
|
||||
def get_new_address(self):
|
||||
return self.session.wallet.get_new_address()
|
||||
|
||||
def _create_directory(self):
|
||||
if not os.path.exists(self.conf_dir):
|
||||
os.makedirs(self.conf_dir)
|
||||
logging.debug("Created the configuration directory: %s", str(self.conf_dir))
|
||||
if not os.path.exists(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
logging.debug("Created the data directory: %s", str(self.data_dir))
|
||||
if not os.path.exists(self.wallet_dir):
|
||||
os.makedirs(self.wallet_dir)
|
||||
if not os.path.exists(self.wallet_conf):
|
||||
lbrycrd_conf = open(self.wallet_conf, mode='w')
|
||||
self.wallet_user = "rpcuser"
|
||||
lbrycrd_conf.write("rpcuser=%s\n" % self.wallet_user)
|
||||
self.wallet_password = binascii.hexlify(Random.new().read(20))
|
||||
lbrycrd_conf.write("rpcpassword=%s\n" % self.wallet_password)
|
||||
lbrycrd_conf.write("server=1\n")
|
||||
lbrycrd_conf.close()
|
||||
self.first_run = True
|
||||
else:
|
||||
lbrycrd_conf = open(self.wallet_conf)
|
||||
for l in lbrycrd_conf:
|
||||
if l.startswith("rpcuser="):
|
||||
self.wallet_user = l[8:-1]
|
||||
if l.startswith("rpcpassword="):
|
||||
self.wallet_password = l[12:-1]
|
||||
if l.startswith("rpcport="):
|
||||
self.wallet_rpc_port = int(l[8:-1])
|
||||
|
||||
def _get_session(self):
|
||||
wallet = LBRYcrdWallet(self.wallet_user, self.wallet_password, "127.0.0.1", self.wallet_rpc_port,
|
||||
start_lbrycrdd=True, wallet_dir=self.wallet_dir, wallet_conf=self.wallet_conf)
|
||||
self.session = LBRYSession(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=self.conf_dir, blob_dir=self.data_dir,
|
||||
use_upnp=False, wallet=wallet,
|
||||
known_dht_nodes=self.known_dht_nodes, dht_node_port=4446)
|
||||
return self.session.setup()
|
||||
|
||||
def _setup_stream_info_manager(self):
|
||||
self.stream_info_manager = TempLBRYFileMetadataManager()
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_stream_identifier(self):
|
||||
self.sd_identifier.add_stream_info_validator(LBRYFileStreamType, LBRYFileStreamDescriptorValidator)
|
||||
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
|
||||
self.session.blob_manager, self.stream_info_manager,
|
||||
self.session.wallet, self.download_directory)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_saver_factory)
|
||||
file_opener_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
|
||||
self.session.blob_manager, self.stream_info_manager,
|
||||
self.session.wallet)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_opener_factory)
|
||||
|
||||
def do_first_run(self):
|
||||
if self.first_run is True:
|
||||
d = self.session.wallet.get_new_address()
|
||||
|
||||
def send_request(url, data):
|
||||
r = requests.post(url, json=data)
|
||||
if r.status_code == 200:
|
||||
return r.json()['credits_sent']
|
||||
return 0.0
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("unable to request free credits. %s", err.getErrorMessage())
|
||||
return 0.0
|
||||
|
||||
def request_credits(address):
|
||||
url = "http://credreq.lbry.io/requestcredits"
|
||||
data = {"address": address}
|
||||
d = threads.deferToThread(send_request, url, data)
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
d.addCallback(request_credits)
|
||||
return d
|
||||
return defer.succeed(0.0)
|
||||
|
||||
def _resolve_name(self, uri):
|
||||
return self.session.wallet.get_stream_info_for_name(uri)
|
||||
|
||||
def download_stream(self, stream_frame, uri):
|
||||
resolve_d = self._resolve_name(uri)
|
||||
|
||||
stream_frame.show_metadata_status("resolving name...")
|
||||
|
||||
stream_frame.cancel_func = resolve_d.cancel
|
||||
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
|
||||
def update_stream_name(value):
|
||||
if 'name' in value:
|
||||
stream_frame.show_name(value['name'])
|
||||
if 'description' in value:
|
||||
stream_frame.show_description(value['description'])
|
||||
return value
|
||||
|
||||
def get_sd_hash(value):
|
||||
if 'stream_hash' in value:
|
||||
return value['stream_hash']
|
||||
raise ValueError("Invalid stream")
|
||||
|
||||
def get_sd_blob(sd_hash):
|
||||
stream_frame.show_metadata_status("name resolved, fetching metadata...")
|
||||
get_sd_d = StreamDescriptor.download_sd_blob(self.session, sd_hash,
|
||||
payment_rate_manager)
|
||||
get_sd_d.addCallback(self.sd_identifier.get_info_and_factories_for_sd_blob)
|
||||
get_sd_d.addCallbacks(choose_download_factory, bad_sd_blob)
|
||||
return get_sd_d
|
||||
|
||||
def get_info_from_validator(info_validator):
|
||||
stream_name = None
|
||||
stream_size = None
|
||||
for field, val in info_validator.info_to_show():
|
||||
if field == "suggested_file_name":
|
||||
stream_name = val
|
||||
elif field == "stream_name" and stream_name is None:
|
||||
stream_name = val
|
||||
elif field == "stream_size":
|
||||
stream_size = int(val)
|
||||
if stream_size is None:
|
||||
stream_size = "unknown"
|
||||
if stream_name is None:
|
||||
stream_name = "unknown"
|
||||
return stream_name, stream_size
|
||||
|
||||
def choose_download_factory(info_and_factories):
|
||||
info_validator, factories = info_and_factories
|
||||
stream_name, stream_size = get_info_from_validator(info_validator)
|
||||
if isinstance(stream_size, (int, long)):
|
||||
price = payment_rate_manager.get_effective_min_blob_data_payment_rate()
|
||||
estimated_cost = stream_size * 1.0 / 2**20 * price
|
||||
else:
|
||||
estimated_cost = "unknown"
|
||||
|
||||
stream_frame.show_stream_metadata(stream_name, stream_size, estimated_cost)
|
||||
|
||||
get_downloader_d = defer.Deferred()
|
||||
|
||||
def create_downloader(f):
|
||||
|
||||
def fire_get_downloader_d(downloader):
|
||||
if not get_downloader_d.called:
|
||||
get_downloader_d.callback(downloader)
|
||||
|
||||
stream_frame.disable_download_buttons()
|
||||
download_options = [o.default for o in f.get_downloader_options(info_validator, payment_rate_manager)]
|
||||
d = f.make_downloader(info_validator, download_options,
|
||||
payment_rate_manager)
|
||||
d.addCallback(fire_get_downloader_d)
|
||||
|
||||
for factory in factories:
|
||||
|
||||
def choose_factory(f=factory):
|
||||
create_downloader(f)
|
||||
|
||||
stream_frame.add_download_factory(factory, choose_factory)
|
||||
|
||||
get_downloader_d.addCallback(start_download)
|
||||
|
||||
return get_downloader_d
|
||||
|
||||
def show_stream_status(downloader):
|
||||
total_bytes = downloader.get_total_bytes()
|
||||
bytes_left_to_download = downloader.get_bytes_left_to_download()
|
||||
bytes_left_to_output = downloader.get_bytes_left_to_output()
|
||||
points_paid = payment_rate_manager.points_paid
|
||||
payment_rate = payment_rate_manager.get_effective_min_blob_data_payment_rate()
|
||||
points_remaining = 1.0 * bytes_left_to_download * payment_rate / 2**20
|
||||
stream_frame.show_progress(total_bytes, bytes_left_to_download, bytes_left_to_output,
|
||||
points_paid, points_remaining)
|
||||
|
||||
def show_finished(arg, downloader):
|
||||
show_stream_status(downloader)
|
||||
stream_frame.show_download_done(payment_rate_manager.points_paid)
|
||||
return arg
|
||||
|
||||
def start_download(downloader):
|
||||
l = task.LoopingCall(show_stream_status, downloader)
|
||||
l.start(1)
|
||||
d = downloader.start()
|
||||
stream_frame.cancel_func = downloader.stop
|
||||
|
||||
def stop_looping_call(arg):
|
||||
l.stop()
|
||||
stream_frame.cancel_func = resolve_d.cancel
|
||||
return arg
|
||||
|
||||
d.addBoth(stop_looping_call)
|
||||
d.addCallback(show_finished, downloader)
|
||||
return d
|
||||
|
||||
def lookup_failed(err):
|
||||
stream_frame.show_metadata_status("name lookup failed")
|
||||
return err
|
||||
|
||||
def bad_sd_blob(err):
|
||||
stream_frame.show_metadata_status("Unknown type or badly formed metadata")
|
||||
return err
|
||||
|
||||
resolve_d.addCallback(update_stream_name)
|
||||
resolve_d.addCallback(get_sd_hash)
|
||||
resolve_d.addCallbacks(get_sd_blob, lookup_failed)
|
||||
|
||||
def show_err(err):
|
||||
tkMessageBox.showerror(title="Download Error", message=err.getErrorMessage())
|
||||
logging.error(err.getErrorMessage())
|
||||
stream_frame.show_download_done(payment_rate_manager.points_paid)
|
||||
|
||||
resolve_d.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
resolve_d.addErrback(show_err)
|
||||
self._add_download_deferred(resolve_d, stream_frame)
|
||||
|
||||
def _add_download_deferred(self, d, stream_frame):
|
||||
self.download_deferreds.append(d)
|
||||
self.stream_frames.append(stream_frame)
|
||||
|
||||
def remove_from_list():
|
||||
self.download_deferreds.remove(d)
|
||||
self.stream_frames.remove(stream_frame)
|
||||
|
||||
d.addBoth(lambda _: remove_from_list())
|
||||
|
||||
|
||||
class StreamFrame(object):
|
||||
def __init__(self, app, uri):
|
||||
self.app = app
|
||||
self.uri = uri
|
||||
self.cancel_func = None
|
||||
|
||||
self.stream_frame = ttk.Frame(self.app.streams_frame, style="B.TFrame")
|
||||
|
||||
self.stream_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(30, 0))
|
||||
|
||||
self.stream_frame_header = ttk.Frame(self.stream_frame, style="C.TFrame")
|
||||
self.stream_frame_header.grid(sticky=tk.E + tk.W)
|
||||
|
||||
self.uri_font = tkFont.Font(size=8)
|
||||
self.uri_label = ttk.Label(
|
||||
self.stream_frame_header, text=self.uri, font=self.uri_font, foreground="#666666"
|
||||
)
|
||||
self.uri_label.grid(row=0, column=0, sticky=tk.W)
|
||||
|
||||
if os.name == "nt":
|
||||
close_cursor = ""
|
||||
else:
|
||||
close_cursor = "hand1"
|
||||
|
||||
close_file_name = "close2.gif"
|
||||
try:
|
||||
close_file = os.path.join(os.path.dirname(__file__), close_file_name)
|
||||
except NameError:
|
||||
close_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "lbrynet",
|
||||
"lbrynet_downloader_gui", close_file_name)
|
||||
|
||||
self.close_picture = tk.PhotoImage(
|
||||
file=close_file
|
||||
)
|
||||
self.close_button = ttk.Button(
|
||||
self.stream_frame_header, command=self.cancel, style="Stop.TButton", cursor=close_cursor
|
||||
)
|
||||
self.close_button.config(image=self.close_picture)
|
||||
self.close_button.grid(row=0, column=1, sticky=tk.E + tk.N)
|
||||
|
||||
self.stream_frame_header.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.stream_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.stream_frame_body = ttk.Frame(self.stream_frame, style="C.TFrame")
|
||||
self.stream_frame_body.grid(row=1, column=0, sticky=tk.E + tk.W)
|
||||
|
||||
self.name_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.name_frame.grid(sticky=tk.W + tk.E)
|
||||
self.name_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.stream_frame_body.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.info_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.info_frame.grid(sticky=tk.W + tk.E, row=1)
|
||||
self.info_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.metadata_frame = ttk.Frame(self.info_frame, style="E.TFrame")
|
||||
self.metadata_frame.grid(sticky=tk.W + tk.E)
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.outer_button_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.outer_button_frame.grid(sticky=tk.W + tk.E, row=2)
|
||||
|
||||
self.button_frame = ttk.Frame(self.outer_button_frame, style="E.TFrame")
|
||||
self.button_frame.pack(side=tk.TOP)
|
||||
|
||||
self.status_label = None
|
||||
self.name_label = None
|
||||
self.bytes_downloaded_label = None
|
||||
self.bytes_outputted_label = None
|
||||
|
||||
self.download_buttons = []
|
||||
self.name_font = None
|
||||
self.description_label = None
|
||||
self.file_name_frame = None
|
||||
self.cost_frame = None
|
||||
self.cost_description = None
|
||||
self.remaining_cost_description = None
|
||||
self.cost_label = None
|
||||
self.remaining_cost_label = None
|
||||
|
||||
def cancel(self):
|
||||
if self.cancel_func is not None:
|
||||
self.cancel_func()
|
||||
self.stream_frame.destroy()
|
||||
self.app.stream_removed()
|
||||
|
||||
def show_name(self, name):
|
||||
self.name_font = tkFont.Font(size=16)
|
||||
self.name_label = ttk.Label(
|
||||
self.name_frame, text=name, font=self.name_font
|
||||
)
|
||||
self.name_label.grid(row=0, column=0, sticky=tk.W)
|
||||
|
||||
def show_description(self, description):
|
||||
if os.name == "nt":
|
||||
wraplength = 580
|
||||
else:
|
||||
wraplength = 600
|
||||
self.description_label = ttk.Label(
|
||||
self.name_frame, text=description, wraplength=wraplength
|
||||
)
|
||||
self.description_label.grid(row=1, column=0, sticky=tk.W)
|
||||
|
||||
def show_metadata_status(self, value):
|
||||
if self.status_label is None:
|
||||
self.status_label = ttk.Label(
|
||||
self.metadata_frame, text=value
|
||||
)
|
||||
self.status_label.grid()
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1)
|
||||
else:
|
||||
self.status_label.config(text=value)
|
||||
|
||||
@staticmethod
|
||||
def get_formatted_stream_size(stream_size):
|
||||
if isinstance(stream_size, (int, long)):
|
||||
if stream_size >= 2**40:
|
||||
units = "TB"
|
||||
factor = 2**40
|
||||
elif stream_size >= 2**30:
|
||||
units = "GB"
|
||||
factor = 2**30
|
||||
elif stream_size >= 2**20:
|
||||
units = "MB"
|
||||
factor = 2**20
|
||||
elif stream_size >= 2**10:
|
||||
units = "KB"
|
||||
factor = 2**10
|
||||
else:
|
||||
return str(stream_size) + " B"
|
||||
return "%.1f %s" % (round((stream_size * 1.0 / factor), 1), units)
|
||||
return stream_size
|
||||
|
||||
def show_stream_metadata(self, stream_name, stream_size, estimated_cost):
|
||||
if self.status_label is not None:
|
||||
self.status_label.destroy()
|
||||
|
||||
self.file_name_frame = ttk.Frame(self.metadata_frame, style="F.TFrame")
|
||||
self.file_name_frame.grid(row=0, column=0, sticky=tk.W)
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1, uniform="metadata")
|
||||
|
||||
file_size_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=self.get_formatted_stream_size(stream_size)
|
||||
)
|
||||
file_size_label.grid(row=0, column=2)
|
||||
|
||||
file_name_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=" - " + stream_name,
|
||||
)
|
||||
file_name_label.grid(row=0, column=3)
|
||||
|
||||
self.outer_button_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.outer_button_frame.grid(sticky=tk.W + tk.E, row=2)
|
||||
|
||||
self.cost_frame = ttk.Frame(self.outer_button_frame, style="F.TFrame")
|
||||
self.cost_frame.grid(row=0, column=0, sticky=tk.W+tk.N, pady=(0, 12))
|
||||
|
||||
self.cost_label = ttk.Label(
|
||||
self.cost_frame,
|
||||
text=locale.format_string("%.2f LBC", (round(estimated_cost, 2),), grouping=True),
|
||||
foreground="red"
|
||||
)
|
||||
self.cost_label.grid(row=0, column=1, padx=(1, 0))
|
||||
|
||||
self.button_frame = ttk.Frame(self.outer_button_frame, style="E.TFrame")
|
||||
self.button_frame.grid(row=0, column=1)
|
||||
|
||||
self.outer_button_frame.grid_columnconfigure(0, weight=1, uniform="buttons")
|
||||
self.outer_button_frame.grid_columnconfigure(1, weight=2, uniform="buttons1")
|
||||
self.outer_button_frame.grid_columnconfigure(2, weight=1, uniform="buttons")
|
||||
|
||||
def add_download_factory(self, factory, download_func):
|
||||
if os.name == "nt":
|
||||
button_cursor = ""
|
||||
else:
|
||||
button_cursor = "hand1"
|
||||
download_button = ttk.Button(
|
||||
self.button_frame, text=factory.get_description(), command=download_func,
|
||||
style='LBRY.TButton', cursor=button_cursor
|
||||
)
|
||||
self.download_buttons.append(download_button)
|
||||
download_button.grid(row=0, column=len(self.download_buttons) - 1, padx=5, pady=(1, 2))
|
||||
|
||||
def disable_download_buttons(self):
|
||||
for download_button in self.download_buttons:
|
||||
download_button.config(state=tk.DISABLED)
|
||||
|
||||
def remove_download_buttons(self):
|
||||
for download_button in self.download_buttons:
|
||||
download_button.destroy()
|
||||
self.download_buttons = []
|
||||
|
||||
def show_progress(self, total_bytes, bytes_left_to_download, bytes_left_to_output, points_paid,
|
||||
points_remaining):
|
||||
if self.bytes_outputted_label is None:
|
||||
self.remove_download_buttons()
|
||||
self.button_frame.destroy()
|
||||
self.outer_button_frame.grid_columnconfigure(2, weight=0, uniform="")
|
||||
|
||||
self.bytes_outputted_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=""
|
||||
)
|
||||
self.bytes_outputted_label.grid(row=0, column=0)
|
||||
|
||||
self.bytes_downloaded_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=""
|
||||
)
|
||||
self.bytes_downloaded_label.grid(row=0, column=1)
|
||||
|
||||
if self.bytes_outputted_label.winfo_exists():
|
||||
self.bytes_outputted_label.config(
|
||||
text=self.get_formatted_stream_size(total_bytes - bytes_left_to_output) + " / "
|
||||
)
|
||||
if self.bytes_downloaded_label.winfo_exists():
|
||||
self.bytes_downloaded_label.config(
|
||||
text=self.get_formatted_stream_size(total_bytes - bytes_left_to_download) + " / "
|
||||
)
|
||||
if self.cost_label.winfo_exists():
|
||||
total_points = points_remaining + points_paid
|
||||
self.cost_label.config(text=locale.format_string("%.2f/%.2f LBC",
|
||||
(round(points_paid, 2), round(total_points, 2)),
|
||||
grouping=True))
|
||||
|
||||
def show_download_done(self, total_points_paid):
|
||||
if self.bytes_outputted_label is not None and self.bytes_outputted_label.winfo_exists():
|
||||
self.bytes_outputted_label.destroy()
|
||||
if self.bytes_downloaded_label is not None and self.bytes_downloaded_label.winfo_exists():
|
||||
self.bytes_downloaded_label.destroy()
|
||||
if self.cost_label is not None and self.cost_label.winfo_exists():
|
||||
self.cost_label.config(text=locale.format_string("%.2f LBC",
|
||||
(round(total_points_paid, 2),),
|
||||
grouping=True))
|
||||
|
||||
|
||||
class AddressWindow(object):
|
||||
def __init__(self, root, address):
|
||||
self.root = root
|
||||
self.address = address
|
||||
|
||||
def show(self):
|
||||
window = tk.Toplevel(self.root, background="#FFFFFF")
|
||||
window.transient(self.root)
|
||||
window.wm_title("New address")
|
||||
window.protocol("WM_DELETE_WINDOW", window.destroy)
|
||||
window.resizable(0, 0)
|
||||
|
||||
text_box = tk.Text(window, width=35, height=1, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0)
|
||||
text_box.insert(tk.END, self.address)
|
||||
text_box.grid(row=0, padx=10, pady=5, columnspan=2)
|
||||
text_box.config(state='normal')
|
||||
|
||||
def copy_to_clipboard():
|
||||
self.root.clipboard_clear()
|
||||
self.root.clipboard_append(text_box.get('1.0', 'end-1c'))
|
||||
|
||||
def copy_command():
|
||||
text_box.event_generate("<Control-c>")
|
||||
|
||||
copy_menu = tk.Menu(
|
||||
self.root, tearoff=0
|
||||
)
|
||||
copy_menu.add_command(label=" Copy ", command=copy_command)
|
||||
|
||||
def popup(event):
|
||||
if text_box.tag_ranges("sel"):
|
||||
copy_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
text_box.bind("<Button-3>", popup)
|
||||
|
||||
copy_button = ttk.Button(
|
||||
window, text="Copy", command=copy_to_clipboard, style="LBRY.TButton"
|
||||
)
|
||||
copy_button.grid(row=1, column=0, pady=(0, 5), padx=5, sticky=tk.E)
|
||||
|
||||
done_button = ttk.Button(
|
||||
window, text="OK", command=window.destroy, style="LBRY.TButton"
|
||||
)
|
||||
done_button.grid(row=1, column=1, pady=(0, 5), padx=5, sticky=tk.W)
|
||||
window.focus_set()
|
||||
|
||||
|
||||
class WelcomeWindow(object):
|
||||
def __init__(self, root, points_sent):
|
||||
self.root = root
|
||||
self.points_sent = points_sent
|
||||
|
||||
def show(self):
|
||||
window = tk.Toplevel(self.root, background="#FFFFFF")
|
||||
window.transient(self.root)
|
||||
window.wm_title("Welcome to LBRY")
|
||||
window.protocol("WM_DELETE_WINDOW", window.destroy)
|
||||
window.resizable(0, 0)
|
||||
|
||||
text_box = tk.Text(window, width=45, height=3, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0)
|
||||
|
||||
points_string = locale.format_string("%.2f LBC", (round(self.points_sent, 2),),
|
||||
grouping=True)
|
||||
|
||||
text_box.insert(tk.END, "Thank you for using LBRY! You have been\n"
|
||||
"given %s for free because we love\n"
|
||||
"you. Please give them 60 seconds to show up." % points_string)
|
||||
text_box.grid(row=0, padx=10, pady=5, columnspan=2)
|
||||
text_box.config(state='normal')
|
||||
|
||||
window.focus_set()
|
||||
|
||||
|
||||
class App(object):
|
||||
def __init__(self):
|
||||
self.master = None
|
||||
self.downloader = None
|
||||
self.wallet_balance_check = None
|
||||
self.streams_frame = None
|
||||
|
||||
def start(self):
|
||||
|
||||
d = defer.maybeDeferred(self._start_root)
|
||||
d.addCallback(lambda _: self._draw_main())
|
||||
d.addCallback(lambda _: self._start_downloader())
|
||||
d.addCallback(lambda _: self._start_checking_wallet_balance())
|
||||
d.addCallback(lambda _: self._enable_lookup())
|
||||
|
||||
def show_error_and_stop(err):
|
||||
logging.error(err.getErrorMessage())
|
||||
tkMessageBox.showerror(title="Start Error", message=err.getErrorMessage())
|
||||
return self.stop()
|
||||
|
||||
d.addErrback(show_error_and_stop)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
|
||||
def log_error(err):
|
||||
logging.error(err.getErrorMessage())
|
||||
|
||||
if self.downloader is not None:
|
||||
d = self.downloader.stop()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addErrback(log_error)
|
||||
d.addCallback(lambda _: self._stop_checking_wallet_balance())
|
||||
d.addErrback(log_error)
|
||||
d.addCallback(lambda _: reactor.stop())
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
def _start_root(self):
|
||||
if os.name == "nt":
|
||||
button_foreground = "#104639"
|
||||
lookup_button_padding = 10
|
||||
else:
|
||||
button_foreground = "#FFFFFF"
|
||||
lookup_button_padding = 11
|
||||
|
||||
root = tk.Tk()
|
||||
root.resizable(0, 0)
|
||||
root.wm_title("LBRY")
|
||||
|
||||
tksupport.install(root)
|
||||
|
||||
if os.name == "nt":
|
||||
root.iconbitmap(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
"lbrynet", "lbrynet_downloader_gui", "lbry-dark-icon.ico"))
|
||||
else:
|
||||
root.wm_iconbitmap("@" + os.path.join(os.path.dirname(__file__), "lbry-dark-icon.xbm"))
|
||||
|
||||
root.button_font = tkFont.Font(size=9)
|
||||
|
||||
ttk.Style().configure(".", background="#FFFFFF")
|
||||
ttk.Style().configure("LBRY.TButton", background="#104639", foreground=button_foreground,
|
||||
borderwidth=1, relief="solid", font=root.button_font)
|
||||
ttk.Style().map("LBRY.TButton",
|
||||
background=[('pressed', "#104639"),
|
||||
('active', "#104639")])
|
||||
#ttk.Style().configure("LBRY.TButton.border", background="#808080")
|
||||
ttk.Style().configure("Lookup.LBRY.TButton", padding=lookup_button_padding)
|
||||
ttk.Style().configure("Stop.TButton", padding=1, background="#FFFFFF", relief="flat", borderwidth=0)
|
||||
ttk.Style().configure("TEntry", padding=11)
|
||||
#ttk.Style().configure("A.TFrame", background="red")
|
||||
#ttk.Style().configure("B.TFrame", background="green")
|
||||
#ttk.Style().configure("B2.TFrame", background="#80FF80")
|
||||
#ttk.Style().configure("C.TFrame", background="orange")
|
||||
#ttk.Style().configure("D.TFrame", background="blue")
|
||||
#ttk.Style().configure("E.TFrame", background="yellow")
|
||||
#ttk.Style().configure("F.TFrame", background="#808080")
|
||||
#ttk.Style().configure("LBRY.TProgressbar", background="#104639", orient="horizontal", thickness=5)
|
||||
#ttk.Style().configure("LBRY.TProgressbar")
|
||||
#ttk.Style().layout("Horizontal.LBRY.TProgressbar", ttk.Style().layout("Horizontal.TProgressbar"))
|
||||
|
||||
root.configure(background="#FFFFFF")
|
||||
|
||||
root.protocol("WM_DELETE_WINDOW", self.stop)
|
||||
|
||||
self.master = root
|
||||
|
||||
def _draw_main(self):
|
||||
self.frame = ttk.Frame(self.master, style="A.TFrame")
|
||||
self.frame.grid(padx=20, pady=20)
|
||||
|
||||
logo_file_name = "lbry-dark-242x80.gif"
|
||||
try:
|
||||
logo_file = os.path.join(os.path.dirname(__file__), logo_file_name)
|
||||
except NameError:
|
||||
logo_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "lbrynet",
|
||||
"lbrynet_downloader_gui", logo_file_name)
|
||||
|
||||
self.logo_picture = tk.PhotoImage(file=logo_file)
|
||||
|
||||
self.logo_frame = ttk.Frame(self.frame, style="B.TFrame")
|
||||
self.logo_frame.grid(pady=5, sticky=tk.W + tk.E)
|
||||
|
||||
self.dummy_frame = ttk.Frame(self.logo_frame, style="C.TFrame") # keeps the logo in the middle
|
||||
self.dummy_frame.grid(row=0, column=1, padx=5)
|
||||
|
||||
self.logo_label = ttk.Label(self.logo_frame, image=self.logo_picture)
|
||||
self.logo_label.grid(row=0, column=1, padx=5)
|
||||
|
||||
self.wallet_balance_frame = ttk.Frame(self.logo_frame, style="C.TFrame")
|
||||
self.wallet_balance_frame.grid(sticky=tk.E + tk.N, row=0, column=2)
|
||||
|
||||
self.logo_frame.grid_columnconfigure(0, weight=1, uniform="a")
|
||||
self.logo_frame.grid_columnconfigure(1, weight=2, uniform="b")
|
||||
self.logo_frame.grid_columnconfigure(2, weight=1, uniform="a")
|
||||
|
||||
self.wallet_balance = ttk.Label(
|
||||
self.wallet_balance_frame,
|
||||
text=" -- LBC"
|
||||
)
|
||||
self.wallet_balance.grid(row=0, column=0)
|
||||
|
||||
dropdown_file_name = "drop_down.gif"
|
||||
try:
|
||||
dropdown_file = os.path.join(os.path.dirname(__file__), dropdown_file_name)
|
||||
except NameError:
|
||||
dropdown_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "lbrynet",
|
||||
"lbrynet_downloader_gui", dropdown_file_name)
|
||||
|
||||
self.dropdown_picture = tk.PhotoImage(
|
||||
file=dropdown_file
|
||||
)
|
||||
|
||||
def get_new_address():
|
||||
def show_address(address):
|
||||
w = AddressWindow(self.master, address)
|
||||
w.show()
|
||||
d = defer.maybeDeferred(self.downloader.get_new_address)
|
||||
d.addCallback(show_address)
|
||||
|
||||
def show_error(err):
|
||||
tkMessageBox.showerror(title="Failed to get new address", message=err.getErrorMessage())
|
||||
|
||||
d.addErrback(show_error)
|
||||
|
||||
self.wallet_menu = tk.Menu(
|
||||
self.master, tearoff=0
|
||||
)
|
||||
self.wallet_menu.add_command(label="Get new LBRYcrd address", command=get_new_address)
|
||||
|
||||
if os.name == "nt":
|
||||
button_cursor = ""
|
||||
else:
|
||||
button_cursor = "hand1"
|
||||
|
||||
self.wallet_menu_button = ttk.Button(self.wallet_balance_frame, image=self.dropdown_picture,
|
||||
style="Stop.TButton", cursor=button_cursor)
|
||||
self.wallet_menu_button.grid(row=0, column=1, padx=(5, 0))
|
||||
|
||||
def popup(event):
|
||||
self.wallet_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
self.wallet_menu_button.bind("<Button-1>", popup)
|
||||
|
||||
self.uri_frame = ttk.Frame(self.frame, style="B.TFrame")
|
||||
self.uri_frame.grid()
|
||||
|
||||
self.uri_label = ttk.Label(
|
||||
self.uri_frame, text="lbry://"
|
||||
)
|
||||
self.uri_label.grid(row=0, column=0, sticky=tk.E, pady=2)
|
||||
|
||||
self.entry_font = tkFont.Font(size=11)
|
||||
|
||||
self.uri_entry = ttk.Entry(self.uri_frame, width=50, foreground="#222222", font=self.entry_font)
|
||||
self.uri_entry.grid(row=0, column=1, padx=2, pady=2)
|
||||
|
||||
def copy_command():
|
||||
self.uri_entry.event_generate('<Control-c>')
|
||||
|
||||
def cut_command():
|
||||
self.uri_entry.event_generate('<Control-x>')
|
||||
|
||||
def paste_command():
|
||||
self.uri_entry.event_generate('<Control-v>')
|
||||
|
||||
def popup(event):
|
||||
selection_menu = tk.Menu(
|
||||
self.master, tearoff=0
|
||||
)
|
||||
if self.uri_entry.selection_present():
|
||||
selection_menu.add_command(label=" Cut ", command=cut_command)
|
||||
selection_menu.add_command(label=" Copy ", command=copy_command)
|
||||
selection_menu.add_command(label=" Paste ", command=paste_command)
|
||||
selection_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
self.uri_entry.bind("<Button-3>", popup)
|
||||
|
||||
self.uri_button = ttk.Button(
|
||||
self.uri_frame, text="Go", command=self._open_stream,
|
||||
style='Lookup.LBRY.TButton', cursor=button_cursor
|
||||
)
|
||||
self.uri_button.grid(row=0, column=2, pady=2, padx=0)
|
||||
|
||||
def _start_downloader(self):
|
||||
self.downloader = LBRYDownloader()
|
||||
d = self.downloader.start()
|
||||
d.addCallback(lambda _: self.downloader.do_first_run())
|
||||
d.addCallback(self._show_welcome_message)
|
||||
return d
|
||||
|
||||
def _show_welcome_message(self, points_sent):
|
||||
if points_sent != 0.0:
|
||||
w = WelcomeWindow(self.master, points_sent)
|
||||
w.show()
|
||||
|
||||
def stream_removed(self):
|
||||
if self.streams_frame is not None:
|
||||
if len(self.streams_frame.winfo_children()) == 0:
|
||||
self.streams_frame.destroy()
|
||||
self.streams_frame = None
|
||||
|
||||
def _start_checking_wallet_balance(self):
|
||||
|
||||
def set_balance(balance):
|
||||
self.wallet_balance.configure(text=locale.format_string("%.2f LBC", (round(balance, 2),),
|
||||
grouping=True))
|
||||
|
||||
def update_balance():
|
||||
balance = self.downloader.session.wallet.get_available_balance()
|
||||
set_balance(balance)
|
||||
|
||||
def start_looping_call():
|
||||
self.wallet_balance_check = task.LoopingCall(update_balance)
|
||||
self.wallet_balance_check.start(5)
|
||||
|
||||
d = self.downloader.session.wallet.get_balance()
|
||||
d.addCallback(set_balance)
|
||||
d.addCallback(lambda _: start_looping_call())
|
||||
|
||||
def _stop_checking_wallet_balance(self):
|
||||
if self.wallet_balance_check is not None:
|
||||
self.wallet_balance_check.stop()
|
||||
|
||||
def _enable_lookup(self):
|
||||
self.uri_entry.bind('<Return>', self._open_stream)
|
||||
|
||||
def _open_stream(self, event=None):
|
||||
if self.streams_frame is None:
|
||||
self.streams_frame = ttk.Frame(self.frame, style="B2.TFrame")
|
||||
self.streams_frame.grid(sticky=tk.E + tk.W)
|
||||
uri = self.uri_entry.get()
|
||||
self.uri_entry.delete(0, tk.END)
|
||||
stream_frame = StreamFrame(self, "lbry://" + uri)
|
||||
|
||||
self.downloader.download_stream(stream_frame, uri)
|
||||
|
||||
|
||||
def start_downloader():
|
||||
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="downloader.log")
|
||||
sys.stdout = open("downloader.out.log", 'w')
|
||||
sys.stderr = open("downloader.err.log", 'w')
|
||||
|
||||
locale.setlocale(locale.LC_ALL, '')
|
||||
|
||||
app = App()
|
||||
|
||||
d = task.deferLater(reactor, 0, app.start)
|
||||
|
||||
reactor.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_downloader()
|
||||
@@ -0,0 +1,352 @@
|
||||
import Tkinter as tk
|
||||
import logging
|
||||
import sys
|
||||
import tkFont
|
||||
import tkMessageBox
|
||||
import ttk
|
||||
from lbrynet.lbrynet_gui.LBRYGui import LBRYDownloader
|
||||
from lbrynet.lbrynet_gui.StreamFrame import StreamFrame
|
||||
import locale
|
||||
import os
|
||||
from twisted.internet import defer, reactor, tksupport, task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloaderApp(object):
|
||||
def __init__(self):
|
||||
self.master = None
|
||||
self.downloader = None
|
||||
self.wallet_balance_check = None
|
||||
self.streams_frame = None
|
||||
|
||||
def start(self):
|
||||
|
||||
d = defer.maybeDeferred(self._start_root)
|
||||
d.addCallback(lambda _: self._draw_main())
|
||||
d.addCallback(lambda _: self._start_downloader())
|
||||
d.addCallback(lambda _: self._start_checking_wallet_balance())
|
||||
d.addCallback(lambda _: self._enable_lookup())
|
||||
|
||||
def show_error_and_stop(err):
|
||||
log.error(err.getErrorMessage())
|
||||
tkMessageBox.showerror(title="Start Error", message=err.getErrorMessage())
|
||||
return self.stop()
|
||||
|
||||
d.addErrback(show_error_and_stop)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
|
||||
def log_error(err):
|
||||
log.error(err.getErrorMessage())
|
||||
|
||||
if self.downloader is not None:
|
||||
d = self.downloader.stop()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addErrback(log_error)
|
||||
d.addCallback(lambda _: self._stop_checking_wallet_balance())
|
||||
d.addErrback(log_error)
|
||||
d.addCallback(lambda _: reactor.stop())
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
def _start_root(self):
|
||||
if os.name == "nt":
|
||||
button_foreground = "#104639"
|
||||
lookup_button_padding = 10
|
||||
else:
|
||||
button_foreground = "#FFFFFF"
|
||||
lookup_button_padding = 11
|
||||
|
||||
root = tk.Tk()
|
||||
root.resizable(0, 0)
|
||||
root.wm_title("LBRY")
|
||||
|
||||
tksupport.install(root)
|
||||
|
||||
if os.name == "nt":
|
||||
root.iconbitmap(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
"lbry-dark-icon.ico"))
|
||||
else:
|
||||
root.wm_iconbitmap("@" + os.path.join(os.path.dirname(__file__), "lbry-dark-icon.xbm"))
|
||||
|
||||
root.button_font = tkFont.Font(size=9)
|
||||
|
||||
ttk.Style().configure(".", background="#FFFFFF")
|
||||
ttk.Style().configure("LBRY.TButton", background="#104639", foreground=button_foreground,
|
||||
borderwidth=1, relief="solid", font=root.button_font)
|
||||
ttk.Style().map("LBRY.TButton",
|
||||
background=[('pressed', "#104639"),
|
||||
('active', "#104639")])
|
||||
#ttk.Style().configure("LBRY.TButton.border", background="#808080")
|
||||
ttk.Style().configure("Lookup.LBRY.TButton", padding=lookup_button_padding)
|
||||
ttk.Style().configure("Stop.TButton", padding=1, background="#FFFFFF", relief="flat", borderwidth=0)
|
||||
ttk.Style().configure("TEntry", padding=11)
|
||||
ttk.Style().configure("Float.TEntry", padding=2)
|
||||
#ttk.Style().configure("A.TFrame", background="red")
|
||||
#ttk.Style().configure("B.TFrame", background="green")
|
||||
#ttk.Style().configure("B2.TFrame", background="#80FF80")
|
||||
#ttk.Style().configure("C.TFrame", background="orange")
|
||||
#ttk.Style().configure("D.TFrame", background="blue")
|
||||
#ttk.Style().configure("E.TFrame", background="yellow")
|
||||
#ttk.Style().configure("F.TFrame", background="#808080")
|
||||
#ttk.Style().configure("G.TFrame", background="#FF80FF")
|
||||
#ttk.Style().configure("H.TFrame", background="#0080FF")
|
||||
#ttk.Style().configure("LBRY.TProgressbar", background="#104639", orient="horizontal", thickness=5)
|
||||
#ttk.Style().configure("LBRY.TProgressbar")
|
||||
#ttk.Style().layout("Horizontal.LBRY.TProgressbar", ttk.Style().layout("Horizontal.TProgressbar"))
|
||||
|
||||
root.configure(background="#FFFFFF")
|
||||
|
||||
root.protocol("WM_DELETE_WINDOW", self.stop)
|
||||
|
||||
self.master = root
|
||||
|
||||
def _draw_main(self):
|
||||
self.frame = ttk.Frame(self.master, style="A.TFrame")
|
||||
self.frame.grid(padx=20, pady=20)
|
||||
|
||||
logo_file_name = "lbry-dark-242x80.gif"
|
||||
if os.name == "nt":
|
||||
logo_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), logo_file_name)
|
||||
else:
|
||||
logo_file = os.path.join(os.path.dirname(__file__), logo_file_name)
|
||||
|
||||
self.logo_picture = tk.PhotoImage(file=logo_file)
|
||||
|
||||
self.logo_frame = ttk.Frame(self.frame, style="B.TFrame")
|
||||
self.logo_frame.grid(pady=5, sticky=tk.W + tk.E)
|
||||
|
||||
self.dummy_frame = ttk.Frame(self.logo_frame, style="C.TFrame") # keeps the logo in the middle
|
||||
self.dummy_frame.grid(row=0, column=1, padx=5)
|
||||
|
||||
self.logo_label = ttk.Label(self.logo_frame, image=self.logo_picture)
|
||||
self.logo_label.grid(row=0, column=1, padx=5)
|
||||
|
||||
self.wallet_balance_frame = ttk.Frame(self.logo_frame, style="C.TFrame")
|
||||
self.wallet_balance_frame.grid(sticky=tk.E + tk.N, row=0, column=2)
|
||||
|
||||
self.logo_frame.grid_columnconfigure(0, weight=1, uniform="a")
|
||||
self.logo_frame.grid_columnconfigure(1, weight=2, uniform="b")
|
||||
self.logo_frame.grid_columnconfigure(2, weight=1, uniform="a")
|
||||
|
||||
self.wallet_balance = ttk.Label(
|
||||
self.wallet_balance_frame,
|
||||
text=" -- LBC"
|
||||
)
|
||||
self.wallet_balance.grid(row=0, column=0)
|
||||
|
||||
dropdown_file_name = "drop_down.gif"
|
||||
if os.name == "nt":
|
||||
dropdown_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
dropdown_file_name)
|
||||
else:
|
||||
dropdown_file = os.path.join(os.path.dirname(__file__), dropdown_file_name)
|
||||
|
||||
self.dropdown_picture = tk.PhotoImage(
|
||||
file=dropdown_file
|
||||
)
|
||||
|
||||
def get_new_address():
|
||||
def show_address(address):
|
||||
w = AddressWindow(self.master, address)
|
||||
w.show()
|
||||
d = defer.maybeDeferred(self.downloader.get_new_address)
|
||||
d.addCallback(show_address)
|
||||
|
||||
def show_error(err):
|
||||
tkMessageBox.showerror(title="Failed to get new address", message=err.getErrorMessage())
|
||||
|
||||
d.addErrback(show_error)
|
||||
|
||||
self.wallet_menu = tk.Menu(
|
||||
self.master, tearoff=0
|
||||
)
|
||||
self.wallet_menu.add_command(label="Get new LBRYcrd address", command=get_new_address)
|
||||
|
||||
if os.name == "nt":
|
||||
button_cursor = ""
|
||||
else:
|
||||
button_cursor = "hand1"
|
||||
|
||||
self.wallet_menu_button = ttk.Button(self.wallet_balance_frame, image=self.dropdown_picture,
|
||||
style="Stop.TButton", cursor=button_cursor)
|
||||
self.wallet_menu_button.grid(row=0, column=1, padx=(5, 0))
|
||||
|
||||
def popup(event):
|
||||
self.wallet_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
self.wallet_menu_button.bind("<Button-1>", popup)
|
||||
|
||||
self.uri_frame = ttk.Frame(self.frame, style="B.TFrame")
|
||||
self.uri_frame.grid()
|
||||
|
||||
self.uri_label = ttk.Label(
|
||||
self.uri_frame, text="lbry://"
|
||||
)
|
||||
self.uri_label.grid(row=0, column=0, sticky=tk.E, pady=2)
|
||||
|
||||
self.entry_font = tkFont.Font(size=11)
|
||||
|
||||
self.uri_entry = ttk.Entry(self.uri_frame, width=50, foreground="#222222", font=self.entry_font,
|
||||
state=tk.DISABLED)
|
||||
self.uri_entry.grid(row=0, column=1, padx=2, pady=2)
|
||||
|
||||
def copy_command():
|
||||
self.uri_entry.event_generate('<Control-c>')
|
||||
|
||||
def cut_command():
|
||||
self.uri_entry.event_generate('<Control-x>')
|
||||
|
||||
def paste_command():
|
||||
self.uri_entry.event_generate('<Control-v>')
|
||||
|
||||
def popup(event):
|
||||
selection_menu = tk.Menu(
|
||||
self.master, tearoff=0
|
||||
)
|
||||
if self.uri_entry.selection_present():
|
||||
selection_menu.add_command(label=" Cut ", command=cut_command)
|
||||
selection_menu.add_command(label=" Copy ", command=copy_command)
|
||||
selection_menu.add_command(label=" Paste ", command=paste_command)
|
||||
selection_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
self.uri_entry.bind("<Button-3>", popup)
|
||||
|
||||
self.uri_button = ttk.Button(
|
||||
self.uri_frame, text="Go", command=self._open_stream,
|
||||
style='Lookup.LBRY.TButton', cursor=button_cursor
|
||||
)
|
||||
self.uri_button.grid(row=0, column=2, pady=2, padx=0)
|
||||
|
||||
def _start_downloader(self):
|
||||
self.downloader = LBRYDownloader()
|
||||
d = self.downloader.start()
|
||||
d.addCallback(lambda _: self.downloader.check_first_run())
|
||||
d.addCallback(self._show_welcome_message)
|
||||
return d
|
||||
|
||||
def _show_welcome_message(self, points_sent):
|
||||
if points_sent != 0.0:
|
||||
w = WelcomeWindow(self.master, points_sent)
|
||||
w.show()
|
||||
|
||||
def stream_removed(self):
|
||||
if self.streams_frame is not None:
|
||||
if len(self.streams_frame.winfo_children()) == 0:
|
||||
self.streams_frame.destroy()
|
||||
self.streams_frame = None
|
||||
|
||||
def _start_checking_wallet_balance(self):
|
||||
|
||||
def set_balance(balance):
|
||||
self.wallet_balance.configure(text=locale.format_string("%.2f LBC", (round(balance, 2),),
|
||||
grouping=True))
|
||||
|
||||
def update_balance():
|
||||
balance = self.downloader.session.wallet.get_available_balance()
|
||||
set_balance(balance)
|
||||
|
||||
def start_looping_call():
|
||||
self.wallet_balance_check = task.LoopingCall(update_balance)
|
||||
self.wallet_balance_check.start(5)
|
||||
|
||||
d = self.downloader.session.wallet.get_balance()
|
||||
d.addCallback(set_balance)
|
||||
d.addCallback(lambda _: start_looping_call())
|
||||
|
||||
def _stop_checking_wallet_balance(self):
|
||||
if self.wallet_balance_check is not None:
|
||||
self.wallet_balance_check.stop()
|
||||
|
||||
def _enable_lookup(self):
|
||||
self.uri_entry.bind('<Return>', self._open_stream)
|
||||
self.uri_entry.config(state=tk.NORMAL)
|
||||
|
||||
def _open_stream(self, event=None):
|
||||
if self.streams_frame is None:
|
||||
self.streams_frame = ttk.Frame(self.frame, style="B2.TFrame")
|
||||
self.streams_frame.grid(sticky=tk.E + tk.W)
|
||||
uri = self.uri_entry.get()
|
||||
self.uri_entry.delete(0, tk.END)
|
||||
stream_frame = StreamFrame(self, "lbry://" + uri)
|
||||
|
||||
self.downloader.download_stream(stream_frame, uri)
|
||||
|
||||
|
||||
class WelcomeWindow(object):
|
||||
def __init__(self, root, points_sent):
|
||||
self.root = root
|
||||
self.points_sent = points_sent
|
||||
|
||||
def show(self):
|
||||
window = tk.Toplevel(self.root, background="#FFFFFF")
|
||||
window.transient(self.root)
|
||||
window.wm_title("Welcome to LBRY")
|
||||
window.protocol("WM_DELETE_WINDOW", window.destroy)
|
||||
window.resizable(0, 0)
|
||||
|
||||
text_box = tk.Text(window, width=45, height=3, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0)
|
||||
|
||||
points_string = locale.format_string("%.2f LBC", (round(self.points_sent, 2),),
|
||||
grouping=True)
|
||||
|
||||
text_box.insert(tk.END, "Thank you for using LBRY! You have been\n"
|
||||
"given %s for free because we love\n"
|
||||
"you. Please give them 60 seconds to show up." % points_string)
|
||||
text_box.grid(row=0, padx=10, pady=5, columnspan=2)
|
||||
text_box.config(state='normal')
|
||||
|
||||
window.focus_set()
|
||||
|
||||
|
||||
class AddressWindow(object):
|
||||
def __init__(self, root, address):
|
||||
self.root = root
|
||||
self.address = address
|
||||
|
||||
def show(self):
|
||||
window = tk.Toplevel(self.root, background="#FFFFFF")
|
||||
window.transient(self.root)
|
||||
window.wm_title("New address")
|
||||
window.protocol("WM_DELETE_WINDOW", window.destroy)
|
||||
window.resizable(0, 0)
|
||||
|
||||
text_box = tk.Text(window, width=35, height=1, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0)
|
||||
text_box.insert(tk.END, self.address)
|
||||
text_box.grid(row=0, padx=10, pady=5, columnspan=2)
|
||||
text_box.config(state='normal')
|
||||
|
||||
def copy_to_clipboard():
|
||||
self.root.clipboard_clear()
|
||||
self.root.clipboard_append(text_box.get('1.0', 'end-1c'))
|
||||
|
||||
def copy_command():
|
||||
text_box.event_generate("<Control-c>")
|
||||
|
||||
copy_menu = tk.Menu(
|
||||
self.root, tearoff=0
|
||||
)
|
||||
copy_menu.add_command(label=" Copy ", command=copy_command)
|
||||
|
||||
def popup(event):
|
||||
if text_box.tag_ranges("sel"):
|
||||
copy_menu.tk_popup(event.x_root, event.y_root)
|
||||
|
||||
text_box.bind("<Button-3>", popup)
|
||||
|
||||
copy_button = ttk.Button(
|
||||
window, text="Copy", command=copy_to_clipboard, style="LBRY.TButton"
|
||||
)
|
||||
copy_button.grid(row=1, column=0, pady=(0, 5), padx=5, sticky=tk.E)
|
||||
|
||||
done_button = ttk.Button(
|
||||
window, text="OK", command=window.destroy, style="LBRY.TButton"
|
||||
)
|
||||
done_button.grid(row=1, column=1, pady=(0, 5), padx=5, sticky=tk.W)
|
||||
window.focus_set()
|
||||
@@ -0,0 +1,571 @@
|
||||
import binascii
|
||||
import logging
|
||||
import tkMessageBox
|
||||
from Crypto import Random
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE
|
||||
from lbrynet.core import StreamDescriptor
|
||||
from lbrynet.core.Error import UnknownNameError, UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.Error import InvalidStreamInfoError
|
||||
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.lbryfile.LBRYFileMetadataManager import TempLBRYFileMetadataManager
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory
|
||||
from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
|
||||
import os
|
||||
import requests
|
||||
import shutil
|
||||
import sys
|
||||
from twisted.internet import threads, defer, task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYDownloader(object):
|
||||
def __init__(self):
|
||||
self.session = None
|
||||
self.known_dht_nodes = [('104.236.42.182', 4000)]
|
||||
self.db_dir = os.path.join(os.path.expanduser("~"), ".lbrydownloader")
|
||||
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
|
||||
self.peer_port = 3333
|
||||
self.dht_node_port = 4444
|
||||
self.run_server = True
|
||||
self.first_run = False
|
||||
self.current_db_revision = 1
|
||||
if os.name == "nt":
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
self.download_directory = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbrycrd")
|
||||
else:
|
||||
if sys.platform == 'darwin':
|
||||
self.download_directory = os.path.join(os.path.expanduser("~"), "Downloads")
|
||||
self.wallet_dir = os.path.join(os.path.expanduser("~"), "Library/Application Support/lbrycrd")
|
||||
else:
|
||||
self.download_directory = os.getcwd()
|
||||
self.wallet_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
|
||||
self.wallet_conf = os.path.join(self.wallet_dir, "lbrycrd.conf")
|
||||
self.wallet_user = None
|
||||
self.wallet_password = None
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.wallet_rpc_port = 8332
|
||||
self.download_deferreds = []
|
||||
self.stream_frames = []
|
||||
self.default_blob_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
|
||||
self.use_upnp = True
|
||||
self.start_lbrycrdd = True
|
||||
if os.name == "nt":
|
||||
self.lbrycrdd_path = "lbrycrdd.exe"
|
||||
else:
|
||||
self.lbrycrdd_path = None
|
||||
self.default_lbrycrdd_path = "./lbrycrdd"
|
||||
self.delete_blobs_on_remove = True
|
||||
self.blob_request_payment_rate_manager = None
|
||||
|
||||
def start(self):
|
||||
d = self._load_conf_options()
|
||||
d.addCallback(lambda _: threads.deferToThread(self._create_directory))
|
||||
d.addCallback(lambda _: self._check_db_migration())
|
||||
d.addCallback(lambda _: self._get_session())
|
||||
d.addCallback(lambda _: self._setup_stream_info_manager())
|
||||
d.addCallback(lambda _: self._setup_stream_identifier())
|
||||
d.addCallback(lambda _: self.start_server())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
dl = defer.DeferredList(self.download_deferreds)
|
||||
for stream_frame in self.stream_frames:
|
||||
stream_frame.cancel_func()
|
||||
if self.session is not None:
|
||||
dl.addBoth(lambda _: self.stop_server())
|
||||
dl.addBoth(lambda _: self.session.shut_down())
|
||||
return dl
|
||||
|
||||
def get_new_address(self):
|
||||
return self.session.wallet.get_new_address()
|
||||
|
||||
def _check_db_migration(self):
|
||||
old_revision = 0
|
||||
db_revision_file = os.path.join(self.db_dir, "db_revision")
|
||||
if os.path.exists(db_revision_file):
|
||||
old_revision = int(open(db_revision_file).read().strip())
|
||||
if old_revision < self.current_db_revision:
|
||||
if os.name == "nt":
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
def run_migrator():
|
||||
migrator_exe = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
"dbmigrator", "migrator.exe")
|
||||
print "trying to find the migrator at", migrator_exe
|
||||
si = subprocess.STARTUPINFO
|
||||
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
||||
si.wShowWindow = subprocess.SW_HIDE
|
||||
print "trying to run the migrator"
|
||||
migrator_proc = subprocess.Popen([migrator_exe, self.db_dir, str(old_revision),
|
||||
str(self.current_db_revision)], startupinfo=si)
|
||||
print "started the migrator"
|
||||
migrator_proc.wait()
|
||||
print "migrator has returned"
|
||||
|
||||
return threads.deferToThread(run_migrator)
|
||||
else:
|
||||
from lbrynet.db_migrator import dbmigrator
|
||||
return threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision,
|
||||
self.current_db_revision)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _load_conf_options(self):
|
||||
|
||||
def get_lbrycrdd_path_conf_file():
|
||||
if os.name == "nt":
|
||||
return ""
|
||||
lbrycrdd_path_conf_path = os.path.join(os.path.expanduser("~"), ".lbrycrddpath.conf")
|
||||
if not os.path.exists(lbrycrdd_path_conf_path):
|
||||
return ""
|
||||
lbrycrdd_path_conf = open(lbrycrdd_path_conf_path)
|
||||
lines = lbrycrdd_path_conf.readlines()
|
||||
return lines
|
||||
|
||||
d = threads.deferToThread(get_lbrycrdd_path_conf_file)
|
||||
|
||||
def load_lbrycrdd_path(conf):
|
||||
for line in conf:
|
||||
if len(line.strip()) and line.strip()[0] != "#":
|
||||
self.lbrycrdd_path = line.strip()
|
||||
|
||||
d.addCallback(load_lbrycrdd_path)
|
||||
|
||||
def get_configuration_file():
|
||||
if os.name == "nt":
|
||||
lbry_conf_path = "lbry.conf"
|
||||
if not os.path.exists(lbry_conf_path):
|
||||
log.debug("Could not read lbry.conf")
|
||||
return ""
|
||||
else:
|
||||
lbry_conf_path = os.path.join(os.path.expanduser("~"), ".lbrynetgui.conf")
|
||||
if not os.path.exists(lbry_conf_path):
|
||||
clean_conf_path = os.path.join(os.path.dirname(__file__), "lbry.conf")
|
||||
shutil.copy(clean_conf_path, lbry_conf_path)
|
||||
lbry_conf = open(lbry_conf_path)
|
||||
log.debug("Loading configuration options from %s", lbry_conf_path)
|
||||
lines = lbry_conf.readlines()
|
||||
log.debug("%s file contents:\n%s", lbry_conf_path, str(lines))
|
||||
return lines
|
||||
|
||||
d.addCallback(lambda _: threads.deferToThread(get_configuration_file))
|
||||
|
||||
def load_configuration_file(conf):
|
||||
for line in conf:
|
||||
if len(line.strip()) and line.strip()[0] != "#":
|
||||
try:
|
||||
field_name, field_value = map(lambda x: x.strip(), line.strip().split("=", 1))
|
||||
field_name = field_name.lower()
|
||||
except ValueError:
|
||||
raise ValueError("Invalid configuration line: %s" % line)
|
||||
if field_name == "known_dht_nodes":
|
||||
known_nodes = []
|
||||
nodes = field_value.split(",")
|
||||
for n in nodes:
|
||||
if n.strip():
|
||||
try:
|
||||
ip_address, port_string = map(lambda x: x.strip(), n.split(":"))
|
||||
ip_numbers = ip_address.split(".")
|
||||
assert len(ip_numbers) == 4
|
||||
for ip_num in ip_numbers:
|
||||
num = int(ip_num)
|
||||
assert 0 <= num <= 255
|
||||
known_nodes.append((ip_address, int(port_string)))
|
||||
except (ValueError, AssertionError):
|
||||
raise ValueError("Expected known nodes in format 192.168.1.1:4000,192.168.1.2:4001. Got %s" % str(field_value))
|
||||
log.debug("Setting known_dht_nodes to %s", str(known_nodes))
|
||||
self.known_dht_nodes = known_nodes
|
||||
elif field_name == "run_server":
|
||||
if field_value.lower() == "true":
|
||||
run_server = True
|
||||
elif field_value.lower() == "false":
|
||||
run_server = False
|
||||
else:
|
||||
raise ValueError("run_server must be set to True or False. Got %s" % field_value)
|
||||
log.debug("Setting run_server to %s", str(run_server))
|
||||
self.run_server = run_server
|
||||
elif field_name == "data_dir":
|
||||
log.debug("Setting data_dir to %s", str(field_value))
|
||||
self.db_dir = field_value
|
||||
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
|
||||
elif field_name == "wallet_dir":
|
||||
log.debug("Setting wallet_dir to %s", str(field_value))
|
||||
self.wallet_dir = field_value
|
||||
elif field_name == "wallet_conf":
|
||||
log.debug("Setting wallet_conf to %s", str(field_value))
|
||||
self.wallet_conf = field_value
|
||||
elif field_name == "peer_port":
|
||||
try:
|
||||
peer_port = int(field_value)
|
||||
assert 0 <= peer_port <= 65535
|
||||
log.debug("Setting peer_port to %s", str(peer_port))
|
||||
self.peer_port = peer_port
|
||||
except (ValueError, AssertionError):
|
||||
raise ValueError("peer_port must be set to an integer between 1 and 65535. Got %s" % field_value)
|
||||
elif field_name == "dht_port":
|
||||
try:
|
||||
dht_port = int(field_value)
|
||||
assert 0 <= dht_port <= 65535
|
||||
log.debug("Setting dht_node_port to %s", str(dht_port))
|
||||
self.dht_node_port = dht_port
|
||||
except (ValueError, AssertionError):
|
||||
raise ValueError("dht_port must be set to an integer between 1 and 65535. Got %s" % field_value)
|
||||
elif field_name == "use_upnp":
|
||||
if field_value.lower() == "true":
|
||||
use_upnp = True
|
||||
elif field_value.lower() == "false":
|
||||
use_upnp = False
|
||||
else:
|
||||
raise ValueError("use_upnp must be set to True or False. Got %s" % str(field_value))
|
||||
log.debug("Setting use_upnp to %s", str(use_upnp))
|
||||
self.use_upnp = use_upnp
|
||||
elif field_name == "default_blob_data_payment_rate":
|
||||
try:
|
||||
rate = float(field_value)
|
||||
assert rate >= 0.0
|
||||
log.debug("Setting default_blob_data_payment_rate to %s", str(rate))
|
||||
self.default_blob_data_payment_rate = rate
|
||||
except (ValueError, AssertionError):
|
||||
raise ValueError("default_blob_data_payment_rate must be a positive floating point number, e.g. 0.5. Got %s" % str(field_value))
|
||||
elif field_name == "start_lbrycrdd":
|
||||
if field_value.lower() == "true":
|
||||
start_lbrycrdd = True
|
||||
elif field_value.lower() == "false":
|
||||
start_lbrycrdd = False
|
||||
else:
|
||||
raise ValueError("start_lbrycrdd must be set to True or False. Got %s" % field_value)
|
||||
log.debug("Setting start_lbrycrdd to %s", str(start_lbrycrdd))
|
||||
self.start_lbrycrdd = start_lbrycrdd
|
||||
elif field_name == "lbrycrdd_path":
|
||||
self.lbrycrdd_path = field_value
|
||||
elif field_name == "download_directory":
|
||||
log.debug("Setting download_directory to %s", str(field_value))
|
||||
self.download_directory = field_value
|
||||
elif field_name == "delete_blobs_on_stream_remove":
|
||||
if field_value.lower() == "true":
|
||||
self.delete_blobs_on_remove = True
|
||||
elif field_value.lower() == "false":
|
||||
self.delete_blobs_on_remove = False
|
||||
else:
|
||||
raise ValueError("delete_blobs_on_stream_remove must be set to True or False")
|
||||
else:
|
||||
log.warning("Got unknown configuration field: %s", field_name)
|
||||
|
||||
d.addCallback(load_configuration_file)
|
||||
return d
|
||||
|
||||
def _create_directory(self):
|
||||
if not os.path.exists(self.db_dir):
|
||||
os.makedirs(self.db_dir)
|
||||
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
|
||||
db_revision.write(str(self.current_db_revision))
|
||||
db_revision.close()
|
||||
log.debug("Created the configuration directory: %s", str(self.db_dir))
|
||||
if not os.path.exists(self.blobfile_dir):
|
||||
os.makedirs(self.blobfile_dir)
|
||||
log.debug("Created the data directory: %s", str(self.blobfile_dir))
|
||||
if os.name == "nt":
|
||||
if not os.path.exists(self.wallet_dir):
|
||||
os.makedirs(self.wallet_dir)
|
||||
if not os.path.exists(self.wallet_conf):
|
||||
lbrycrd_conf = open(self.wallet_conf, mode='w')
|
||||
self.wallet_user = "rpcuser"
|
||||
lbrycrd_conf.write("rpcuser=%s\n" % self.wallet_user)
|
||||
self.wallet_password = binascii.hexlify(Random.new().read(20))
|
||||
lbrycrd_conf.write("rpcpassword=%s\n" % self.wallet_password)
|
||||
lbrycrd_conf.write("server=1\n")
|
||||
lbrycrd_conf.close()
|
||||
else:
|
||||
lbrycrd_conf = open(self.wallet_conf)
|
||||
for l in lbrycrd_conf:
|
||||
if l.startswith("rpcuser="):
|
||||
self.wallet_user = l[8:].rstrip('\n')
|
||||
if l.startswith("rpcpassword="):
|
||||
self.wallet_password = l[12:].rstrip('\n')
|
||||
if l.startswith("rpcport="):
|
||||
self.wallet_rpc_port = int(l[8:-1].rstrip('\n'))
|
||||
|
||||
def _get_session(self):
|
||||
lbrycrdd_path = None
|
||||
if self.start_lbrycrdd is True:
|
||||
lbrycrdd_path = self.lbrycrdd_path
|
||||
if not lbrycrdd_path:
|
||||
lbrycrdd_path = self.default_lbrycrdd_path
|
||||
|
||||
if sys.platform == 'darwin':
|
||||
os.chdir("/Applications/LBRY.app/Contents/Resources")
|
||||
|
||||
wallet = LBRYcrdWallet(self.db_dir, wallet_dir=self.wallet_dir, wallet_conf=self.wallet_conf,
|
||||
lbrycrdd_path=lbrycrdd_path)
|
||||
|
||||
peer_port = None
|
||||
if self.run_server:
|
||||
peer_port = self.peer_port
|
||||
self.session = LBRYSession(self.default_blob_data_payment_rate, db_dir=self.db_dir,
|
||||
blob_dir=self.blobfile_dir, use_upnp=self.use_upnp, wallet=wallet,
|
||||
known_dht_nodes=self.known_dht_nodes, dht_node_port=self.dht_node_port,
|
||||
peer_port=peer_port)
|
||||
return self.session.setup()
|
||||
|
||||
def _setup_stream_info_manager(self):
|
||||
self.stream_info_manager = TempLBRYFileMetadataManager()
|
||||
return defer.succeed(True)
|
||||
|
||||
def start_server(self):
|
||||
|
||||
if self.run_server:
|
||||
self.blob_request_payment_rate_manager = PaymentRateManager(
|
||||
self.session.base_payment_rate_manager,
|
||||
self.default_blob_data_payment_rate
|
||||
)
|
||||
handlers = [
|
||||
BlobAvailabilityHandlerFactory(self.session.blob_manager),
|
||||
self.session.wallet.get_wallet_info_query_handler_factory(),
|
||||
BlobRequestHandlerFactory(self.session.blob_manager, self.session.wallet,
|
||||
self.blob_request_payment_rate_manager)
|
||||
]
|
||||
|
||||
server_factory = ServerProtocolFactory(self.session.rate_limiter,
|
||||
handlers,
|
||||
self.session.peer_manager)
|
||||
from twisted.internet import reactor
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop_server(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, p = None, self.lbry_server_port
|
||||
return defer.maybeDeferred(p.stopListening)
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_stream_identifier(self):
|
||||
add_lbry_file_to_sd_identifier(self.sd_identifier)
|
||||
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
|
||||
self.session.blob_manager, self.stream_info_manager,
|
||||
self.session.wallet, self.download_directory)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_saver_factory)
|
||||
file_opener_factory = LBRYFileOpenerFactory(self.session.peer_finder, self.session.rate_limiter,
|
||||
self.session.blob_manager, self.stream_info_manager,
|
||||
self.session.wallet)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, file_opener_factory)
|
||||
|
||||
def check_first_run(self):
|
||||
d = self.session.wallet.is_first_run()
|
||||
d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else 0.0)
|
||||
return d
|
||||
|
||||
def _do_first_run(self):
|
||||
d = self.session.wallet.get_new_address()
|
||||
|
||||
def send_request(url, data):
|
||||
r = requests.post(url, json=data)
|
||||
if r.status_code == 200:
|
||||
return r.json()['credits_sent']
|
||||
return 0.0
|
||||
|
||||
def log_error(err):
|
||||
log.warning("unable to request free credits. %s", err.getErrorMessage())
|
||||
return 0.0
|
||||
|
||||
def request_credits(address):
|
||||
url = "http://credreq.lbry.io/requestcredits"
|
||||
data = {"address": address}
|
||||
d = threads.deferToThread(send_request, url, data)
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
d.addCallback(request_credits)
|
||||
return d
|
||||
|
||||
def _resolve_name(self, uri):
|
||||
return self.session.wallet.get_stream_info_for_name(uri)
|
||||
|
||||
def download_stream(self, stream_frame, uri):
|
||||
resolve_d = self._resolve_name(uri)
|
||||
|
||||
stream_frame.show_metadata_status("resolving name...")
|
||||
|
||||
stream_frame.cancel_func = resolve_d.cancel
|
||||
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
|
||||
def update_stream_name(value):
|
||||
if 'name' in value:
|
||||
stream_frame.show_name(value['name'])
|
||||
if 'description' in value:
|
||||
stream_frame.show_description(value['description'])
|
||||
if 'thumbnail' in value:
|
||||
stream_frame.show_thumbnail(value['thumbnail'])
|
||||
return value
|
||||
|
||||
def get_sd_hash(value):
|
||||
if 'stream_hash' in value:
|
||||
return value['stream_hash']
|
||||
raise UnknownNameError(uri)
|
||||
|
||||
def get_sd_blob(sd_hash):
|
||||
stream_frame.show_metadata_status("name resolved, fetching metadata...")
|
||||
get_sd_d = StreamDescriptor.download_sd_blob(self.session, sd_hash,
|
||||
payment_rate_manager)
|
||||
get_sd_d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
|
||||
get_sd_d.addCallbacks(choose_download_factory, bad_sd_blob)
|
||||
return get_sd_d
|
||||
|
||||
def get_info_from_validator(info_validator):
|
||||
stream_name = None
|
||||
stream_size = None
|
||||
for field, val in info_validator.info_to_show():
|
||||
if field == "suggested_file_name":
|
||||
stream_name = val
|
||||
elif field == "stream_name" and stream_name is None:
|
||||
stream_name = val
|
||||
elif field == "stream_size":
|
||||
stream_size = int(val)
|
||||
if stream_size is None:
|
||||
stream_size = "unknown"
|
||||
if stream_name is None:
|
||||
stream_name = "unknown"
|
||||
return stream_name, stream_size
|
||||
|
||||
def choose_download_factory(metadata):
|
||||
#info_validator, options, factories = info_and_factories
|
||||
stream_name, stream_size = get_info_from_validator(metadata.validator)
|
||||
if isinstance(stream_size, (int, long)):
|
||||
price = payment_rate_manager.get_effective_min_blob_data_payment_rate()
|
||||
estimated_cost = stream_size * 1.0 / 2**20 * price
|
||||
else:
|
||||
estimated_cost = "unknown"
|
||||
|
||||
stream_frame.show_stream_metadata(stream_name, stream_size, estimated_cost)
|
||||
|
||||
available_options = metadata.options.get_downloader_options(metadata.validator,
|
||||
payment_rate_manager)
|
||||
|
||||
stream_frame.show_download_options(available_options)
|
||||
|
||||
get_downloader_d = defer.Deferred()
|
||||
|
||||
def create_downloader(f, chosen_options):
|
||||
|
||||
def fire_get_downloader_d(downloader):
|
||||
if not get_downloader_d.called:
|
||||
get_downloader_d.callback(downloader)
|
||||
|
||||
stream_frame.disable_download_buttons()
|
||||
d = f.make_downloader(metadata, chosen_options,
|
||||
payment_rate_manager)
|
||||
d.addCallback(fire_get_downloader_d)
|
||||
|
||||
for factory in metadata.factories:
|
||||
|
||||
def choose_factory(f=factory):
|
||||
chosen_options = stream_frame.get_chosen_options()
|
||||
create_downloader(f, chosen_options)
|
||||
|
||||
stream_frame.add_download_factory(factory, choose_factory)
|
||||
|
||||
get_downloader_d.addCallback(start_download)
|
||||
|
||||
return get_downloader_d
|
||||
|
||||
def show_stream_status(downloader):
|
||||
total_bytes = downloader.get_total_bytes_cached()
|
||||
bytes_left_to_download = downloader.get_bytes_left_to_download()
|
||||
points_paid = payment_rate_manager.points_paid
|
||||
payment_rate = payment_rate_manager.get_effective_min_blob_data_payment_rate()
|
||||
points_remaining = 1.0 * bytes_left_to_download * payment_rate / 2**20
|
||||
stream_frame.show_progress(total_bytes, bytes_left_to_download,
|
||||
points_paid, points_remaining)
|
||||
|
||||
def show_finished(arg, downloader):
|
||||
show_stream_status(downloader)
|
||||
stream_frame.show_download_done(payment_rate_manager.points_paid)
|
||||
return arg
|
||||
|
||||
def start_download(downloader):
|
||||
stream_frame.stream_hash = downloader.stream_hash
|
||||
l = task.LoopingCall(show_stream_status, downloader)
|
||||
l.start(1)
|
||||
d = downloader.start()
|
||||
stream_frame.cancel_func = downloader.stop
|
||||
|
||||
def stop_looping_call(arg):
|
||||
l.stop()
|
||||
stream_frame.cancel_func = resolve_d.cancel
|
||||
return arg
|
||||
|
||||
d.addBoth(stop_looping_call)
|
||||
d.addCallback(show_finished, downloader)
|
||||
return d
|
||||
|
||||
def lookup_failed(err):
|
||||
stream_frame.show_metadata_status("name lookup failed")
|
||||
return err
|
||||
|
||||
def bad_sd_blob(err):
|
||||
stream_frame.show_metadata_status("Unknown type or badly formed metadata")
|
||||
return err
|
||||
|
||||
resolve_d.addCallback(update_stream_name)
|
||||
resolve_d.addCallback(get_sd_hash)
|
||||
resolve_d.addCallbacks(get_sd_blob, lookup_failed)
|
||||
|
||||
def show_err(err):
|
||||
tkMessageBox.showerror(title="Download Error", message=err.getErrorMessage())
|
||||
log.error(err.getErrorMessage())
|
||||
stream_frame.show_download_done(payment_rate_manager.points_paid)
|
||||
|
||||
resolve_d.addErrback(lambda err: err.trap(defer.CancelledError, UnknownNameError,
|
||||
UnknownStreamTypeError, InvalidStreamDescriptorError,
|
||||
InvalidStreamInfoError))
|
||||
resolve_d.addErrback(show_err)
|
||||
|
||||
def delete_associated_blobs():
|
||||
if stream_frame.stream_hash is None or self.delete_blobs_on_remove is False:
|
||||
return defer.succeed(True)
|
||||
d1 = self.stream_info_manager.get_blobs_for_stream(stream_frame.stream_hash)
|
||||
|
||||
def get_blob_hashes(blob_infos):
|
||||
return [b[0] for b in blob_infos if b[0] is not None]
|
||||
|
||||
d1.addCallback(get_blob_hashes)
|
||||
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(stream_frame.stream_hash)
|
||||
|
||||
def combine_blob_hashes(results):
|
||||
blob_hashes = []
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
blob_hashes.extend(result)
|
||||
return blob_hashes
|
||||
|
||||
def delete_blobs(blob_hashes):
|
||||
return self.session.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
dl.addCallback(combine_blob_hashes)
|
||||
dl.addCallback(delete_blobs)
|
||||
return dl
|
||||
|
||||
resolve_d.addCallback(lambda _: delete_associated_blobs())
|
||||
self._add_download_deferred(resolve_d, stream_frame)
|
||||
|
||||
def _add_download_deferred(self, d, stream_frame):
|
||||
self.download_deferreds.append(d)
|
||||
self.stream_frames.append(stream_frame)
|
||||
|
||||
def remove_from_list():
|
||||
self.download_deferreds.remove(d)
|
||||
self.stream_frames.remove(stream_frame)
|
||||
|
||||
d.addBoth(lambda _: remove_from_list())
|
||||
@@ -0,0 +1,463 @@
|
||||
import Tkinter as tk
|
||||
import sys
|
||||
import tkFont
|
||||
import ttk
|
||||
import locale
|
||||
import os
|
||||
|
||||
|
||||
class StreamFrame(object):
|
||||
def __init__(self, app, uri):
|
||||
self.app = app
|
||||
self.uri = uri
|
||||
self.stream_hash = None
|
||||
self.cancel_func = None
|
||||
|
||||
self.stream_frame = ttk.Frame(self.app.streams_frame, style="B.TFrame")
|
||||
|
||||
self.stream_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(30, 0))
|
||||
|
||||
self.stream_frame_header = ttk.Frame(self.stream_frame, style="C.TFrame")
|
||||
self.stream_frame_header.grid(sticky=tk.E + tk.W)
|
||||
|
||||
self.uri_font = tkFont.Font(size=8)
|
||||
self.uri_label = ttk.Label(
|
||||
self.stream_frame_header, text=self.uri, font=self.uri_font, foreground="#666666"
|
||||
)
|
||||
self.uri_label.grid(row=0, column=0, sticky=tk.W)
|
||||
|
||||
if os.name == "nt":
|
||||
self.button_cursor = ""
|
||||
else:
|
||||
self.button_cursor = "hand1"
|
||||
|
||||
close_file_name = "close2.gif"
|
||||
if os.name == "nt":
|
||||
close_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), close_file_name)
|
||||
else:
|
||||
close_file = os.path.join(os.path.dirname(__file__), close_file_name)
|
||||
|
||||
self.close_picture = tk.PhotoImage(
|
||||
file=close_file
|
||||
)
|
||||
self.close_button = ttk.Button(
|
||||
self.stream_frame_header, command=self.cancel, style="Stop.TButton", cursor=self.button_cursor
|
||||
)
|
||||
self.close_button.config(image=self.close_picture)
|
||||
self.close_button.grid(row=0, column=1, sticky=tk.E + tk.N)
|
||||
|
||||
self.stream_frame_header.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.stream_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.stream_frame_body = ttk.Frame(self.stream_frame, style="C.TFrame")
|
||||
self.stream_frame_body.grid(row=1, column=0, sticky=tk.E + tk.W)
|
||||
|
||||
self.name_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.name_frame.grid(sticky=tk.W + tk.E)
|
||||
self.name_frame.grid_columnconfigure(0, weight=16)
|
||||
self.name_frame.grid_columnconfigure(1, weight=1)
|
||||
|
||||
self.stream_frame_body.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.metadata_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.metadata_frame.grid(sticky=tk.W + tk.E, row=1)
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self.options_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
|
||||
self.outer_button_frame = ttk.Frame(self.stream_frame_body, style="D.TFrame")
|
||||
self.outer_button_frame.grid(sticky=tk.W + tk.E, row=4)
|
||||
|
||||
#show_options_picture_file_name = "show_options.gif"
|
||||
#if os.name == "nt":
|
||||
# show_options_picture_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
# "lbrynet", "lbrynet_downloader_gui",
|
||||
# show_options_picture_file_name)
|
||||
#else:
|
||||
# show_options_picture_file = os.path.join(os.path.dirname(__file__),
|
||||
# show_options_picture_file_name)
|
||||
|
||||
#self.show_options_picture = tk.PhotoImage(
|
||||
# file=show_options_picture_file
|
||||
#)
|
||||
|
||||
#hide_options_picture_file_name = "hide_options.gif"
|
||||
#if os.name == "nt":
|
||||
# hide_options_picture_file = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
# "lbrynet", "lbrynet_downloader_gui",
|
||||
# hide_options_picture_file_name)
|
||||
#else:
|
||||
# hide_options_picture_file = os.path.join(os.path.dirname(__file__),
|
||||
# hide_options_picture_file_name)
|
||||
|
||||
#self.hide_options_picture = tk.PhotoImage(
|
||||
# file=hide_options_picture_file
|
||||
#)
|
||||
|
||||
#self.show_options_button = None
|
||||
|
||||
self.status_label = None
|
||||
#self.name_label = None
|
||||
self.name_text = None
|
||||
self.estimated_cost_frame = None
|
||||
self.bytes_downloaded_label = None
|
||||
self.button_frame = None
|
||||
self.download_buttons = []
|
||||
self.option_frames = []
|
||||
self.name_font = None
|
||||
self.description_text = None
|
||||
#self.description_label = None
|
||||
self.file_name_frame = None
|
||||
self.cost_frame = None
|
||||
self.cost_description = None
|
||||
self.remaining_cost_description = None
|
||||
self.cost_label = None
|
||||
self.remaining_cost_label = None
|
||||
self.progress_frame = None
|
||||
|
||||
def cancel(self):
|
||||
if self.cancel_func is not None:
|
||||
self.cancel_func()
|
||||
self.stream_frame.destroy()
|
||||
self.app.stream_removed()
|
||||
|
||||
def _resize_text(self, text_widget):
|
||||
actual_height = text_widget.tk.call(text_widget._w, "count", "-displaylines", "0.0", "end")
|
||||
text_widget.config(height=int(actual_height))
|
||||
|
||||
def show_name(self, name):
|
||||
self.name_font = tkFont.Font(size=16)
|
||||
#self.name_label = ttk.Label(
|
||||
# self.name_frame, text=name, font=self.name_font
|
||||
#)
|
||||
#self.name_label.grid(row=0, column=0, sticky=tk.W)
|
||||
self.name_text = tk.Text(
|
||||
self.name_frame, font=self.name_font, wrap=tk.WORD, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0, width=1, height=1
|
||||
)
|
||||
self.name_text.insert(tk.INSERT, name)
|
||||
self.name_text.config(state=tk.DISABLED)
|
||||
self.name_text.grid(row=0, column=0, sticky=tk.W+tk.E+tk.N+tk.S)
|
||||
self.name_text.update()
|
||||
self._resize_text(self.name_text)
|
||||
|
||||
def show_description(self, description):
|
||||
#if os.name == "nt":
|
||||
# wraplength = 580
|
||||
#else:
|
||||
# wraplength = 600
|
||||
#self.description_label = ttk.Label(
|
||||
# self.name_frame, text=description, wraplength=wraplength
|
||||
#)
|
||||
#self.description_label.grid(row=1, column=0, sticky=tk.W)
|
||||
self.description_text = tk.Text(
|
||||
self.name_frame, wrap=tk.WORD, relief=tk.FLAT, borderwidth=0,
|
||||
highlightthickness=0, width=1, height=1
|
||||
)
|
||||
self.description_text.insert(tk.INSERT, description)
|
||||
self.description_text.config(state=tk.DISABLED)
|
||||
self.description_text.grid(row=1, column=0, sticky=tk.W+tk.E+tk.N+tk.S)
|
||||
self.description_text.update()
|
||||
self._resize_text(self.description_text)
|
||||
|
||||
def show_thumbnail(self, url=None):
|
||||
thumbnail = None
|
||||
if url is not None:
|
||||
import urllib2
|
||||
import base64
|
||||
response = urllib2.urlopen(url)
|
||||
thumbnail_data = base64.b64encode(response.read())
|
||||
thumbnail = tk.PhotoImage(data=thumbnail_data)
|
||||
current_width = thumbnail.width()
|
||||
current_height = thumbnail.height()
|
||||
max_width = 130
|
||||
max_height = 90
|
||||
scale_ratio = max(1.0 * current_width / max_width, 1.0 * current_height / max_height)
|
||||
if scale_ratio < 1:
|
||||
scale_ratio = 1
|
||||
else:
|
||||
scale_ratio = int(scale_ratio + 1)
|
||||
thumbnail = thumbnail.subsample(scale_ratio)
|
||||
if thumbnail is not None:
|
||||
label = ttk.Label(self.name_frame, image=thumbnail)
|
||||
label.safekeeping = thumbnail
|
||||
label.grid(row=0, column=1, rowspan=2, sticky=tk.E+tk.N+tk.W+tk.S)
|
||||
label.update()
|
||||
self.description_text.update()
|
||||
self.name_text.update()
|
||||
self._resize_text(self.description_text)
|
||||
self._resize_text(self.name_text)
|
||||
|
||||
def show_metadata_status(self, value):
|
||||
if self.status_label is None:
|
||||
self.status_label = ttk.Label(
|
||||
self.metadata_frame, text=value
|
||||
)
|
||||
self.status_label.grid()
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1)
|
||||
else:
|
||||
self.status_label.config(text=value)
|
||||
|
||||
@staticmethod
|
||||
def get_formatted_stream_size(stream_size):
|
||||
if isinstance(stream_size, (int, long)):
|
||||
if stream_size >= 2**40:
|
||||
units = "TB"
|
||||
factor = 2**40
|
||||
elif stream_size >= 2**30:
|
||||
units = "GB"
|
||||
factor = 2**30
|
||||
elif stream_size >= 2**20:
|
||||
units = "MB"
|
||||
factor = 2**20
|
||||
elif stream_size >= 2**10:
|
||||
units = "KB"
|
||||
factor = 2**10
|
||||
else:
|
||||
return str(stream_size) + " B"
|
||||
return "%.1f %s" % (round((stream_size * 1.0 / factor), 1), units)
|
||||
return stream_size
|
||||
|
||||
def show_stream_metadata(self, stream_name, stream_size, stream_cost):
|
||||
if self.status_label is not None:
|
||||
self.status_label.destroy()
|
||||
|
||||
self.file_name_frame = ttk.Frame(self.metadata_frame, style="F.TFrame")
|
||||
self.file_name_frame.grid(row=0, column=0, sticky=tk.W)
|
||||
self.metadata_frame.grid_columnconfigure(0, weight=1, uniform="metadata")
|
||||
|
||||
file_size_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=self.get_formatted_stream_size(stream_size)
|
||||
)
|
||||
file_size_label.grid(row=0, column=2)
|
||||
|
||||
file_name_label = ttk.Label(
|
||||
self.file_name_frame,
|
||||
text=" - " + stream_name,
|
||||
)
|
||||
file_name_label.grid(row=0, column=3)
|
||||
|
||||
self.estimated_cost_frame = ttk.Frame(self.metadata_frame, style="F.TFrame")
|
||||
self.estimated_cost_frame.grid(row=1, column=0, sticky=tk.W)
|
||||
|
||||
estimated_cost_label = ttk.Label(
|
||||
self.estimated_cost_frame,
|
||||
text=locale.format_string("%.2f LBC",
|
||||
(round(stream_cost, 2)), grouping=True),
|
||||
foreground="red"
|
||||
)
|
||||
estimated_cost_label.grid(row=1, column=2)
|
||||
|
||||
self.button_frame = ttk.Frame(self.outer_button_frame, style="E.TFrame")
|
||||
self.button_frame.grid(row=0, column=1)
|
||||
|
||||
self.outer_button_frame.grid_columnconfigure(0, weight=1, uniform="buttons")
|
||||
self.outer_button_frame.grid_columnconfigure(1, weight=2, uniform="buttons1")
|
||||
self.outer_button_frame.grid_columnconfigure(2, weight=1, uniform="buttons")
|
||||
|
||||
def add_download_factory(self, factory, download_func):
|
||||
download_button = ttk.Button(
|
||||
self.button_frame, text=factory.get_description(), command=download_func,
|
||||
style='LBRY.TButton', cursor=self.button_cursor
|
||||
)
|
||||
self.download_buttons.append(download_button)
|
||||
download_button.grid(row=0, column=len(self.download_buttons) - 1, padx=5, pady=(1, 2))
|
||||
|
||||
def disable_download_buttons(self):
|
||||
for download_button in self.download_buttons:
|
||||
download_button.config(state=tk.DISABLED)
|
||||
|
||||
def remove_download_buttons(self):
|
||||
for download_button in self.download_buttons:
|
||||
download_button.destroy()
|
||||
self.download_buttons = []
|
||||
|
||||
def get_option_widget(self, option_type, option_frame):
|
||||
if option_type.value == float:
|
||||
entry_frame = ttk.Frame(
|
||||
option_frame,
|
||||
style="H.TFrame"
|
||||
)
|
||||
entry_frame.grid()
|
||||
col = 0
|
||||
if option_type.short_description is not None:
|
||||
entry_label = ttk.Label(
|
||||
entry_frame,
|
||||
#text=option_type.short_description
|
||||
text=""
|
||||
)
|
||||
entry_label.grid(row=0, column=0, sticky=tk.W)
|
||||
col = 1
|
||||
entry = ttk.Entry(
|
||||
entry_frame,
|
||||
width=10,
|
||||
style="Float.TEntry"
|
||||
)
|
||||
entry_frame.entry = entry
|
||||
entry.grid(row=0, column=col, sticky=tk.W)
|
||||
return entry_frame
|
||||
if option_type.value == bool:
|
||||
bool_frame = ttk.Frame(
|
||||
option_frame,
|
||||
style="H.TFrame"
|
||||
)
|
||||
bool_frame.chosen_value = tk.BooleanVar()
|
||||
true_text = "True"
|
||||
false_text = "False"
|
||||
if option_type.bool_options_description is not None:
|
||||
true_text, false_text = option_type.bool_options_description
|
||||
true_radio_button = ttk.Radiobutton(
|
||||
bool_frame, text=true_text, variable=bool_frame.chosen_value, value=True
|
||||
)
|
||||
true_radio_button.grid(row=0, sticky=tk.W)
|
||||
false_radio_button = ttk.Radiobutton(
|
||||
bool_frame, text=false_text, variable=bool_frame.chosen_value, value=False
|
||||
)
|
||||
false_radio_button.grid(row=1, sticky=tk.W)
|
||||
return bool_frame
|
||||
label = ttk.Label(
|
||||
option_frame,
|
||||
text=""
|
||||
)
|
||||
return label
|
||||
|
||||
def show_download_options(self, options):
|
||||
left_padding = 20
|
||||
for option in options:
|
||||
f = ttk.Frame(
|
||||
self.options_frame,
|
||||
style="E.TFrame"
|
||||
)
|
||||
f.grid(sticky=tk.W + tk.E, padx=left_padding)
|
||||
self.option_frames.append((option, f))
|
||||
description_label = ttk.Label(
|
||||
f,
|
||||
text=option.long_description
|
||||
)
|
||||
description_label.grid(row=0, sticky=tk.W)
|
||||
if len(option.option_types) > 1:
|
||||
f.chosen_type = tk.IntVar()
|
||||
choices_frame = ttk.Frame(
|
||||
f,
|
||||
style="F.TFrame"
|
||||
)
|
||||
f.choices_frame = choices_frame
|
||||
choices_frame.grid(row=1, sticky=tk.W, padx=left_padding)
|
||||
choices_frame.choices = []
|
||||
for i, option_type in enumerate(option.option_types):
|
||||
choice_frame = ttk.Frame(
|
||||
choices_frame,
|
||||
style="G.TFrame"
|
||||
)
|
||||
choice_frame.grid(sticky=tk.W)
|
||||
option_text = ""
|
||||
if option_type.short_description is not None:
|
||||
option_text = option_type.short_description
|
||||
option_radio_button = ttk.Radiobutton(
|
||||
choice_frame, text=option_text, variable=f.chosen_type, value=i
|
||||
)
|
||||
option_radio_button.grid(row=0, column=0, sticky=tk.W)
|
||||
option_widget = self.get_option_widget(option_type, choice_frame)
|
||||
option_widget.grid(row=0, column=1, sticky=tk.W)
|
||||
choices_frame.choices.append(option_widget)
|
||||
if i == 0:
|
||||
option_radio_button.invoke()
|
||||
else:
|
||||
choice_frame = ttk.Frame(
|
||||
f,
|
||||
style="F.TFrame"
|
||||
)
|
||||
choice_frame.grid(sticky=tk.W, padx=left_padding)
|
||||
option_widget = self.get_option_widget(option.option_types[0], choice_frame)
|
||||
option_widget.grid(row=0, column=0, sticky=tk.W)
|
||||
f.option_widget = option_widget
|
||||
#self.show_options_button = ttk.Button(
|
||||
# self.stream_frame_body, command=self._toggle_show_options, style="Stop.TButton",
|
||||
# cursor=self.button_cursor
|
||||
#)
|
||||
#self.show_options_button.config(image=self.show_options_picture)
|
||||
#self.show_options_button.grid(sticky=tk.W, row=2, column=0)
|
||||
|
||||
def _get_chosen_option(self, option_type, option_widget):
|
||||
if option_type.value == float:
|
||||
return float(option_widget.entry.get())
|
||||
if option_type.value == bool:
|
||||
return option_widget.chosen_value.get()
|
||||
return option_type.value
|
||||
|
||||
def get_chosen_options(self):
|
||||
chosen_options = []
|
||||
for o, f in self.option_frames:
|
||||
if len(o.option_types) > 1:
|
||||
chosen_index = f.chosen_type.get()
|
||||
option_type = o.option_types[chosen_index]
|
||||
option_widget = f.choices_frame.choices[chosen_index]
|
||||
chosen_options.append(self._get_chosen_option(option_type, option_widget))
|
||||
else:
|
||||
option_type = o.option_types[0]
|
||||
option_widget = f.option_widget
|
||||
chosen_options.append(self._get_chosen_option(option_type, option_widget))
|
||||
return chosen_options
|
||||
|
||||
#def _toggle_show_options(self):
|
||||
# if self.options_frame.winfo_ismapped():
|
||||
# self.show_options_button.config(image=self.show_options_picture)
|
||||
# self.options_frame.grid_forget()
|
||||
# else:
|
||||
# self.show_options_button.config(image=self.hide_options_picture)
|
||||
# self.options_frame.grid(sticky=tk.W + tk.E, row=3)
|
||||
|
||||
def show_progress(self, total_bytes, bytes_left_to_download, points_paid,
|
||||
points_remaining):
|
||||
if self.bytes_downloaded_label is None:
|
||||
self.remove_download_buttons()
|
||||
self.button_frame.destroy()
|
||||
self.estimated_cost_frame.destroy()
|
||||
for option, frame in self.option_frames:
|
||||
frame.destroy()
|
||||
self.options_frame.destroy()
|
||||
#self.show_options_button.destroy()
|
||||
|
||||
self.progress_frame = ttk.Frame(self.outer_button_frame, style="F.TFrame")
|
||||
self.progress_frame.grid(row=0, column=0, sticky=tk.W, pady=(0, 8))
|
||||
|
||||
self.bytes_downloaded_label = ttk.Label(
|
||||
self.progress_frame,
|
||||
text=""
|
||||
)
|
||||
self.bytes_downloaded_label.grid(row=0, column=0)
|
||||
|
||||
self.cost_frame = ttk.Frame(self.outer_button_frame, style="F.TFrame")
|
||||
self.cost_frame.grid(row=1, column=0, sticky=tk.W, pady=(0, 4))
|
||||
|
||||
self.cost_label = ttk.Label(
|
||||
self.cost_frame,
|
||||
text="",
|
||||
foreground="red"
|
||||
)
|
||||
self.cost_label.grid(row=0, column=1, padx=(1, 0))
|
||||
self.outer_button_frame.grid_columnconfigure(2, weight=0, uniform="")
|
||||
|
||||
if self.bytes_downloaded_label.winfo_exists():
|
||||
percent_done = 0
|
||||
if total_bytes > 0:
|
||||
percent_done = 100.0 * (total_bytes - bytes_left_to_download) / total_bytes
|
||||
percent_done_string = locale.format_string(" (%.2f%%)", percent_done)
|
||||
self.bytes_downloaded_label.config(
|
||||
text=self.get_formatted_stream_size(total_bytes - bytes_left_to_download) + percent_done_string
|
||||
)
|
||||
if self.cost_label.winfo_exists():
|
||||
total_points = points_remaining + points_paid
|
||||
self.cost_label.config(text=locale.format_string("%.2f/%.2f LBC",
|
||||
(round(points_paid, 2), round(total_points, 2)),
|
||||
grouping=True))
|
||||
|
||||
def show_download_done(self, total_points_paid):
|
||||
if self.bytes_downloaded_label is not None and self.bytes_downloaded_label.winfo_exists():
|
||||
self.bytes_downloaded_label.destroy()
|
||||
if self.cost_label is not None and self.cost_label.winfo_exists():
|
||||
self.cost_label.config(text=locale.format_string("%.2f LBC",
|
||||
(round(total_points_paid, 2),),
|
||||
grouping=True))
|
||||
|
Before Width: | Height: | Size: 188 B After Width: | Height: | Size: 188 B |
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |
|
Before Width: | Height: | Size: 195 B After Width: | Height: | Size: 195 B |
|
Before Width: | Height: | Size: 72 B After Width: | Height: | Size: 72 B |
@@ -0,0 +1,33 @@
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from lbrynet.lbrynet_gui.GuiApp import DownloaderApp
|
||||
from twisted.internet import reactor, task
|
||||
import locale
|
||||
|
||||
|
||||
def start_gui():
|
||||
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
formatter = logging.Formatter(log_format)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
file_handler = logging.FileHandler("gui.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(logging.Filter("lbrynet"))
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
sys.stdout = open("gui.out.log", 'w')
|
||||
sys.stderr = open("gui.err.log", 'w')
|
||||
|
||||
locale.setlocale(locale.LC_ALL, '')
|
||||
|
||||
app = DownloaderApp()
|
||||
|
||||
d = task.deferLater(reactor, 0, app.start)
|
||||
|
||||
reactor.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_gui()
|
||||
|
After Width: | Height: | Size: 174 B |
|
Before Width: | Height: | Size: 3.6 KiB After Width: | Height: | Size: 3.6 KiB |
|
Before Width: | Height: | Size: 361 KiB After Width: | Height: | Size: 361 KiB |
@@ -0,0 +1,108 @@
|
||||
# ===== known_dht_nodes =====
|
||||
# A comma-separated list of dht nodes to use to bootstrap into the
|
||||
# DHT network. The nodes must be specified in dotted quad format
|
||||
# followed by a colon and the port number. For example:
|
||||
# 192.168.1.1:4000,10.0.0.1:4000,172.16.1.1:5000
|
||||
#
|
||||
# known_dht_nodes = 104.236.42.182:4000
|
||||
|
||||
|
||||
# ===== download_directory =====
|
||||
# On Windows, the default location for "download_directory" is the
|
||||
# directory the user has registered with the OS as the default
|
||||
# download directory, usually the 'Downloads' folder in the user's
|
||||
# home directory. On Linux, it is the current directory.
|
||||
#
|
||||
# download_directory = /path/to/example_directory
|
||||
|
||||
|
||||
# ===== data_dir =====
|
||||
# The data directory is the directory in which the application
|
||||
# stores information about the encrypted chunks stored on disk
|
||||
# and the streams they belong to, the encrypted chunks themselves,
|
||||
# and any other metadata. By default, set to '.lbrydownloader'
|
||||
# in the user's home directory.
|
||||
#
|
||||
# data_dir = /path/to/home/.lbrydownloader
|
||||
|
||||
|
||||
# ===== run_server =====
|
||||
# Turn on or off the server, which uploads encrypted chunks of data
|
||||
# to other clients on the network.
|
||||
#
|
||||
# run_server = True
|
||||
|
||||
|
||||
# ===== start_lbrycrdd =====
|
||||
# Whether to launch an lbyrcrdd server to use to send and receive
|
||||
# LBC and to look up names regisered on the LBRYcrd network.
|
||||
# Defaults to True on Windows and False on Linux.
|
||||
#
|
||||
# start_lbrycrdd = True/False
|
||||
|
||||
# ===== lbrycrdd_path =====
|
||||
# If start_lbrycrdd is set to True, the path the application will
|
||||
# use to try to run lbrycrdd. On Windows, the default path will
|
||||
# be "lbrycrdd.exe". On Linux, the default path will be "./lbrycrdd".
|
||||
#
|
||||
# lbrycrdd_path = /path/to/lbrycrdd
|
||||
|
||||
# ===== wallet_dir =====
|
||||
# The directory which the lbrycrdd instance will be given as the
|
||||
# base directory for the instance of lbrycrdd, if it is launched
|
||||
# by the application. By default, it is set to the .lbrycrd
|
||||
# directory in the user's home directory.
|
||||
#
|
||||
# wallet_dir = /path/to/home/.lbrycrd
|
||||
|
||||
|
||||
# ===== wallet_conf =====
|
||||
# The configuration file used by the lbrycrd server which will be
|
||||
# used by the application to send and receive LBC and to look up
|
||||
# registered names on the LBRYcrd network. By default, this is
|
||||
# set to the file lbrycrd.conf in the wallet_dir directory.
|
||||
#
|
||||
# wallet_conf = /path/to/home/.lbrycrd/lbrycrd.conf
|
||||
|
||||
|
||||
# ===== use_upnp =====
|
||||
# Whether to try to use UPnP to open ports in a firewall so that
|
||||
# other applications on the network can connect to the application
|
||||
#
|
||||
# use_upnp = True
|
||||
|
||||
|
||||
# ===== default_blob_data_payment_rate =====
|
||||
# The amount of LBC per megabyte to send to uploaders of data, by
|
||||
# default. Must be a positive floating point number.
|
||||
#
|
||||
# default_blob_data_payment_rate = 0.5
|
||||
|
||||
|
||||
# ===== dht_port =====
|
||||
# The UPD port which other applications will connect to in order
|
||||
# to announce their association with sha384 hashsums and to
|
||||
# find other applications which are associated with sha384
|
||||
# hashsums.
|
||||
#
|
||||
# dht_port = 4444
|
||||
|
||||
|
||||
# ===== peer_port =====
|
||||
# The TCP port which other applications will connect to in order
|
||||
# to download encrypted chunks of data from this application,
|
||||
# if this application is acting as a server.
|
||||
#
|
||||
# peer_port = 3333
|
||||
|
||||
|
||||
# ===== delete_blobs_on_stream_remove =====
|
||||
# If this is set to True, all blobs associated with the stream
|
||||
# will be deleted when the stream is removed from the GUI,
|
||||
# whether by clicking the 'x' or by closing the application.
|
||||
# If this is set to False, they will not be deleted then.
|
||||
# However, they may be deleted otherwise. For example if the
|
||||
# option to allow reuploading is set to False, they will be
|
||||
# deleted as soon as they're outputted by the application.
|
||||
#
|
||||
# delete_blobs_on_stream_remove = True
|
||||
|
After Width: | Height: | Size: 10 KiB |
|
After Width: | Height: | Size: 155 B |
@@ -0,0 +1,5 @@
|
||||
# package scripts
|
||||
|
||||
How to build LBRY packages.
|
||||
|
||||
For best results, run on a fresh image.
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
urlencode() {
|
||||
local LANG=C
|
||||
local length="${#1}"
|
||||
for (( i = 0; i < length; i++ )); do
|
||||
local c="${1:i:1}"
|
||||
case $c in
|
||||
[a-zA-Z0-9.~_-]) printf "$c" ;;
|
||||
*) printf '%%%02X' "'$c" ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
if [ -z "$(pgrep lbrynet-daemon)" ]; then
|
||||
echo "running lbrynet-daemon..."
|
||||
$DIR/lbrynet-daemon &
|
||||
sleep 3 # let the daemon load before connecting
|
||||
fi
|
||||
|
||||
ARG=${1:-}
|
||||
|
||||
if [ -z "$ARG" ]; then
|
||||
URL=""
|
||||
else
|
||||
URL="view?name=$(urlencode "$(echo "$ARG" | cut -c 8-)")"
|
||||
fi
|
||||
|
||||
/usr/bin/xdg-open "http://localhost:5279/$URL"
|
||||