Compare commits

...
496 Commits
Author SHA1 Message Date
Victor Shyba 574c2ce794 adds integration test for release_time 2019-01-23 11:16:43 -03:00
Victor Shyba 7f5541eb5c adds release_time parameter to RPC publish 2019-01-23 11:16:43 -03:00
Victor Shyba 7b49241c38 adjust balance assertions for integration test using new signature model due new fees from new size 2019-01-23 11:16:43 -03:00
Victor Shyba da716b2e3d remove new signature model integration test as its the default now 2019-01-23 11:16:43 -03:00
Victor Shyba a144fae8b2 activate new signature model 2019-01-23 11:16:43 -03:00
Victor Shyba 0d90b82287 clear releaseTime when not set 2019-01-23 11:16:43 -03:00
Victor Shyba 567c23e10f regen for metadata releaseTime field 2019-01-23 11:16:43 -03:00
Victor Shyba 16970b7d37 regen proto from lbry/types, removes local .proto files 2019-01-23 11:16:43 -03:00
Lex Berezhny 8593cd362a check darwin before windows because darwin has win, face plam! 2019-01-22 18:04:17 -05:00
Lex Berezhny 2dbf96da81 blob downer test fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny 83c7797187 pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny 6606deafa5 fixed after testing with desktop app 2019-01-22 18:04:17 -05:00
Lex Berezhny 19b0d59159 config options api_host and api_port changed to just api 2019-01-22 18:04:17 -05:00
Lex Berezhny 9ccb3fa2a3 unit/test_cli tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 48533df523 fix integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 1ae1137005 fixing integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny d5f2832b2f pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny d0230b4893 command line fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny a6cd53b97c test_Daemon.py uses /tmp for data_dir 2019-01-22 18:04:17 -05:00
Lex Berezhny 7a28171a72 conf no longer global 2019-01-22 18:04:17 -05:00
Lex Berezhny 150bcb1116 new configuration manager, first pass 2019-01-22 18:04:17 -05:00
Jack Robison 6fe9c5bf00 Merge pull request #1794 from lbryio/async-exchange-rate-manager
refactor exchange rate manager to use asyncio
2019-01-22 12:05:07 -05:00
Jack Robison 3a86fc6680 refactor exchange rate manager to use asyncio 2019-01-22 11:34:49 -05:00
Jack Robison 2805cbb331 refactor loggly handler to use asyncio 2019-01-22 10:52:50 -05:00
Jack Robison 68d7a7014c Merge pull request #1792 from lbryio/delete-unused-scripts
delete unused scripts
2019-01-22 09:53:24 -05:00
Jack Robison 883fa267a5 delete unused scripts 2019-01-22 09:33:42 -05:00
Lex Berezhny 5f2c3ac965 dont run customlogger tests 2019-01-19 16:32:11 -05:00
Lex Berezhny bf8e32ebd2 remove custom logging 2019-01-19 16:30:19 -05:00
Lex Berezhny 5971be5bef v0.30.5 2019-01-18 16:14:48 -05:00
Lex Berezhny 677d76a3e4 v0.30.5rc4 2019-01-17 23:26:12 -05:00
Victor Shyba 6ed3207cb7 phase1: legacy claim format, ability to decode new format 2019-01-17 23:24:58 -05:00
Victor Shyba be6448b74f fix str/bytes confusion from resolve/claimdict on certificate_id 2019-01-17 23:24:58 -05:00
Victor Shyba b707ee4844 encode and decode detached signatures from claim dict 2019-01-17 23:24:58 -05:00
Victor Shyba 0e9888be3b fixes from review 2019-01-17 23:24:58 -05:00
Victor Shyba 68b56b7858 adds test for claim_show 2019-01-17 23:24:58 -05:00
Victor Shyba d32697f611 adjust assertions for fee changes 2019-01-17 23:24:58 -05:00
Victor Shyba 3c54b2bcd7 activate detached sigs 2019-01-17 23:24:58 -05:00
Victor Shyba 899b182811 fix getclaimsfortx in server 2019-01-17 23:24:58 -05:00
Victor Shyba 0dfb5e008d add integration test with unserializable claim 2019-01-17 23:24:58 -05:00
Victor Shyba abf94357eb tests and fixes for sig validation on undecode-able claims 2019-01-17 23:24:58 -05:00
Victor Shyba 36aded3830 store original payload for signing and verifying 2019-01-17 23:24:58 -05:00
Victor Shyba d7ebf50602 detached as a parameter instead of class attribute 2019-01-17 23:24:58 -05:00
Victor Shyba 523c4c86a9 change default curve to SECP256k1, pass name parameter 2019-01-17 23:24:58 -05:00
Victor Shyba 74cccdbfc7 simplify detached signing and verifying 2019-01-17 23:24:58 -05:00
Victor Shyba 70471eebfa wallet server and resolve working, functional test 2019-01-17 23:24:58 -05:00
Victor Shyba 6339224980 tests for detached signatures 2019-01-17 23:24:58 -05:00
Victor Shyba fe97f7a119 adds a detached signature attribute to ClaimDict 2019-01-17 23:24:58 -05:00
Victor Shyba f8ed605da2 adds a way to verify detached signatures 2019-01-17 23:24:58 -05:00
Victor Shyba e38ef2af1f adds a way to sign detached 2019-01-17 23:24:58 -05:00
Victor Shyba 5d8e627d02 adds a separate representation for a detached signature 2019-01-17 23:24:58 -05:00
Lex Berezhny c9d4ad5450 pylint unused import 2019-01-17 19:04:52 -05:00
Lex Berezhny 31586bf588 error out when user tries --http-auth 2019-01-17 19:04:52 -05:00
Lex Berezhny 974d3c83e3 keep wallet_balance to test deprecation with, until we find better way to do this 2019-01-17 17:35:15 -05:00
Lex Berezhny 8dc8ca382b removed previously deprecated commands 2019-01-17 17:35:15 -05:00
Lex Berezhny 1dbb250413 fix bytes vs str error in migrate5to6 script 2019-01-15 21:44:16 -05:00
Lex Berezhny 9f9ec692c7 orchstr8 docker for lbry 2019-01-15 17:15:24 -05:00
Thomas Zarebczan 2c1b54c76c Add note about wallet needing to be unlocked (grin was confused about this) 2019-01-15 15:59:45 -05:00
Thomas Zarebczan 7314494fb9 add m4v
It's a container type very similar to mp4. Someone recently added one in the community top bids section.
2019-01-15 15:55:59 -05:00
Jack Robison e74a5e8ef3 cleanup 2019-01-11 16:36:55 -05:00
Jack Robison b9af999739 rename -> time_to_first_byte.py 2019-01-11 16:36:55 -05:00
hackrush 2de155f90f Profile get using rpc calls 2019-01-11 16:36:55 -05:00
hackrush ab3db2e626 Get paths from args 2019-01-11 16:36:55 -05:00
hackrush f9d26b04d8 Get averages! 2019-01-11 16:36:55 -05:00
hackrush 24e0673198 Profile downloading of sd blobs 2019-01-11 16:36:55 -05:00
hackrush 833261801f Profile downloading of sd blobs 2019-01-11 16:36:55 -05:00
hackrush 6c8cbc1b9e added additional fields to uri parser 2019-01-11 15:10:22 -05:00
Lex Berezhny 6ae4e68d1c looping_call shouldnt need to mess with cancellation 2019-01-11 14:33:45 -05:00
Oleg Silkin d22a2dee33 Removes await from shutdown call 2019-01-11 14:33:45 -05:00
Oleg Silkin 0f1ed589a5 Modifies the unit tests for the analytics manager to have async routines
Signed-off-by: Oleg Silkin <o.silkin98@gmail.com>
2019-01-11 14:33:45 -05:00
Oleg Silkin b032e99dd1 Removes the twisted dependency from the analytics file and replaces it
with `asyncio` and `aiohttp`. Also removes the calling loop from the
analytics in favor of a `dict` that stores the names of the methods
and their coroutines that are wrapped inside a looping task. The tasks
are canceled when the analytics manager is asked to shutdown

Signed-off-by: Oleg Silkin <o.silkin98@gmail.com>
2019-01-11 14:33:45 -05:00
Lex Berezhny fd7811f38d v0.30.5rc3 2019-01-11 01:17:43 -05:00
Lex Berezhny 071db3a73d release.py script now extracts backwards-incompatible: value from PR description and adds it to changelog under Backwards Incompatible Changes 2019-01-11 01:16:59 -05:00
Lex Berezhny 950c122850 fix blob announce command argument constraints 2019-01-11 00:54:15 -05:00
Thomas Zarebczan e0386ea9dc rename lbnyum_wallet_dir to wallet_dir 2019-01-11 00:02:52 -05:00
Thomas Zarebczan 6c3eee0f0e Update conf.py 2019-01-10 23:50:59 -05:00
Thomas Zarebczan 4c23add193 fix unavailable download directory
If an uploaded file had a directory that was removed, or if the old download directory doesn't exist, use the default.
2019-01-10 23:05:43 -05:00
Lex Berezhny 248d1a18c7 pylint 2019-01-10 22:38:27 -05:00
Lex Berezhny 17d1b85423 run brew upgrade twice and ignore result 2019-01-10 22:38:27 -05:00
Lex Berezhny b0deb63129 blob_reflect fix 2019-01-10 22:38:27 -05:00
Lex Berezhny ebdb33bd11 blob_availability fix 2019-01-10 22:38:27 -05:00
Lex Berezhny 69ccbda168 fixed error handling when calling commands 2019-01-10 22:38:27 -05:00
Lex Berezhny a38643cca6 remove d2f from storage function call 2019-01-10 22:38:27 -05:00
Lex Berezhny 712b19605e Revert "fix breaking integration tests after fees changed due to content_type modification"
This reverts commit 0613cf2d59.
2019-01-10 10:22:37 -05:00
Alex Grintsvayg 8a5f3842fc fix media type for files without extension 2019-01-10 10:22:37 -05:00
Lex Berezhny db3e578bcd v0.30.5rc2 2019-01-08 23:56:07 -05:00
Lex Berezhny 7e22adff7c added contributor username to each PR listed in changelog 2019-01-08 23:55:34 -05:00
Lex Berezhny 76fbaac658 do not log aiohttp.access 2019-01-08 23:54:32 -05:00
hackrush c4fac29ecb actually get the certificate_id 2019-01-08 23:45:20 -05:00
Lex Berezhny 33392b3351 migration fix 2019-01-08 23:15:25 -05:00
Lex Berezhny 0613cf2d59 fix breaking integration tests after fees changed due to content_type modification 2019-01-08 17:54:40 -05:00
Alex Grintsvayg a930ca0631 if media type is not listed, default to a type that includes the file extension 2019-01-08 16:06:44 -05:00
Lex Berezhny 63a7271008 v0.30.5rc1 2019-01-07 22:15:37 -05:00
Lex Berezhny 5ca0ebcbb2 store loop on storage 2019-01-07 21:56:05 -05:00
Lex Berezhny 2c84b7a7dd get_running_loop() -> get_event_loop() 2019-01-07 21:56:05 -05:00
Lex Berezhny 71a0c704bd storage.clock.seconds() -> asyncio.get_running_loop().time() 2019-01-07 21:56:05 -05:00
Lex Berezhny d30e7cd29e use loop.time() instead of reactor.seconds() in SQLite storage 2019-01-07 21:56:05 -05:00
Lex Berezhny aeb7c6fe76 pylint, removed unused import 2019-01-07 21:56:05 -05:00
Lex Berezhny 134c20da93 stream cost estimate command refactor 2019-01-07 21:56:05 -05:00
Lex Berezhny 893b4fc3ca fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 9b27e81940 fix logging 2019-01-07 21:56:05 -05:00
Lex Berezhny 72c0acf269 test_HTTPBlobDownloader 2019-01-07 21:56:05 -05:00
Lex Berezhny 7be53ad692 more bug fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 16e596ec87 lots o bugs fixed discovered while using desktop app 2019-01-07 21:56:05 -05:00
Lex Berezhny 3fdcb80960 minor test fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 26a3221bac hash announcer fix 2019-01-07 21:56:05 -05:00
Lex Berezhny eb16cb0aa8 fix test_HashBlob 2019-01-07 21:56:05 -05:00
Lex Berezhny b5340505e5 fix blobmanager tests and dont run conf tests 2019-01-07 21:56:05 -05:00
Lex Berezhny 4bb8129bb7 fixing tests to work with asyncio daemon 2019-01-07 21:56:05 -05:00
Lex Berezhny 7f838d3a31 all integration tests run with plain unittest now 2019-01-07 21:56:05 -05:00
Lex Berezhny c186c31bf7 simplified component setup 2019-01-07 21:56:05 -05:00
Lex Berezhny 86fb99a37c fully asyncio daemon api 2019-01-07 21:56:05 -05:00
Lex Berezhny 6c52450858 temporary module providing d2f and f2d for converting between twisted and asyncio 2019-01-07 21:56:05 -05:00
Lex Berezhny fb4e94b04a asyncifying minimal amount of the old blob/p2p code 2019-01-07 21:56:05 -05:00
Lex Berezhny 248baf58b4 asyncio daemon 2019-01-07 21:56:05 -05:00
Lex Berezhny 0845d65f4e rebase master 2019-01-07 21:56:05 -05:00
Lex Berezhny dcc5415648 v0.30.4 2019-01-04 22:03:39 -05:00
Lex Berezhny 3c38a88b22 v0.30.4rc2 2019-01-04 11:53:54 -05:00
Victor Shyba 187b7670b1 adds a dockerfile for wallet server 2019-01-04 11:10:52 -05:00
Victor Shyba 049c17ff08 adds testenet to wallet server 2019-01-04 11:10:52 -05:00
Thomas Zarebczan 8de97bb441 Add lbrynet4 dht seed node (#1734)
added `lbrynet4.lbry.io` to list of default DHT nodes
2019-01-04 11:07:02 -05:00
Lex Berezhny 97c0092923 v0.30.4rc1 2019-01-04 03:14:56 -05:00
Lex Berezhny e65e1fb5a2 added utxo_release command 2019-01-04 03:14:37 -05:00
Victor Shyba 20dd1e55f5 fix data_dir parameter parsing 2019-01-04 01:25:35 -05:00
Lex Berezhny 6aa709e34e v0.30.3 2018-12-21 18:45:34 -05:00
Lex Berezhny e73f121c08 v0.30.3rc1 2018-12-19 09:12:56 -05:00
Victor Shyba 3b9d6c4f8e Instantiate resolver before the request to hold the selected height 2018-12-19 09:04:30 -05:00
Alyssa Callahan 69b60c19c3 Merge pull request #1712 from lbryio/readme-update
Update README.md
2018-12-17 10:47:29 -05:00
Lex Berezhny 04278cb1ab release script automatically figures out prerelease status 2018-12-14 22:25:06 -05:00
Lex Berezhny e8dec5b440 v0.30.2 2018-12-14 21:50:48 -05:00
Lex Berezhny 75f82b107e updated release script with unittests and ability to go from rc to final release 2018-12-14 21:50:30 -05:00
Jack Robison 0e90c4b611 Merge pull request #1717 from lbryio/fix-directory-args
fix directory cli args and update help message
2018-12-14 17:09:15 -05:00
Jack Robison fb66b68bc8 fix directory cli args, update help message 2018-12-14 15:51:11 -05:00
Victor Shyba c601677386 run schema unit tests on travis 2018-12-14 09:21:31 -05:00
Lex Berezhny 1fbd1cd04e v0.30.2rc2 2018-12-12 19:52:11 -05:00
Jack Robison 4a71b91fbf Merge pull request #1671 from lbryio/remove-android-helpers
remove `lbrynet.androidhelpers` module
2018-12-12 18:35:31 -05:00
Jack Robison b8b0d8b1ac refactor default directory setup, remove lbrynet.androidhelpers
fixes https://github.com/lbryio/lbry/issues/1606
2018-12-12 17:46:10 -05:00
Jack Robison 8ef73d3985 update docs 2018-12-12 12:19:26 -05:00
Jack Robison 05e556a5e3 fix generate_json_api script 2018-12-12 12:18:32 -05:00
Lex Berezhny d1da42bea8 Update README.md
* updated description and added bullet points for main features of this SDK
* lex (@eukreign ) is the official product owner of lbrysdk
2018-12-12 11:47:28 -05:00
Lex Berezhny 90685e8357 deploy release for tags 2018-12-12 00:10:50 -05:00
Lex Berezhny a471d5d167 v0.30.2rc1 2018-12-11 23:44:02 -05:00
Lex Berezhny 16122299e4 revert version back 0.30.1 2018-12-11 23:39:08 -05:00
Lex Berezhny 9f18da4bae release script (#1697)
* travis deploying to github
* release script
2018-12-11 23:24:19 -05:00
Lex Berezhny 1abce52867 v0.30.2rc1 2018-12-11 23:09:31 -05:00
Lex Berezhny 072e2b65d3 revert version change 2018-12-11 22:31:45 -05:00
Lex Berezhny f6d11e5467 0.30.1rc1 2018-12-11 22:29:40 -05:00
Oleg Silkin 458bddcb65 Removes Faker from Dependencies (#1687)
* Removes faker import and its usage
* Replaces data generated with faker with static data
* Drops faker as a dependency & removes extras require section
* Removes installation instruction for the test dependencies in setup.py
2018-12-11 20:19:58 -05:00
Victor Shyba 5fad43ff69 fix test tear down and unify test_commands 2018-12-11 16:34:37 -05:00
Victor Shyba 6992d0d947 add new test to tox.ini 2018-12-11 16:34:37 -05:00
Victor Shyba d1b0180437 add tests for claim_list_by_channel 2018-12-11 16:34:37 -05:00
Victor Shyba ba59e126cd forward errors outside of uris on channel listing 2018-12-11 16:34:37 -05:00
Victor Shyba 4e3467bb7e fix windows release type set script 2018-12-11 16:30:35 -05:00
Lex Berezhny 4d4c705eb9 0.30.1 changelog updates 2018-12-09 23:30:09 -05:00
Lex Berezhny ce7b3184c4 0.30.1 2018-12-09 23:15:11 -05:00
Lex Berezhny b71faddaac upload zip files into the same directory 2018-12-08 19:01:13 -05:00
Lex Berezhny 1328cb4cce check file exists before reading config 2018-12-08 19:01:13 -05:00
Lex Berezhny 946488bce9 correctly save daemon_settings.yml on fresh install 2018-12-08 19:01:13 -05:00
Victor Shyba 7713881e71 additional tests for resolution border cases 2018-12-08 14:40:37 -05:00
Victor Shyba 26fcc48c5a fix for name collision on channel path 2018-12-08 14:40:37 -05:00
Thomas Zarebczan bf12fb68a2 use xcode8.3 for mac build (#1692)
use xcode8.3 for mac build
2018-12-08 14:38:35 -05:00
Jack Robison 3a4c9ff4f9 bump to 0.30.1rc11 2018-12-07 13:07:06 -05:00
Jack Robison 6180a2917d add epub and wmv content types 2018-12-07 13:06:36 -05:00
Lex Berezhny 014cbb4623 v0.30.1rc10 2018-12-07 12:04:28 -05:00
Jack Robison ee98251d79 Merge pull request #1700 from lbryio/mimetypes
improve contentType detection
2018-12-07 12:02:41 -05:00
Jack Robison e3d08ec773 improve contentType detection
fixes https://github.com/lbryio/lbry/issues/1690
2018-12-07 11:31:05 -05:00
Lex Berezhny f03e68e63e 0.30.1rc9 2018-12-06 00:51:07 -05:00
Lex Berezhny 72090b45dd confirmations is returned as int 2018-12-06 00:49:07 -05:00
Lex Berezhny 93f9622798 fix json_response_encoder to also return correct confirmations 2018-12-06 00:49:07 -05:00
Lex Berezhny 16ff4b4e2c confirmations fix 2018-12-06 00:49:07 -05:00
Lex Berezhny c90cca4b20 fix #1683 2018-12-05 14:38:35 -05:00
Lex Berezhny 98d83ea74f fix for integration test 2018-12-05 12:14:43 -05:00
Lex Berezhny a234f92d2c 0.30.1rc8 2018-12-05 11:53:37 -05:00
Lex Berezhny 8ba3575901 dropped aiorpcx 2018-12-05 09:17:36 -05:00
Jack Robison a857c83c59 Merge pull request #1685 from lbryio/fix-1674
fix int too large to convert to integer when updating claim
2018-12-04 18:16:24 -05:00
Jack Robison a6128ac8a2 fix https://github.com/lbryio/lbry/compare/fix-1674 2018-12-04 17:35:40 -05:00
hackrush 2693301ad0 Fix python and pip3 not available in xcode8 2018-12-04 15:12:07 -05:00
Victor Shyba f7c7cb9535 fix transaction_show 2018-12-04 12:12:08 -05:00
Jack Robison b36a5f9b6b Merge pull request #1678 from lbryio/warn_migration_bad_claim_id
check and log claim id validity during migration
2018-11-30 15:58:26 -05:00
Jack Robison 4826eb9969 migrate double hex encoded certificates 2018-11-30 14:00:35 -05:00
Victor Shyba c34b150794 check and log claim id validity during migration 2018-11-30 14:00:35 -05:00
Lex Berezhny c4c7670a7c do not run brew update 2018-11-30 12:20:07 -05:00
Lex Berezhny 0918bac86e mac build stuff 2018-11-30 11:50:32 -05:00
Victor Shyba a60a921706 resolve claims on claim_list (#1483) 2018-11-30 11:46:56 -05:00
Lex Berezhny b3ee29b66d maybe python is already installed on mac 2018-11-30 11:30:03 -05:00
Lex Berezhny 3a1a9b988c on mac brew, python 3 is installed with plain python while python2 now requires python@2 2018-11-30 10:55:15 -05:00
Lex Berezhny 12f2bebc46 build for mac using xcode8 2018-11-30 10:34:45 -05:00
Jack Robison 31a3812c8a bump 2018-11-29 17:23:32 -05:00
Lex Berezhny 70ec08e30b icons! 2018-11-29 17:15:18 -05:00
Jack Robison 23e8f0d849 Merge pull request #1676 from lbryio/fix-1573
unlock wallet after encrypting it
2018-11-29 16:13:21 -05:00
Jack Robison 1cdfcd5b4e unlock wallet after encrypting it 2018-11-29 15:16:53 -05:00
Lex Berezhny dc2786fbc0 assert instead of if 2018-11-28 18:03:33 -05:00
Lex Berezhny 7e39b0c4fe run ClaimManagement.test_create_update_and_abandon_claim 2018-11-28 18:03:33 -05:00
Lex Berezhny 4af830c405 tests for transaction_list going through claim creation -> claim update -> claim abandon 2018-11-28 18:03:33 -05:00
Lex Berezhny fe844581fd correctly split value/fee in transaction_list 2018-11-28 18:03:33 -05:00
Lex Berezhny 443ce6352b typo 2018-11-28 18:03:33 -05:00
Lex Berezhny 158c354a27 split out fee from value when possible 2018-11-28 18:03:33 -05:00
Lex Berezhny 92d1230a90 remove fees from value, abandon is positive 2018-11-28 18:03:33 -05:00
Jack Robison d64916a061 bump 2018-11-27 16:28:55 -05:00
Alyssa Callahan 20a0568253 Merge pull request #1567 from preethamvishy/master
Update pull request template
2018-11-27 15:45:54 -05:00
Alyssa Callahan 5a1f42ee54 Merge branch 'master' into master 2018-11-27 15:39:48 -05:00
Alyssa Callahan ab95f14e21 Update PULL_REQUEST_TEMPLATE.md 2018-11-27 15:38:06 -05:00
Jack Robison bae2676b69 Merge pull request #1669 from lbryio/fix_invalid_sigs_from_name
allow invalid sigs claims to show outside a channel
2018-11-27 14:41:28 -05:00
Victor Shyba ea6869a63e allow invalid sigs claims to show outside a channel 2018-11-27 15:36:53 -03:00
Jack Robison 953327c150 bump 2018-11-27 09:52:33 -05:00
Lex Berezhny e01ad6bc85 disable travis cache on mac build 2018-11-26 20:55:03 -05:00
Victor Shyba 47a33009ff remove python 3.6 steps from travis 2018-11-26 20:55:03 -05:00
Victor Shyba bc844013a1 (win build) add pywin32 and netifaces wheels 2018-11-26 20:55:03 -05:00
Victor Shyba 2fd2319cdb use lbry dockerhub image 2018-11-26 20:55:03 -05:00
Victor Shyba a50657c472 add dockerfile and readme on how win docker builder is generated 2018-11-26 20:55:03 -05:00
Victor Shyba ef5f6e06ba use vshyba/temporary_pyinstaller_32bits_py37:latest 2018-11-26 20:55:03 -05:00
Victor Shyba 977cf9f506 log python version on windows build script 2018-11-26 20:55:03 -05:00
Lex Berezhny ffae10d11f drop pytest 2018-11-26 18:02:12 -05:00
Thomas Zarebczan 221107e76e test mac build, do not merge (#1659)
* Update .travis.yml to use xcode 6.4 for mac build
2018-11-26 14:13:45 -05:00
Jack Robison e74c78b607 Merge pull request #1432 from lbryio/get-block-on-bytes-written
[API] block get on bytes being written to the output file
2018-11-26 12:16:42 -05:00
Jack Robison 434c7b57b1 tests 2018-11-26 11:36:28 -05:00
Jack Robison 9dc4608a62 combine StreamProgressManager and FullStreamProgressManager
-add wrote_first_data Deferred to FullStreamProgressManager
2018-11-26 11:36:28 -05:00
Jack Robison fc69366092 set the download manager before creating the first deferred 2018-11-26 11:36:28 -05:00
Victor Shyba 9f0b1f3e25 fix pylint failures unveiled after astroid pin 2018-11-26 11:32:41 -05:00
Victor Shyba 3f03a845ec only return invalid signatures if its a direct resolution 2018-11-26 11:32:41 -05:00
Victor Shyba bc28976e12 adds test for claims on abandoned channels 2018-11-26 11:32:41 -05:00
Victor Shyba 8f826098f9 validate claim id before resolving 2018-11-26 11:32:41 -05:00
Victor Shyba 0a765e6207 handle errors on resolve 2018-11-26 11:32:41 -05:00
Lex Berezhny f83aa61923 force astroid==2.0.4 in .travis.yml 2018-11-26 10:47:48 -05:00
Oleg Silkin f9ff3e5d14 Removes third-party mock dependency (#1661)
* Removes mock from setup.py dependencies
* Refactors name resetTime to reset_time
* Replaces import mock with built-in unittest.mock
* Refactors imports to separate built-in, external, and project modules
2018-11-25 11:12:30 -05:00
Oleg Silkin b3fde9d78d Removes six From Project (#1660)
* Replaces `six` urllib with python 3's urllib
* Replaces `six`'s int2byte method with native `bytes` class
* Removes edge case testing for python2 vs python3
* Removes useless object inheritance
* Uses native io.BytesIO instead of six.BytesIO
* Removes six from dependencies
2018-11-22 23:24:21 -05:00
Lex Berezhny 0c999083cc added jsonrpc_account_send and dropped torba[server] 2018-11-21 01:14:39 -05:00
Victor Shyba 561215db8a fix download script 2018-11-21 00:25:43 -05:00
Jack Robison 8dbb7f2c78 bump 2018-11-20 21:37:49 -05:00
Lex Berezhny 6f7f3bb316 fixes #1653 2018-11-20 16:12:51 -05:00
Jack Robison 8d507c4b73 bump 2018-11-19 21:12:39 -05:00
Lex Berezhny 2beb0eda94 SupportingSupports test does not exist 2018-11-19 20:34:38 -05:00
Lex Berezhny b7e07b72ca update tox.ini to run correct tests 2018-11-19 20:34:38 -05:00
Lex Berezhny fde8718c8c make sure only actual claims/updates are found when updating a claim or sending it to someone 2018-11-19 20:34:38 -05:00
Lex Berezhny b2d1bddcfe update to latest torba 2018-11-19 20:34:38 -05:00
Lex Berezhny dd1d1be994 working claim_send_to_address 2018-11-19 20:34:38 -05:00
Jack Robison a93957178a bump 2018-11-19 17:31:31 -05:00
Jack Robison 5026d0b2f3 Merge branch 'fix-get-bad-claim-error' 2018-11-19 17:31:01 -05:00
Jack Robison cc8b47d882 improve error when get fails to decode the resolved claim 2018-11-19 17:30:02 -05:00
Jack Robison db0b5961b3 Merge pull request #1642 from jessopb/readmeStuff
adds daemon config to and updates links in readme
2018-11-19 14:37:15 -05:00
Jessop Breth cb6f7ada42 adds sample config and links to settings documentation 2018-11-19 10:10:02 -05:00
Jack Robison 59464099b4 add example daemon_settings.yml 2018-11-19 09:42:13 -05:00
preserveddarnell aed00b53fd Update various typos & edits 2018-11-18 20:52:18 -05:00
YULIUS KURNIAWAN KRISTIANTO cdb0f97cb4 added more social links 2018-11-18 10:29:47 -05:00
Jack Robison 33bcc8256c Merge branch 'fix-1624' 2018-11-16 11:45:05 -05:00
Jack Robison 0a8d44f12f Merge branch 'populate-buckets' 2018-11-16 11:44:25 -05:00
Jack Robison 1b74594f5b don't create ssl/auth keys if disabled
fixes https://github.com/lbryio/lbry/issues/1624
2018-11-16 11:12:33 -05:00
Jack Robison b0718d295e fix empty buckets in routing_table_get 2018-11-16 10:59:12 -05:00
Jack Robison 9bb91b126d use midpoint id instead of random for refresh list 2018-11-16 10:59:12 -05:00
Jack Robison 6dbb00d8d5 populate buckets when joining the dht 2018-11-16 10:59:12 -05:00
Jack Robison 3a27a4509a Merge pull request #1639 from lbryio/fix-1638
fix bytes/str in iterativeFindValue
2018-11-14 16:22:30 -05:00
Jack Robison 1d438f5d23 fixes https://github.com/lbryio/lbry/issues/1638 2018-11-14 15:35:59 -05:00
YULIUS KURNIAWAN KRISTIANTO 3840f19869 fix mistake code 2018-11-14 14:20:02 -05:00
Lex Berezhny ec0483b975 header progress percent starts at 0 instead of None 2018-11-12 21:16:05 -05:00
Jack Robison 2fba1f417f less verbose logging 2018-11-12 14:47:11 -05:00
Jack Robison 5b419b55c5 fix peer_list 2018-11-12 14:45:41 -05:00
Victor Shyba b31379e51c log and handle misformed certificate claims from #1609 2018-11-10 20:49:36 -05:00
Victor Shyba 8394dcb663 log and handle misformed claims from #1611 2018-11-10 20:49:36 -05:00
Lex Berezhny 457a56ce04 Merge pull request #1604 from lbryio/fix-external-ip
Fix setting the external ip
2018-11-10 20:48:40 -05:00
Jack Robison 7c8016d807 remove accidentally added lbrynet.schema.signature 2018-11-09 16:52:46 -05:00
Jack Robison d704336f20 absolute imports 2018-11-09 16:52:46 -05:00
Jack Robison 1b4230b06e move is_valid_blobhash to lbrynet.blob.blob_file 2018-11-09 16:52:46 -05:00
Jack Robison 88095af1cd move system_info.py into lbrynet.extras 2018-11-09 16:52:46 -05:00
Jack Robison e9efd9f68a improve get_external_ip 2018-11-09 16:52:46 -05:00
Jack Robison 7b32e2ff7c don't block the dht component on populating the routing table 2018-11-09 16:52:46 -05:00
Jack Robison 572777305e reduce analytics event frequency 2018-11-09 16:52:46 -05:00
Jack Robison 7e27a6e37d fix 2018-11-09 16:52:46 -05:00
Jack Robison b98621d7ea avoid name errors 2018-11-09 16:52:46 -05:00
Jack Robison bc5824b8bc better setup of custom logger class 2018-11-09 16:52:46 -05:00
Jack Robison a6b643bb34 move files 2018-11-09 16:52:42 -05:00
Jack Robison 488b976734 move HashAnnouncer.py to extras 2018-11-09 14:16:33 -05:00
Jack Robison 253912b52e fix skipping dht, peer_protocol_server, and hash_announcer components
-move PeerFinder and PeerManager initialization to the ComponentManager

-remove dht component requirement from all but the hash_announcer component. This allows running the file manager component without either of the server components.
2018-11-09 14:16:33 -05:00
Jack Robison 0bf8416d9f fix components_to_skip setting 2018-11-09 14:16:33 -05:00
Jack Robison 2e921437d8 fix external ip setup
fixes https://github.com/lbryio/lbry/issues/1561
2018-11-09 14:16:33 -05:00
Jack Robison b6f9b60683 remove commit in version for source installs 2018-11-09 14:16:33 -05:00
Lex Berezhny 57430e9f5f fixed #1616 2018-11-09 12:03:55 -05:00
Victor Shyba 1e13fbee8d await missing on resolve.py 2018-11-09 11:11:57 -05:00
Lex Berezhny 3d8b9e2b66 block=False for abandon during erics epic adventure tests 2018-11-07 18:28:02 -05:00
Lex Berezhny 64898d43f1 tearDown() should run before daemon shutdown 2018-11-07 18:28:02 -05:00
Lex Berezhny b2117fddab on_transaction -> ledger.wait 2018-11-07 18:28:02 -05:00
Lex Berezhny 5658623f29 claims include supports 2018-11-07 18:28:02 -05:00
Lex Berezhny 934682bd0f --blocking support in abandon command 2018-11-07 18:28:02 -05:00
Victor Shyba daecc582f3 simplify bind statement 2018-11-07 11:29:56 -05:00
Victor Shyba 3cb4012dbf unify loop 2018-11-07 11:29:56 -05:00
Victor Shyba f7bd44a4f5 fix query for windows (999 vars limit) 2018-11-07 11:29:56 -05:00
hackrush 52317b05bb Better help message instructions 2018-11-07 11:02:20 -05:00
Lex Berezhny 83c0baa3cb test abandoning claim at a loss, required a fix in torba 2018-11-07 10:57:11 -05:00
Lex Berezhny 340c5addab insufficientfundserror test 2018-11-07 10:57:11 -05:00
Lex Berezhny e3fd94dbf0 pylint fix 2018-11-07 10:57:11 -05:00
Lex Berezhny 923fcfb9e9 run publish integration tests separately 2018-11-07 10:57:11 -05:00
Lex Berezhny 0ae5f1a4c6 added fix for #1533 2018-11-07 10:57:11 -05:00
Lex Berezhny 90e06abba2 fixes for #1569 2018-11-07 10:57:11 -05:00
Victor Shyba 6ecfc94ff1 add balance to startup report 2018-11-07 10:50:13 -05:00
Alyssa Callahan 60c4a727d3 Merge pull request #1574 from czakarian/patch-1
Fix typos / grammar changes
2018-11-07 10:19:59 -05:00
Alyssa Callahan 9fbb9f2e98 Merge branch 'master' into patch-1 2018-11-07 10:18:58 -05:00
Lex Berezhny f1feec66c0 fixed path to cli.py in setup.py 2018-11-06 13:55:22 -05:00
Lex Berezhny 73b9b983b1 pinned protobuf to specific version, fixes #1602 2018-11-06 00:05:40 -05:00
Victor Shyba 588779e3a4 makefile for using types repo 2018-11-05 20:48:21 -05:00
Jack Robison b7ae446059 Merge pull request #1495 from lbryio/retry_dht
[DHT] retry on protocol level
2018-11-05 15:16:53 -05:00
Victor Shyba db898307b5 obey the rpcAttempts constant on retries 2018-11-05 15:16:39 -03:00
Victor Shyba 7e3512cbf2 retry on protocol level 2018-11-05 15:16:39 -03:00
Lex Berezhny cdd65c6783 updated pyinstaller build instructions 2018-11-04 20:25:24 -05:00
Lex Berezhny 320ad717cd pylint cleanup 2018-11-04 20:25:24 -05:00
Lex Berezhny 264e6744e1 updated references after moving conf to lbrynet.extras.daemon.conf 2018-11-04 20:25:24 -05:00
Lex Berezhny 64e2b79c07 moved conf into daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 9255bb5abe updated references after moving analytics to lbrynet.extras.daemon.analytics 2018-11-04 20:25:24 -05:00
Lex Berezhny d7525dacc3 moved analytics to lbrynet/extra/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 5cb07100f8 deleted unused interfaces file 2018-11-04 20:25:24 -05:00
Lex Berezhny 171c080d33 updated references lbrynet.file_manager -> lbrynet.blob 2018-11-04 20:25:24 -05:00
Lex Berezhny cfc3ec2a05 merged lbrynet/file_manager into lbrynet/blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 1392b8f83b removed lbrynet/lbry_file/__init__.py 2018-11-04 20:25:24 -05:00
Lex Berezhny 8b265801df updated references lbrynet.lbry_file.client -> lbrynet.file_manager.client 2018-11-04 20:25:24 -05:00
Lex Berezhny 6c4fb8dab1 moved lbrynet/lbry_file/client into lbrynet/file_manager/client 2018-11-04 20:25:24 -05:00
Lex Berezhny c1a9cd93ee updated references lbrynet.crypstream -> lbrynet.blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 0d4e83e167 merged lbrynet/cryptstream into lbrynet/blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 1292eb345d fix dbmigrator pylint issue 2018-11-04 20:25:24 -05:00
Lex Berezhny f4c6dc90c9 updated references lbrynet.core -> lbrynet.p2p 2018-11-04 20:25:24 -05:00
Lex Berezhny 8c8f5d8f58 renamed lbrynet/core -> lbrynet/p2p 2018-11-04 20:25:24 -05:00
Lex Berezhny b6d986cb40 updated references after moving lbrynet/database into lbrynet/extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 6cc134c7c8 merged lbrynet/database with lbrynet/extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny eee964cfa0 updated references to daemon after moving it to extras 2018-11-04 20:25:24 -05:00
Lex Berezhny a03c94bfad moved daemon to extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny f1f0917063 updated all references to point to new reflector location 2018-11-04 20:25:24 -05:00
Lex Berezhny fe2476b5aa moved reflector into extras 2018-11-04 20:25:24 -05:00
Lex Berezhny 9c5fa8bf76 winhelpers/winpaths.py -> extras/winpaths.py 2018-11-04 20:25:24 -05:00
Lex Berezhny cd43a71172 update winpaths.py references 2018-11-04 20:25:24 -05:00
Lex Berezhny e2c6de76f4 renamed knownpaths.py -> winpaths.py 2018-11-04 20:25:24 -05:00
Lex Berezhny 05aeadc9fe extraneous LICENSE file since the whole project is already MIT, moved copyright directly into file header 2018-11-04 20:25:24 -05:00
Lex Berezhny 7153417372 #egg=torba[server] 2018-11-04 10:50:15 -05:00
Lex Berezhny fceb2ae0a2 hmm 2018-11-04 10:50:15 -05:00
Lex Berezhny 0cef2fdd82 install extras from github 2018-11-04 10:50:15 -05:00
Lex Berezhny 9c91b135bd pylint requires [wallet-server] 2018-11-04 10:50:15 -05:00
Lex Berezhny e3b127192c install .[test] includes the [wallet-server] 2018-11-04 10:50:15 -05:00
Lex Berezhny ee1738da6e lbrynet.extras.wallet.server pylint fixes 2018-11-04 10:50:15 -05:00
Lex Berezhny 8ca2b75c80 lbryumx -> lbrynet.extras.wallet.server 2018-11-04 10:50:15 -05:00
Lex Berezhny dc81b6def1 install lbryumx after lbrynet so that we dont get the old protobuf 2018-11-04 10:50:15 -05:00
Lex Berezhny 22f5aaee70 regenerated protobufs 2018-11-04 10:50:15 -05:00
Lex Berezhny 4bd6569c9c lbrynet.schema.schema 2018-11-04 10:50:15 -05:00
Lex Berezhny dbf39baeb1 fix lbrynet.schema import 2018-11-04 10:50:15 -05:00
Lex Berezhny 3adf3c26f2 add set_build.py back in 2018-11-04 10:50:15 -05:00
Lex Berezhny fa0d13e630 protobuf==3.2.0 2018-11-04 10:50:15 -05:00
Lex Berezhny 227a82d6d3 update protobuf version 2018-11-04 10:50:15 -05:00
Lex Berezhny 65cfc1f63c removed lbryschema references from scripts and copied lbryschema requirements into local setup.py 2018-11-04 10:50:15 -05:00
Victor Shyba e46f0e2444 fix namespace lbryschema->lbrynet.schema 2018-11-04 10:50:15 -05:00
Victor Shyba 00bc7995c4 move lbryschema/tests/ into tests/unit/schema/ 2018-11-04 10:50:15 -05:00
Victor Shyba 2dcebe48b7 move lbryschema/lbryschema into lbrynet/schema 2018-11-04 10:50:15 -05:00
Lex Berezhny d12b6cddb3 fix integration tests 2018-11-04 02:28:36 -05:00
Lex Berezhny 699c057f2d upgrading to new torba 2018-11-04 02:28:36 -05:00
Lex Berezhny 3ce9f31f4b recursive channel lookup 2018-11-04 02:28:36 -05:00
Lex Berezhny 485d5d55e8 pylint fix 2018-11-04 02:28:36 -05:00
Lex Berezhny 6ee9f2513b add extra debug logging when a channel is not a claim errors 2018-11-04 02:28:36 -05:00
Daniel Brereton 81a6e14a5c added alt text 2018-11-02 19:38:30 -04:00
czakarian 496ffe0096 Fix typos / grammar changes 2018-10-31 16:35:39 -07:00
Preetham Viswanathan 216c760179 Merge pull request #1 from preethamvishy/preethamvishy/pr-template
Update pull request template
2018-10-30 17:16:56 -04:00
Preetham Viswanathan 01eca3004f Update PULL_REQUEST_TEMPLATE.md 2018-10-30 16:57:41 -04:00
GwanYeong Kim 2f47a3758e Remove unused Python modules in tests/unit/test_cli.py 2018-10-30 10:32:34 -04:00
pushkyn 73bcc1796f Add GitHub pull request template 2018-10-30 10:27:47 -04:00
Jack Robison e03e41ad31 bump 2018-10-26 16:51:23 -04:00
Jack Robison 6cb30f8e05 Merge branch 'fix-update-stream'
[API] add `aioupnp_version` to `upnp` field in the response to `status`
[API] fix uncaught claim signature errors
2018-10-26 16:49:51 -04:00
Jack Robison d9b1be6214 add aioupnp version to upnp status
-set non-blank not found message if upnp fails
2018-10-26 16:48:52 -04:00
Jack Robison 5f2f06c269 fix loggly 2018-10-26 13:01:50 -04:00
Jack Robison e1dcdcf4db catch claim signature error 2018-10-26 13:01:50 -04:00
Jack Robison 416472c5d8 log stream update failure 2018-10-26 13:01:46 -04:00
Alyssa Callahan 196f3ba92d added Internal use section 2018-10-24 15:47:07 -04:00
Lex Berezhny 6feae93c58 Update lbrynet/wallet/claim_proofs.py 2018-10-23 12:02:10 -04:00
Lex Berezhny 203d30d4f7 Update lbrynet/wallet/claim_proofs.py 2018-10-23 12:02:10 -04:00
Hugo d0a7ca841b Upgrade unit tests to use more useful asserts 2018-10-23 12:02:10 -04:00
Hugo 2e6c848c39 Upgrade Python syntax with pyupgrade --py36-plus 2018-10-23 12:02:10 -04:00
Jack Robison 18a0204105 set_build regex 2018-10-22 22:27:57 -04:00
Jack Robison 01baa193d5 fix set_build 2018-10-22 20:58:52 -04:00
Jack Robison ba57af5502 fix set_build 2018-10-22 17:55:32 -04:00
Jack Robison 8139f93ff1 set build type 2018-10-22 16:31:05 -04:00
Jack Robison 4596d5c26f bump 2018-10-22 09:12:49 -04:00
Nikita Titov 810a63782d Fixed long description of package 2018-10-20 10:15:29 -04:00
Nikita Titov ea83827d57 Update macOS naming 2018-10-20 10:02:24 -04:00
Jack Robison e19f8f3d95 bump 2018-10-19 12:28:08 -04:00
Jack Robison 7af501e0bc Merge branch 'temp-blob-pass-sqlite-object' 2018-10-19 12:27:27 -04:00
hackrush a692b6ff7d Pass SQLite storage object to DiskBlobManager for single(temp) blob download 2018-10-19 10:39:39 -04:00
Jack Robison 3364ac0976 bump 2018-10-18 18:22:46 -04:00
Jack Robison cf1157f02a Merge branch 'upnp-status' 2018-10-18 18:21:53 -04:00
Jack Robison 0ed56bbe1a tests 2018-10-18 18:12:50 -04:00
Jack Robison 213bdac55f check upnp redirects and external ip in a looping call
-add upnp setup success/fail analytics event
2018-10-18 17:41:49 -04:00
Jack Robison 6d8ab97ca9 fix analytics 2018-10-18 17:41:17 -04:00
Jack Robison fb34476cd7 add upnp status dictionary to status response
-log upnp debugging information as an error
2018-10-18 15:10:00 -04:00
Jack Robison 07d353a947 bump 2018-10-18 13:19:25 -04:00
Jack Robison 3c5fb6490d catch errors 2018-10-18 13:00:27 -04:00
Jack Robison a5d21081b9 Merge pull request #1549 from hugovk/link-to-key
Add link to GPG key
2018-10-18 12:41:42 -04:00
Jack Robison ed98a3cc21 Merge branch 'master' into link-to-key 2018-10-18 12:39:39 -04:00
Jack Robison 45c2cd2986 Merge pull request #1550 from hugovk/fix-spelling
Fix spelling
2018-10-18 12:37:56 -04:00
Hugo 75f21950da Add link to GPG key
And fix some typos.
2018-10-18 14:44:48 +03:00
Hugo 3f704be85c Fix spelling 2018-10-18 14:40:37 +03:00
Jack Robison 67a15ea788 bump 2018-10-18 00:34:58 -04:00
Jack Robison 55bcd2dfee Merge branch 'fix-asyncio-build' 2018-10-18 00:34:04 -04:00
Jack Robison d9757fdae8 remove asyncio from requirements 2018-10-18 00:32:44 -04:00
Jack Robison a4ae8d7c53 bump 2018-10-17 21:36:54 -04:00
Jack Robison 78078b4c9f Merge remote-tracking branch 'origin/fix_dht_logging' 2018-10-17 21:26:46 -04:00
Victor Shyba 1331e2bbe6 fix DHT logging 2018-10-17 22:25:41 -03:00
Lex Berezhny 4c4d652f1b fixes for #1526 and #1483 2018-10-17 21:10:23 -04:00
Lex Berezhny 73bd76b631 format publish output bid value in lbc instead of dewies 2018-10-17 19:50:31 -04:00
Lex Berezhny 59a0f9f602 removed the max response size from Known Issues as it was fixed 2018-10-17 19:38:14 -04:00
Lex Berezhny 1b1847bc31 added Known Issues section 2018-10-17 19:38:14 -04:00
Lex Berezhny 21a569edcc updated CHANGELOG for v0.30 2018-10-17 19:38:14 -04:00
Lex Berezhny 2ddf1a08f6 publish command supports custom account list to lookup channels 2018-10-17 19:37:54 -04:00
Victor Shyba 8754c6fabe fix resolve_name for asyncio 2018-10-17 17:17:25 -04:00
Victor Shyba f049109a38 fix claim_show 2018-10-17 17:17:25 -04:00
Victor Shyba 8db7c15aa1 fix address query handler 2018-10-17 15:33:22 -04:00
Victor Shyba 90e0694d2e fix download from peer script 2018-10-17 15:33:22 -04:00
Victor Shyba 4748cbf654 fix tests for cost estimate 2018-10-17 14:16:00 -04:00
Victor Shyba 74eb991983 fix paid downloads 2018-10-17 14:16:00 -04:00
Victor Shyba e11bae2dda stream cost estimate fix for uri without size 2018-10-17 14:16:00 -04:00
Victor Shyba 8462f56b13 fix blob_list for uris 2018-10-17 14:16:00 -04:00
Victor Shyba 0bec3c7294 fix claim_list_by_channel 2018-10-17 14:16:00 -04:00
Victor Shyba 2bb4052193 fix claim_list 2018-10-17 14:16:00 -04:00
Lex Berezhny 101a3f2b22 respect --show_seed argument to account_list command 2018-10-17 13:45:01 -04:00
Lex Berezhny 04a2bde8dd removed extraneous asyncio import 2018-10-17 13:29:28 -04:00
Jack Robison a1368b05da txupnp --> aioupnp
-update INSTALL.md
2018-10-17 13:29:28 -04:00
Victor Shyba 99ea7a26b0 fix get for asyncio torba 2018-10-17 10:27:57 -04:00
Lex Berezhny 8b3a457925 improve integration test reliability 2018-10-16 21:28:47 -04:00
Lex Berezhny e02e2a249a fix wallet component 2018-10-16 17:09:21 -04:00
Lex Berezhny 3019a1c222 revert tox config 2018-10-16 17:09:21 -04:00
Lex Berezhny 1f3a19c100 pylint 2018-10-16 17:09:21 -04:00
Lex Berezhny 5eac4247e1 working integration tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 1ca82b9d52 major facepalm 2018-10-16 17:09:21 -04:00
Lex Berezhny 21702ec43c asyncio progress for victor 2018-10-16 17:09:21 -04:00
Lex Berezhny d2083bc17e for victor 2018-10-16 17:09:21 -04:00
Lex Berezhny 52edf880e0 fix typo in .travis.yml file 2018-10-16 17:09:21 -04:00
Lex Berezhny 1f45af991f install dependencies needed for asyncio unit tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 8afdd0b2e9 pylint fix 2018-10-16 17:09:21 -04:00
Lex Berezhny 99efc70ba3 run wallet unit tests separate from the other twisted trial based tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 64e306801d updated wallet to use asyncio 2018-10-16 17:09:21 -04:00
Victor Shyba 330db61b36 report balance and summary on startup 2018-10-12 20:53:31 -04:00
Lex Berezhny 357763ff61 respect blockchain_name in daemon settings when creating initial wallet 2018-10-12 12:28:39 -04:00
Lex Berezhny 8a6e6b4d77 add category attribute in JSON RPC output for backwards compatibility 2018-10-12 11:08:01 -04:00
Lex Berezhny 64913c4288 JSON RPC output for TXOs now includes: height, confirmations and valid_signature 2018-10-12 10:41:49 -04:00
Victor Shyba f01ad0e4bd use smart_decode on transaction parsing to support legacy claims 2018-10-11 17:38:23 -04:00
Victor Shyba d9fcc458dc refactor part of peer logic out of try/catch 2018-10-10 15:27:03 -04:00
Victor Shyba 24a11d3f21 make peer search asynchronous 2018-10-10 15:27:03 -04:00
Victor Shyba 234eaf8890 fix download_blob_from_peer script 2018-10-10 15:27:03 -04:00
Tomasz Kopeć 9e2e53147e --conf option improvements (#1455)
* Conf file improvements
* Add test for loading config file
2018-10-10 13:44:51 -04:00
Victor Shyba fc41af5889 set wallet max_gap after sync 2018-10-10 13:38:14 -04:00
Victor Shyba 72ce0cd0b0 explicitly define a default for signatures as expected on 0.21 2018-10-10 11:36:20 -04:00
Jack Robison e4497c1327 Merge remote-tracking branch 'origin/fix-1414' 2018-10-10 11:16:45 -04:00
hackrush a1bf832ea6 fix blob_availability 2018-10-10 16:21:54 +05:30
Lex Berezhny d4e76500fb removed unused import 2018-10-10 00:02:50 -04:00
Lex Berezhny f898bb3d0e database index fix 2018-10-09 23:58:32 -04:00
Lex Berezhny ad13282494 txo (claim_id) column index 2018-10-09 23:40:02 -04:00
Lex Berezhny 73519f8c6a claim_name updated to use new account.get_claims() method 2018-10-09 21:46:54 -04:00
Lex Berezhny b2c95f35c7 fixed test_transaction reliability 2018-10-09 21:32:32 -04:00
Lex Berezhny 0bfccb86ff refactored pagination per @seanyesmunt feedback 2018-10-09 20:46:41 -04:00
Lex Berezhny 5118240897 fix log message in header downloader to log local_height as integer instead of float 2018-10-09 17:39:28 -04:00
Lex Berezhny 427fc179ac fix for #1523 2018-10-09 16:48:12 -04:00
Lex Berezhny 55bcc520e4 pagination method fixes and not showing fee for recieved TX in get_history 2018-10-09 14:46:44 -04:00
Lex Berezhny bc8251696e fix cli version test 2018-10-09 11:37:49 -04:00
Lex Berezhny 677ba25ccd use .format() instead of string concatenation 2018-10-09 11:15:11 -04:00
Alex Grintsvayg 95436b233c fix lbrynet --version 2018-10-09 11:15:11 -04:00
Lex Berezhny 82871b8f6b fix command line argument for account_balance and other paginated commands 2018-10-09 10:30:30 -04:00
Lex Berezhny 9525b87224 inlineCallbacks fix 2018-10-08 22:44:00 -04:00
Lex Berezhny f06098978c ledger.get_transaction dropped in favor of ledger.db.get_transaction 2018-10-08 19:51:44 -04:00
Jack Robison 141491f2da bump version 2018-10-08 16:11:36 -04:00
Victor Shyba 38006f7d29 migrate addresses before starting the migrated wallet 2018-10-08 15:29:57 -04:00
Lex Berezhny 3664c25d98 changed lbry account.get_balance() to have same confirmations default as baseaccount.get_balance() 2018-10-08 15:02:52 -04:00
Lex Berezhny 1747c9d866 removed --include_reserved from account_list command 2018-10-08 15:02:52 -04:00
Lex Berezhny 1066fbaa7c pylint fix 2018-10-08 15:02:52 -04:00
Lex Berezhny 7f7961f4cc updated integration tests after reducing required confirmations to 0 2018-10-08 15:02:52 -04:00
Lex Berezhny 9d9916548b added channel/signature annotation to all TXOs, support for pagination 2018-10-08 15:02:52 -04:00
Lex Berezhny ada5b55f22 remove unused import 2018-10-08 15:02:52 -04:00
Lex Berezhny a2f28e7788 + permanent_url and TXO annotations 2018-10-08 15:02:52 -04:00
Jack Robison c6092a4c79 bump version 2018-10-08 09:16:54 -04:00
Jack Robison 315a557019 Merge remote-tracking branch 'origin/1439' 2018-10-08 09:15:46 -04:00
Jack Robison 20221e1186 Merge remote-tracking branch 'origin/reduce_routing_table_ping_flood' 2018-10-08 09:15:30 -04:00
Victor Shyba 50003d2600 restore delay on ping queue 2018-10-03 19:21:48 -03:00
Victor Shyba ea6b2b98fb refactor ping queue 2018-10-03 19:21:48 -03:00
Victor Shyba 4890fdfb50 remove semaphore from ping queue 2018-10-03 19:21:48 -03:00
Lex Berezhny 2f95c3a9d1 wallet integration test fixes 2018-10-03 18:06:22 -04:00
Lex Berezhny 6039159ecc use d2l/l2d in integration tests 2018-10-03 18:06:22 -04:00
Lex Berezhny 8e4d8345bd removed unused import 2018-10-03 18:06:22 -04:00
Lex Berezhny 22df26eeb6 dewies_to_lbc and lbc_to_dewies 2018-10-03 18:06:22 -04:00
Lex Berezhny 7b9ff3e8b5 updated with torba refactoring and working claim_send_to_address 2018-10-03 12:32:01 -04:00
Jack Robison 1a5b2c08ee update INSTALL.md 2018-10-02 13:31:29 -04:00
Jack Robison b6babb1c9d bump 2018-10-02 11:07:51 -04:00
Jack Robison a88485c5a0 Merge branch 'fix-missing-wallet-dir' 2018-10-02 11:06:59 -04:00
Jack Robison 0cefa07e97 Merge branch 'fix-1501' 2018-10-02 11:06:35 -04:00
Jack Robison ca9d48d574 fix wallet directory not being created 2018-10-02 10:34:05 -04:00
Jack Robison e10e1de92f [API] fix https://github.com/lbryio/lbry/issues/1501 2018-10-02 10:26:02 -04:00
Jack Robison bdf6887791 Merge branch 'fix-reactor-tests' 2018-10-02 09:09:34 -04:00
Jack Robison 295f2c1115 update INSTALL.md and README.md 2018-10-01 19:42:23 -04:00
Jack Robison 78fcf6a44a add asyncio to requirements 2018-10-01 18:42:28 -04:00
Jack Robison 7dcebf99e0 don't delete the reactor from sys.modules unless in a build 2018-10-01 18:41:45 -04:00
Victor Shyba 463cbde4b1 cast dict_keys to list before using as shortlist 2018-10-01 15:23:50 -03:00
Victor Shyba 1adce8b227 reduce routing table ping flood 2018-10-01 15:17:10 -03:00
Jack Robison 673d259d7a Merge branch '1446' 2018-10-01 08:34:57 -04:00
Jack Robison d0b739a0b9 Merge branch 'fix_peer_ping' 2018-10-01 08:32:30 -04:00
Victor Shyba b4aa51444c fix peer_ping for inexistent peers 2018-10-01 02:09:47 -03:00
Victor Shyba 9178ca701c nullify and refresh tokens 2018-09-28 14:19:51 -03:00
Victor Shyba 7eb9f344f4 test recover from invalid tokens 2018-09-28 13:47:37 -03:00
Victor Shyba 9f8d3e69ae implement get_best_blockhash 2018-09-27 15:30:53 -04:00
Victor Shyba e1130147b5 fix format of resolve on inexistent URI 2018-09-27 15:30:53 -04:00
287 changed files with 13498 additions and 9316 deletions

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
.git
.tox
__pycache__
dist
lbrynet.egg-info
docs
tests
+21
View File
@@ -51,3 +51,24 @@ Tell us what happens instead
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
+38
View File
@@ -0,0 +1,38 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+2 -2
View File
@@ -9,7 +9,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
ignore=CVS,schema
# Add files or directories matching the regex patterns to the
# blacklist. The regex matches against base names, not paths.
@@ -62,7 +62,7 @@ confidence=
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# disable everything first and then re-enable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
+51 -29
View File
@@ -9,44 +9,35 @@ jobs:
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install git+https://github.com/lbryio/torba.git
- pip install git+https://github.com/lbryio/lbryschema.git
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
- &tests
stage: test
name: "Unit Tests w/ Python 3.7"
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git
- pip install git+https://github.com/lbryio/lbryschema.git
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .[test]
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit
script:
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- <<: *tests
name: "Unit Tests w/ Python 3.6"
python: "3.6"
- <<: *tests
name: "DHT Tests w/ Python 3.7"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- <<: *tests
name: "DHT Tests w/ Python 3.6"
python: "3.6"
name: "DHT Tests"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/electrumx.git --branch lbryumx && popd
- pushd .. && git clone https://github.com/lbryio/orchstr8.git && popd
- pushd .. && git clone https://github.com/lbryio/lbryschema.git && popd
- pushd .. && git clone https://github.com/lbryio/lbryumx.git && cd lbryumx && git checkout afd34f323dd94c516108a65240f7d17aea8efe85 && cd .. && popd
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
script: tox
after_success:
@@ -59,32 +50,50 @@ jobs:
services:
- docker
install:
- docker pull cdrx/pyinstaller-windows:python3-32bit
- docker pull lbry/pyinstaller34_32bits:py371
script:
- docker run -v "$(pwd):/src/lbry" cdrx/pyinstaller-windows:python3-32bit lbry/scripts/wine_build.sh
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/win/
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
python: "3.6"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- pip3 install git+https://github.com/lbryio/lbryschema.git
- python scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/cli.py
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
env: OS=linux
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
@@ -92,14 +101,27 @@ jobs:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/${OS}/lbrynet
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode9.4
osx_image: xcode8.3
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- brew upgrade python || true
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
cache:
directories:
+64 -13
View File
@@ -7,17 +7,64 @@ regard to the json-rpc api. As we're currently pre-1.0 release, we
can and probably will change functionality and break backwards compatibility
at anytime.
## [Unreleased]
Python 3 upgrade of the entire code base and switching to a brand new wallet
implementation are the major changes in this release.
## [0.30.1] - 2018-12-09
### API
* add `aioupnp_version` to `upnp` field in the response to `status`
* fixed uncaught claim signature errors
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
### Schema
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
### Wallet
* fixed resolving names with invalid channel signatures
* fixed wallet sync issues
* fixed wallet locking/encryption issues
### DHT
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
### Database
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
### Misc
* switch to Python 3.7
* brought back icon for windows binary
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
## [0.30.0] - 2018-10-22
This release is the most ambitious upgrade to lbrynet yet; including a brand
new wallet implementation, switch to Python 3 of the entire code base and
many changes to the API such as pagination and a more consistent output from
most commands.
### Security
* upgraded `cryptography` package.
* switch to `coincurve` for private / public key generation (addresses)
* support for connecting to daemon via SSL
* improved workflow for encrypt/decrypt of wallet
### API
* unified all command line executables into a single `lbrynet` executable.
* added `use_https` configuration setting (off by default), if true the api server will generate a self signed ssl certificate when started and only handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* added `use_https` configuration setting (off by default), if true the api
server will generate a self signed ssl certificate when started and only
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* deprecated `daemon_stop` command, use `stop` instead.
* deprecated `wallet_balance` command, use `account_balance` instead.
* deprecated `wallet_unlock` command, use `account_unlock` instead.
@@ -27,7 +74,7 @@ implementation are the major changes in this release.
* deprecated `wallet_list` command, use `address_list` instead.
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
* deprecated `wallet_public_key` command.
* deprecated `wallet_new_address` command, use `address_generate` instead.
* deprecated `wallet_new_address` command.
* deprecated `wallet_unused_address` command, use `address_unused` instead.
* deprecated `claim_renew` command.
* added `account_list` command to list accounts including their balance.
@@ -44,12 +91,14 @@ implementation are the major changes in this release.
* added `account_max_address_gap` command to find large gaps of unused addresses.
* added `address_list` command to list addresses.
* added `address_is_mine` command to check if an address is one of your addresses.
* added `address_public_key` command to get public key of an address.
* added `address_generate` command to generate a new address.
* added `address_unused` command to get existing or generate a new unused address.
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
`transaction_list` and `utxo_list`.
* added `upnp` field to `status` response
* removed `send_amount_to_address` command previously marked as deprecated
* removed `channel_list_mine` command previously marked as deprecated
* removed `get_availability` command previously marked as deprecated
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
### Wallet
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
@@ -68,11 +117,13 @@ implementation are the major changes in this release.
* Extensive internal changes as a result of porting to Python 3.
### Reflector
*
* No major changes to reflector.
### Database
*
* No changes to existing storage module.
### Known Issues
* Channel certificate import/export is currently not working.
## [0.21.2] - 2018-08-23
### Fixed
@@ -435,7 +486,7 @@ implementation are the major changes in this release.
* Fixed https://github.com/lbryio/lbry/issues/923
* Fixed concurrent reflects opening too many files
* Fixed cases when reflecting would fail on error conditions
* Fixed deadlocks from occuring during blob writes
* Fixed deadlocks from occurring during blob writes
* Fixed and updated`lbrynet.tests.dht`
* Fixed redundant dht id
* Fixed dht `ping` method
@@ -822,7 +873,7 @@ implementation are the major changes in this release.
* Removed check_pending logic from Daemon
* Switched to txrequests so requests can use twisted event loop
* Renamed API command file_seed to file_set_status
* Dont add expected payment to wallet when payment rate is 0
* Don't add expected payment to wallet when payment rate is 0
### Fixed
* Fix restart procedure in DaemonControl
* Create download directory if it doesn't exist
@@ -853,7 +904,7 @@ implementation are the major changes in this release.
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
* Remove unused API commands from daemon
* Fix file filter `outpoint`
* Made dictionary key names in API commmand outputs to be more consistent
* Made dictionary key names in API command outputs to be more consistent
### Added
* Add file filters: `claim_id`, `outpoint`, and `rowid`
* Make loggly logs less verbose
@@ -905,7 +956,7 @@ implementation are the major changes in this release.
* Include download url in version check
### Fixed
* add misssing traceback to logging
* add missing traceback to logging
## [0.8.3] - 2017-02-15
### Fixed
+36 -66
View File
@@ -1,70 +1,48 @@
# Installing LBRY
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
[![Setup for development](https://spee.ch/2018-10-04-17-13-54-017046806.png)](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
## Prerequisites
Before starting, you'll need to install some additional libraries depending on your operating system.
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
### Python
After installing python 3, you'll need to install some additional libraries depending on your operating system.
LBRY requires Python 2.7. Python 3 is not supported.
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
### OS X Prerequisites
### macOS
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
Remaining dependencies can then be installed by running:
```
brew install mpfr libmpc protobuf
sudo easy_install pip
sudo pip install virtualenv
brew install python3 protobuf
```
### Linux Prerequisites
### Linux
On Ubuntu (we recommend 16.04), install the following:
On Ubuntu (we recommend 18.04), install the following:
```
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
If you're running another Linux flavor, install the equivalent of the above packages for your system.
### Windows Prerequisites
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
Add C:\MinGW\bin to the windows PATH.
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
```
[build]
compiler = mingw32
```
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
Currently, available binaries include:
- Cython
- Twisted
- Zope.interface
- pywin32
- requests
Install the above binaries with `pip install *.whl`
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
If you're running another Linux distro, install the equivalent of the above packages for your system.
## Installation
@@ -73,35 +51,27 @@ To install:
```
git clone https://github.com/lbryio/lbry.git
cd lbry
virtualenv lbry-venv
virtualenv lbry-venv --python=python3.7
source lbry-venv/bin/activate
python --version # Python 3 is not supported. Make sure you're on Python 2.7
pip install -U -r requirements.txt
pip install -U -r requirements_testing.txt
pip install --editable .
python --version # Python 2 is not supported. Make sure you're on Python 3.7
pip install -e .
```
To start LBRY, run `lbrynet-daemon` in a terminal.
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Run the tests
To run the unit tests from the repo directory:
```
trial --reactor=asyncio tests.unit
```
## Making a Change
## Usage
If you are not already inside the virtual environment, run:
To start the API server:
`lbrynet start`
```
source lbry-venv/bin/activate
```
Then run:
```
pip install --editable .
```
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
Happy hacking!
+23 -12
View File
@@ -1,28 +1,35 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
This project aims to provide a daemon that can interact with the network via a json-rpc interface over HTTP.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
The project is written in Python 3.7+ and extensively uses Twisted framework.
* Built on Python 3.7+ and `asyncio`.
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
Run `lbrynet start` to launch the daemon.
Run `lbrynet start` to launch the API server.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
The full API is documented [here](https://lbryio.github.io/lbry/cli).
With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Running from source
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
## Contributing
@@ -34,12 +41,16 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. Our PGP key is here if you need it.
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io)
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
## Additional information and links
The documentation for the api can be found [here](https://lbry.io/api).
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
+3 -3
View File
@@ -100,7 +100,7 @@
<div class="md-flex__cell md-flex__cell--shrink">
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
</a>
</div>
@@ -196,7 +196,7 @@
<label class="md-nav__title md-nav__title--site" for="drawer">
<span class="md-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
</span>
LBRY
@@ -304,4 +304,4 @@
</body>
</html>
</html>
+148 -62
View File
@@ -43,19 +43,13 @@
{
"name": "account_id",
"type": "str",
"description": "If provided only the balance for this account will be given",
"description": "If provided only the balance for this account will be given. Otherwise default account.",
"is_required": false
},
{
"name": "address",
"type": "str",
"description": "If provided only the balance for this address will be given",
"is_required": false
},
{
"name": "include_unconfirmed",
"type": "bool",
"description": "Include unconfirmed",
"name": "confirmations",
"type": "int",
"description": "Only include transactions with this many confirmed blocks.",
"is_required": false
}
],
@@ -83,18 +77,25 @@
{
"name": "account_decrypt",
"description": "Decrypt an encrypted account, this will remove the wallet password",
"arguments": [],
"arguments": [
{
"name": "account_id",
"type": "str",
"description": "id for the account to decrypt",
"is_required": false
}
],
"returns": "(bool) true if wallet is decrypted, otherwise false"
},
{
"name": "account_encrypt",
"description": "Encrypt a wallet with a password, if the wallet is already encrypted this will update\nthe password",
"description": "Encrypt an unencrypted account with a password",
"arguments": [
{
"name": "new_password",
"name": "account_id",
"type": "str",
"description": "password string to be used for encrypting wallet",
"is_required": true
"description": "id for the account to encrypt",
"is_required": false
}
],
"returns": "(bool) true if wallet is decrypted, otherwise false"
@@ -107,13 +108,13 @@
"name": "to_account",
"type": "str",
"description": "send to this account",
"is_required": true
"is_required": false
},
{
"name": "from_account",
"type": "str",
"description": "spend from this account",
"is_required": true
"is_required": false
},
{
"name": "amount",
@@ -155,13 +156,7 @@
{
"name": "confirmations",
"type": "int",
"description": "required confirmations (default: 6)",
"is_required": false
},
{
"name": "include_reserved",
"type": "bool",
"description": "include reserved UTXOs (default: false)",
"description": "required confirmations (default: 0)",
"is_required": false
},
{
@@ -179,6 +174,19 @@
],
"returns": "(map) balance of account(s)"
},
{
"name": "account_lock",
"description": "Lock an unlocked account",
"arguments": [
{
"name": "account_id",
"type": "str",
"description": "id for the account to lock",
"is_required": false
}
],
"returns": "(bool) true if account is locked, otherwise false"
},
{
"name": "account_max_address_gap",
"description": "Finds ranges of consecutive addresses that are unused and returns the length\nof the longest such range: for change and receiving address chains. This is\nuseful to figure out ideal values to set for 'receiving_gap' and 'change_gap'\naccount settings.",
@@ -205,6 +213,25 @@
],
"returns": "(map) details of removed account"
},
{
"name": "account_send",
"description": "Send the same number of credits to multiple addresses.",
"arguments": [
{
"name": "account_id",
"type": "str",
"description": "account to fund the transaction",
"is_required": false
},
{
"name": "broadcast",
"type": "bool",
"description": "actually broadcast the transaction, default: false.",
"is_required": false
}
],
"returns": ""
},
{
"name": "account_set",
"description": "Change various settings on an account.",
@@ -259,10 +286,10 @@
"description": "Unlock an encrypted account",
"arguments": [
{
"name": "password",
"name": "account_id",
"type": "str",
"description": "password for unlocking wallet",
"is_required": true
"description": "id for the account to unlock",
"is_required": false
}
],
"returns": "(bool) true if account is unlocked, otherwise false"
@@ -295,23 +322,22 @@
"type": "str",
"description": "id of the account to use",
"is_required": false
},
{
"name": "page",
"type": "int",
"description": "page to return during paginating",
"is_required": false
},
{
"name": "page_size",
"type": "int",
"description": "number of items on page during pagination",
"is_required": false
}
],
"returns": "List of wallet addresses"
},
{
"name": "address_public_key",
"description": "Get public key from wallet address",
"arguments": [
{
"name": "address",
"type": "str",
"description": "address for which to get the public key",
"is_required": true
}
],
"returns": "(list) list of public keys associated with address.\n Could contain more than one public key if multisig."
},
{
"name": "address_unused",
"description": "Return an address containing no balance, will create\na new address if there is none.",
@@ -535,7 +561,26 @@
{
"name": "channel_list",
"description": "Get certificate claim infos for channels that can be published to",
"arguments": [],
"arguments": [
{
"name": "account_id",
"type": "str",
"description": "id of the account to use",
"is_required": false
},
{
"name": "page",
"type": "int",
"description": "page to return during paginating",
"is_required": false
},
{
"name": "page_size",
"type": "int",
"description": "number of items on page during pagination",
"is_required": false
}
],
"returns": "(list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet."
},
{
@@ -553,6 +598,12 @@
"type": "decimal",
"description": "bid amount on the channel",
"is_required": true
},
{
"name": "account_id",
"type": "str",
"description": "id of the account to store channel",
"is_required": false
}
],
"returns": "(dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }"
@@ -584,6 +635,12 @@
"type": "str",
"description": "id of the account to use",
"is_required": false
},
{
"name": "blocking",
"type": "bool",
"description": "wait until abandon is in mempool",
"is_required": false
}
],
"returns": "(dict) Dictionary containing result of the claim\n {\n success: (bool) True if txn is successful\n txid : (str) txid of resulting transaction\n }"
@@ -641,6 +698,18 @@
"type": "str",
"description": "id of the account to query",
"is_required": false
},
{
"name": "page",
"type": "int",
"description": "page to return during paginating",
"is_required": false
},
{
"name": "page_size",
"type": "int",
"description": "number of items on page during pagination",
"is_required": false
}
],
"returns": "(list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) \"claim\", \"update\" , or \"support\"\n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]"
@@ -676,25 +745,6 @@
],
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
},
{
"name": "claim_renew",
"description": "Renew claim(s) or support(s)",
"arguments": [
{
"name": "outpoint",
"type": "str",
"description": "outpoint of the claim to renew",
"is_required": true
},
{
"name": "height",
"type": "str",
"description": "update claims expiring before or at this block height",
"is_required": true
}
],
"returns": "(dict) Dictionary where key is the the original claim's outpoint and\n value is the result of the renewal\n {\n outpoint:{\n\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n },\n }"
},
{
"name": "claim_send_to_address",
"description": "Send a name claim to an address",
@@ -1190,10 +1240,22 @@
"description": "claim id of the publisher channel, does not check for channel claim being in the wallet. This allows publishing to a channel where only the certificate private key is in the wallet.",
"is_required": false
},
{
"name": "channel_account_id",
"type": "str",
"description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.",
"is_required": false
},
{
"name": "account_id",
"type": "str",
"description": "account to use for funding the transaction",
"is_required": false
},
{
"name": "claim_address",
"type": "str",
"description": "address where the claim is sent to, if not specified new address wil automatically be created",
"description": "address where the claim is sent to, if not specified new address will automatically be created",
"is_required": false
}
],
@@ -1363,7 +1425,7 @@
"name": "status",
"description": "Get daemon status",
"arguments": [],
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }"
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
},
{
"name": "stop",
@@ -1424,6 +1486,18 @@
"type": "str",
"description": "id of the account to query",
"is_required": false
},
{
"name": "page",
"type": "int",
"description": "page to return during paginating",
"is_required": false
},
{
"name": "page_size",
"type": "int",
"description": "number of items on page during pagination",
"is_required": false
}
],
"returns": "(list) List of transactions\n\n {\n \"claim_info\": (list) claim info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) bid amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"abandon_info\": (list) abandon info if in txn [{\n \"address\": (str) address of abandoned claim,\n \"balance_delta\": (float) returned amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"confirmations\": (int) number of confirmations for the txn,\n \"date\": (str) date and time of txn,\n \"fee\": (float) txn fee,\n \"support_info\": (list) support info if in txn [{\n \"address\": (str) address of support,\n \"balance_delta\": (float) support amount,\n \"amount\": (float) support amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"is_tip\": (bool),\n \"nout\": (int) nout\n }],\n \"timestamp\": (int) timestamp,\n \"txid\": (str) txn id,\n \"update_info\": (list) update info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) credited/debited\n \"amount\": (float) absolute amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"value\": (float) value of txn\n }"
@@ -1450,6 +1524,18 @@
"type": "str",
"description": "id of the account to query",
"is_required": false
},
{
"name": "page",
"type": "int",
"description": "page to return during paginating",
"is_required": false
},
{
"name": "page_size",
"type": "int",
"description": "number of items on page during pagination",
"is_required": false
}
],
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
+1 -1
View File
@@ -1903,7 +1903,7 @@ Options:
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address wil automatically be created
new address will automatically be created
Returns:
(dict) Dictionary containing result of the claim
+1 -1
View File
@@ -1743,7 +1743,7 @@ Args:
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address wil automatically be created
new address will automatically be created
Returns:
(dict) Dictionary containing result of the claim
+4 -4
View File
@@ -2,7 +2,7 @@
"docs": [
{
"location": "/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"title": "API"
},
{
@@ -167,7 +167,7 @@
},
{
"location": "/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
@@ -292,7 +292,7 @@
},
{
"location": "/cli/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"title": "CLI"
},
{
@@ -457,7 +457,7 @@
},
{
"location": "/cli/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
+23
View File
@@ -0,0 +1,23 @@
# This is an example daemon_settings.yml file.
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
delete_blobs_on_remove: True
dht_node_port: 4444
peer_port: 3333
use_upnp: True
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - dht
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 6.1 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 97 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 361 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 5.3 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

+2 -4
View File
@@ -1,6 +1,4 @@
import logging
__version__ = "0.30.0rc2"
__name__ = "lbrynet"
__version__ = "0.30.5"
version = tuple(__version__.split('.'))
logging.getLogger(__name__).addHandler(logging.NullHandler())
-293
View File
@@ -1,293 +0,0 @@
import collections
import logging
import treq
from twisted.internet import defer, task
from lbrynet import conf
from lbrynet.core import looping_call_manager, utils, system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
SERVER_STARTUP_ERROR = 'Server Startup Error'
DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
log = logging.getLogger(__name__)
class Manager:
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
self.analytics_api = analytics_api
self._tracked_data = collections.defaultdict(list)
self.looping_call_manager = self._setup_looping_calls()
self.context = context or self._make_context(
system_info.get_platform(), conf.settings['wallet'])
self.installation_id = installation_id or conf.settings.installation_id
self.session_id = session_id or conf.settings.get_session_id()
self.is_started = False
@classmethod
def new_instance(cls, enabled=None):
api = Api.new_instance(enabled)
return cls(api)
# Things We Track
def send_new_download_start(self, download_id, name, claim_dict):
self._send_new_download_stats("start", download_id, name, claim_dict)
def send_new_download_success(self, download_id, name, claim_dict):
self._send_new_download_stats("success", download_id, name, claim_dict)
def send_new_download_fail(self, download_id, name, claim_dict, e):
self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
self.analytics_api.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
def send_server_startup(self):
self.analytics_api.track(self._event(SERVER_STARTUP))
def send_server_startup_success(self):
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
def send_server_startup_error(self, message):
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
def send_download_started(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
def send_claim_action(self, action):
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
def send_new_channel(self):
self.analytics_api.track(self._event(NEW_CHANNEL))
def send_credits_sent(self):
self.analytics_api.track(self._event(CREDITS_SENT))
def _send_heartbeat(self):
self.analytics_api.track(self._event(HEARTBEAT))
def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
# Setup / Shutdown
def start(self):
if not self.is_started:
for name, _, interval in self._get_looping_calls():
self.looping_call_manager.start(name, interval)
self.is_started = True
def shutdown(self):
self.looping_call_manager.shutdown()
def register_repeating_metric(self, event_name, value_generator, frequency=300):
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
self.looping_call_manager.register_looping_call(event_name, lcall)
lcall.start(frequency)
def _get_looping_calls(self):
return [
('send_heartbeat', self._send_heartbeat, 60),
('update_tracked_metrics', self._update_tracked_metrics, 300),
]
def _setup_looping_calls(self):
call_manager = looping_call_manager.LoopingCallManager()
for name, fn, _ in self._get_looping_calls():
call_manager.register_looping_call(name, task.LoopingCall(fn))
return call_manager
def _send_repeating_metric(self, event_name, value_generator):
result = value_generator()
self._if_deferred(result, self._send_repeating_metric_value, event_name)
def _send_repeating_metric_value(self, result, event_name):
should_send, value = result
if should_send:
self.analytics_api.track(self._metric_event(event_name, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
@staticmethod
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
if isinstance(maybe_deferred, defer.Deferred):
maybe_deferred.addCallback(callback, *args, **kwargs)
else:
callback(maybe_deferred, *args, **kwargs)
class Api:
def __init__(self, cookies, url, write_key, enabled):
self.cookies = cookies
self.url = url
self._write_key = write_key
self._enabled = enabled
def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
def update_cookies(response):
self.cookies.update(response.cookies())
return response
assert endpoint[0] == '/'
headers = {b"Connection": b"close"}
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
headers=headers, cookies=self.cookies)
d.addCallback(update_cookies)
return d
def track(self, event):
"""Send a single tracking event"""
if not self._enabled:
return defer.succeed('Analytics disabled')
def _log_error(failure, event):
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
log.debug('Sending track event: %s', event)
d = self._post('/track', event)
d.addErrback(_log_error, event)
return d
@classmethod
def new_instance(cls, enabled=None):
"""Initialize an instance using values from the configuration"""
if enabled is None:
enabled = conf.settings['share_usage_data']
return cls(
{},
conf.settings['ANALYTICS_ENDPOINT'],
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
enabled,
)
-1
View File
@@ -1 +0,0 @@
from . import paths
-26
View File
@@ -1,26 +0,0 @@
# App root files dir
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
def android_files_dir():
return None
# Base internal storage path
def android_internal_storage_dir():
return None
# Base external (SD card, if present) storage path
def android_external_storage_dir():
return None
# Internal device storage (private app folder)
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_internal_storage_dir():
return None
# External (app folder on SD card, if present) storage
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_external_storage_dir():
return None
@@ -7,7 +7,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.p2p.BlobInfo import BlobInfo
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
log = logging.getLogger(__name__)
@@ -53,10 +53,10 @@ class StreamBlobDecryptor:
def decrypt(self, write_func):
"""
Decrypt blob and write its content useing write_func
Decrypt blob and write its content using write_func
write_func - function that takes decrypted string as
arugment and writes it somewhere
argument and writes it somewhere
Returns:
@@ -75,7 +75,7 @@ class StreamBlobDecryptor:
def finish_decrypt():
bytes_left = len(self.buff) % (AES.block_size // 8)
if bytes_left != 0:
log.warning(self.buff[-1 * (AES.block_size // 8):].encode('hex'))
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
raise Exception("blob %s has incorrect padding: %i bytes left" %
(self.blob.blob_hash, bytes_left))
data_to_decrypt, self.buff = self.buff, b''
@@ -6,7 +6,7 @@ import logging
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from twisted.internet import defer
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
log = logging.getLogger(__name__)
@@ -2,16 +2,17 @@
Utilities for turning plain files into LBRY Files.
"""
import logging
import os
import logging
from binascii import hexlify
from twisted.internet import defer
from twisted.protocols.basic import FileSender
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
from lbrynet.core.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
from lbrynet.extras.compat import f2d
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
log = logging.getLogger(__name__)
@@ -119,13 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
log.debug("saving the stream")
yield storage.store_stream(
yield f2d(storage.store_stream(
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
sd_info['suggested_file_name'], sd_info['blobs']
)
))
log.debug("adding to the file manager")
lbry_file = yield lbry_file_manager.add_published_file(
lbry_file = yield f2d(lbry_file_manager.add_published_file(
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
payment_rate_manager.min_blob_data_payment_rate
)
))
defer.returnValue(lbry_file)
@@ -5,14 +5,14 @@ import logging
from binascii import hexlify, unhexlify
from twisted.internet import defer
from lbrynet import conf
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.core.HTTPBlobDownloader import HTTPBlobDownloader
from lbrynet.core.utils import short_hash
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileDownloader
from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.core.StreamDescriptor import save_sd_info
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from lbrynet.utils import short_hash
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.p2p.StreamDescriptor import save_sd_info
log = logging.getLogger(__name__)
@@ -34,12 +34,12 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
suggested_file_name, download_mirrors=None):
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
sd_hash, key, suggested_file_name, download_mirrors=None):
super().__init__(
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
wallet, download_directory, key, stream_name, file_name
)
self.sd_hash = sd_hash
self.rowid = rowid
@@ -55,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
self.channel_name = None
self.metadata = None
self.mirror = None
if download_mirrors or conf.settings['download_mirrors']:
if download_mirrors or conf.download_mirrors:
self.mirror = HTTPBlobDownloader(
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
self.blob_manager, servers=download_mirrors or conf.download_mirrors
)
def set_claim_info(self, claim_info):
@@ -70,13 +70,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
self.channel_name = claim_info['channel_name']
self.metadata = claim_info['value']['stream']['metadata']
@defer.inlineCallbacks
def get_claim_info(self, include_supports=True):
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
async def get_claim_info(self, include_supports=True):
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
if claim_info:
self.set_claim_info(claim_info)
defer.returnValue(claim_info)
return claim_info
@property
def saving_status(self):
@@ -93,7 +91,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
self.completed = True
else:
raise Exception("Unknown status for stream %s: %s" % (self.stream_hash, status))
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
@defer.inlineCallbacks
def stop(self, err=None, change_status=True):
@@ -101,16 +99,15 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
if self.mirror:
self.mirror.stop()
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
yield EncryptedFileDownloader.stop(self, err=err)
yield super().stop(err)
if change_status is True:
status = yield self._save_status()
defer.returnValue(status)
@defer.inlineCallbacks
def status(self):
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
async def status(self):
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
num_blobs_completed = len(completed_blobs)
num_blobs_known = len(blob_hashes)
@@ -120,9 +117,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
status = "stopped"
else:
status = "running"
defer.returnValue(EncryptedFileStatusReport(
return EncryptedFileStatusReport(
self.file_name, num_blobs_completed, num_blobs_known, status
))
)
@defer.inlineCallbacks
def _start(self):
@@ -150,7 +148,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
self._saving_status = False
defer.returnValue(status)
return status
def save_status(self):
return self._save_status()
@@ -180,10 +178,10 @@ class ManagedEncryptedFileDownloaderFactory:
if file_name:
file_name = hexlify(file_name.encode())
hex_download_directory = hexlify(download_directory.encode())
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
data_rate, file_name=file_name, download_mirrors=download_mirrors
)
))
defer.returnValue(lbry_file)
@staticmethod
@@ -3,19 +3,20 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
"""
import os
import logging
import random
from binascii import hexlify, unhexlify
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
from lbrynet.reflector.reupload import reflect_file
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType, get_sd_info
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError
from lbrynet.cryptstream.client.CryptStreamDownloader import CurrentlyStoppingError
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.extras.reflector.reupload import reflect_file
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
log = logging.getLogger(__name__)
@@ -28,9 +29,11 @@ class EncryptedFileManager:
# when reflecting files, reflect up to this many files at a time
CONCURRENT_REFLECTS = 5
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier):
self.conf = conf
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
@@ -42,11 +45,9 @@ class EncryptedFileManager:
self.lbry_files = []
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
@defer.inlineCallbacks
def setup(self):
yield self._add_to_sd_identifier()
yield self._start_lbry_files()
log.info("Started file manager")
self._add_to_sd_identifier()
return self._start_lbry_files()
def get_lbry_file_status(self, lbry_file):
return self.storage.get_lbry_file_status(lbry_file.rowid)
@@ -56,7 +57,7 @@ class EncryptedFileManager:
def change_lbry_file_status(self, lbry_file, status):
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
return self.storage.change_file_status(lbry_file.rowid, status)
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
def get_lbry_file_status_reports(self):
ds = []
@@ -80,6 +81,7 @@ class EncryptedFileManager:
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
return ManagedEncryptedFileDownloader(
self.conf,
rowid,
stream_hash,
self.peer_finder,
@@ -116,10 +118,9 @@ class EncryptedFileManager:
except Exception:
log.warning("Failed to start %i", file_info.get('rowid'))
@defer.inlineCallbacks
def _start_lbry_files(self):
files = yield self.storage.get_all_lbry_files()
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
async def _start_lbry_files(self):
files = await self.storage.get_all_lbry_files()
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
prm = self.payment_rate_manager
log.info("Starting %i files", len(files))
@@ -152,14 +153,13 @@ class EncryptedFileManager:
log.info("Stopping %i lbry files", len(self.lbry_files))
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
@defer.inlineCallbacks
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
key = stream_metadata['key']
stream_name = stream_metadata['stream_name']
file_name = stream_metadata['suggested_file_name']
rowid = yield self.storage.save_published_file(
rowid = await self.storage.save_published_file(
stream_hash, file_name, download_directory, blob_data_rate, status
)
lbry_file = self._get_lbry_file(
@@ -167,37 +167,36 @@ class EncryptedFileManager:
stream_metadata['suggested_file_name'], download_mirrors=None
)
lbry_file.restore(status)
yield lbry_file.get_claim_info()
await lbry_file.get_claim_info()
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
self.lbry_files.append(lbry_file)
defer.returnValue(lbry_file)
return lbry_file
@defer.inlineCallbacks
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
key = stream_metadata['key']
stream_name = stream_metadata['stream_name']
file_name = file_name or stream_metadata['suggested_file_name']
# when we save the file we'll atomic touch the nearest file to the suggested file name
# that doesn't yet exist in the download directory
rowid = yield self.storage.save_downloaded_file(
rowid = await self.storage.save_downloaded_file(
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
)
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
lbry_file = self._get_lbry_file(
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
stream_metadata['suggested_file_name'], download_mirrors
)
lbry_file.restore(status)
yield lbry_file.get_claim_info(include_supports=False)
await lbry_file.get_claim_info(include_supports=False)
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
self.lbry_files.append(lbry_file)
defer.returnValue(lbry_file)
return lbry_file
@defer.inlineCallbacks
def delete_lbry_file(self, lbry_file, delete_file=False):
@@ -223,7 +222,7 @@ class EncryptedFileManager:
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
yield lbry_file.delete_data()
yield self.storage.delete_stream(lbry_file.stream_hash)
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
if delete_file and os.path.isfile(full_path):
os.remove(full_path)
@@ -241,10 +240,10 @@ class EncryptedFileManager:
def reflect_lbry_files(self):
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
ds = []
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
for lbry_file in self.lbry_files:
if lbry_file.sd_hash in sd_hashes_to_reflect:
ds.append(sem.run(reflect_file, lbry_file))
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
yield defer.DeferredList(ds)
@defer.inlineCallbacks
-4
View File
@@ -1,4 +0,0 @@
from .blob_file import BlobFile
from .creator import BlobFileCreator
from .writer import HashBlobWriter
from .reader import HashBlobReader
+29 -21
View File
@@ -1,10 +1,10 @@
import logging
import os
from twisted.internet import defer, threads
import logging
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.core.utils import is_valid_blobhash
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
@@ -12,6 +12,24 @@ log = logging.getLogger(__name__)
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
class BlobFile:
"""
@@ -52,7 +70,7 @@ class BlobFile:
def open_for_writing(self, peer):
"""
open a blob file to be written by peer, supports concurrent
writers, as long as they are from differnt peers.
writers, as long as they are from different peers.
returns tuple of (writer, finished_deferred)
@@ -93,29 +111,19 @@ class BlobFile:
if not self.writers and not self.readers:
self._verified = False
self.saved_verified_blob = False
def delete_from_file_system():
try:
if os.path.isfile(self.file_path):
os.remove(self.file_path)
d = threads.deferToThread(delete_from_file_system)
def log_error(err):
log.warning("An error occurred deleting %s: %s",
str(self.file_path), err.getErrorMessage())
return err
d.addErrback(log_error)
return d
except Exception as e:
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
else:
return defer.fail(Failure(
ValueError("File is currently being read or written and cannot be deleted")))
raise ValueError("File is currently being read or written and cannot be deleted")
@property
def verified(self):
"""
Protect verified from being modified by other classes.
verified is True if a write to a blob has completed succesfully,
verified is True if a write to a blob has completed successfully,
or a blob has been read to have the same length as specified
in init
"""
@@ -182,7 +190,7 @@ class BlobFile:
if writer.len_so_far != self.length:
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
else:
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
errback_finished_deferred(Failure(InvalidDataError(err_string)))
d = defer.succeed(None)
else:
@@ -1,6 +1,6 @@
import binascii
from twisted.internet import defer
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
class CryptBlobHandler:
@@ -1,13 +1,15 @@
from binascii import unhexlify
import logging
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
from binascii import unhexlify
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.conf import Config
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
log = logging.getLogger(__name__)
@@ -36,7 +38,7 @@ class CryptStreamDownloader:
#implements(IStreamDownloader)
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
key, stream_name):
"""Initialize a CryptStreamDownloader
@@ -54,7 +56,7 @@ class CryptStreamDownloader:
@return:
"""
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
@@ -2,14 +2,16 @@ import os
import logging
import traceback
from binascii import hexlify, unhexlify
from lbrynet.core.StreamDescriptor import save_sd_info
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.core.Error import FileOpenError
from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
from twisted.internet import defer, threads
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.StreamDescriptor import save_sd_info
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.Error import FileOpenError
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
log = logging.getLogger(__name__)
@@ -17,9 +19,9 @@ log = logging.getLogger(__name__)
class EncryptedFileDownloader(CryptStreamDownloader):
"""Classes which inherit from this class download LBRY files"""
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
storage, payment_rate_manager, wallet, key, stream_name, file_name):
super().__init__(peer_finder, rate_limiter, blob_manager,
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
payment_rate_manager, wallet, key, stream_name)
self.stream_hash = stream_hash
self.storage = storage
@@ -28,24 +30,34 @@ class EncryptedFileDownloader(CryptStreamDownloader):
@defer.inlineCallbacks
def delete_data(self):
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
blob_hashes.append(sd_hash)
yield self.blob_manager.delete_blobs(blob_hashes)
def stop(self, err=None):
d = self._close_output()
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
return d
self._close_output()
return super().stop(err=err)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
def _start(self):
def check_start_succeeded(success):
if success:
self.starting = False
self.stopped = False
self.completed = False
return True
else:
return self._start_failed()
self.download_manager = self._get_download_manager()
d = self._setup_output()
d.addCallback(lambda _: CryptStreamDownloader._start(self))
d.addCallback(lambda _: self.download_manager.start_downloading())
d.addCallbacks(check_start_succeeded)
return d
def _setup_output(self):
@@ -54,14 +66,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
def _close_output(self):
pass
def get_total_bytes(self):
d = self.storage.get_blobs_for_stream(self.stream_hash)
def calculate_size(blobs):
return sum([b.length for b in blobs])
d.addCallback(calculate_size)
return d
async def get_total_bytes(self):
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
return sum([b.length for b in blobs])
def get_total_bytes_cached(self):
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
@@ -91,7 +98,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
class EncryptedFileDownloaderFactory:
#implements(IStreamDownloaderFactory)
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
@@ -124,9 +132,9 @@ class EncryptedFileDownloaderFactory:
class EncryptedFileSaver(EncryptedFileDownloader):
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name):
super().__init__(stream_hash, peer_finder, rate_limiter,
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
blob_manager, storage, payment_rate_manager,
wallet, key, stream_name, file_name)
self.download_directory = unhexlify(download_directory).decode()
@@ -136,10 +144,6 @@ class EncryptedFileSaver(EncryptedFileDownloader):
def __str__(self):
return str(self.file_written_to)
def stop(self, err=None):
d = EncryptedFileDownloader.stop(self, err=err)
return d
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager,
@@ -162,15 +166,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
def _close_output(self):
self.file_handle, file_handle = None, self.file_handle
def close_file():
if file_handle is not None:
name = file_handle.name
file_handle.close()
if self.completed is False:
os.remove(name)
return threads.deferToThread(close_file)
if file_handle is not None:
name = file_handle.name
file_handle.close()
if self.completed is False:
os.remove(name)
def _get_write_func(self):
def write_func(data):
@@ -180,8 +180,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
self.download_directory = hexlify(download_directory.encode())
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
@@ -189,8 +189,9 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
key = stream_info.raw_info['key']
suggested_file_name = stream_info.raw_info['suggested_file_name']
return EncryptedFileSaver(
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
file_name=suggested_file_name
)
@staticmethod
@@ -1,5 +1,6 @@
import logging
from twisted.internet import defer
from lbrynet.extras.compat import f2d
log = logging.getLogger(__name__)
@@ -17,9 +18,8 @@ class EncryptedFileMetadataHandler:
@defer.inlineCallbacks
def get_initial_blobs(self):
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
defer.returnValue(formatted_infos)
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
return self._format_initial_blobs_for_download_manager(blob_infos)
def final_blob_num(self):
return self._final_blob_num
@@ -1,6 +1,6 @@
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType
from lbrynet.core.StreamDescriptor import EncryptedFileStreamDescriptorValidator
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
def add_lbry_file_to_sd_identifier(sd_identifier):
File renamed without changes.
+1 -1
View File
@@ -3,7 +3,7 @@ import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
+2 -2
View File
@@ -1,8 +1,8 @@
import logging
from io import BytesIO
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
+1 -1
View File
@@ -1,2 +1,2 @@
# dont touch this. CI server changes this during build/deployment
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
-167
View File
@@ -1,167 +0,0 @@
import sys
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
import json
import asyncio
from aiohttp.client_exceptions import ClientConnectorError
from requests.exceptions import ConnectionError
from docopt import docopt
from textwrap import dedent
from lbrynet.daemon.Daemon import Daemon
from lbrynet.daemon.DaemonControl import start as daemon_main
from lbrynet.daemon.DaemonConsole import main as daemon_console
from lbrynet.daemon.auth.client import LBRYAPIClient
from lbrynet.core.system_info import get_platform
async def execute_command(method, params, conf_path=None):
# this check if the daemon is running or not
try:
api = await LBRYAPIClient.get_client(conf_path)
await api.status()
except (ClientConnectorError, ConnectionError):
await api.session.close()
print("Could not connect to daemon. Are you sure it's running?")
return 1
# this actually executes the method
try:
resp = await api.call(method, params)
await api.session.close()
print(json.dumps(resp["result"], indent=2))
except KeyError:
if resp["error"]["code"] == -32500:
print(json.dumps(resp["error"], indent=2))
else:
print(json.dumps(resp["error"]["message"], indent=2))
def print_help():
print(dedent("""
NAME
lbrynet - LBRY command line client.
USAGE
lbrynet [--conf <config file>] <command> [<args>]
EXAMPLES
lbrynet commands # list available commands
lbrynet status # get daemon status
lbrynet --conf ~/l1.conf status # like above but using ~/l1.conf as config file
lbrynet resolve_name what # resolve a name
lbrynet help resolve_name # get help for a command
"""))
def print_help_for_command(command):
fn = Daemon.callable_methods.get(command)
if fn:
print(dedent(fn.__doc__))
else:
print("Invalid command name")
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
if x.lower() == 'false':
return False
if x.isdigit():
return int(x)
return x
def remove_brackets(key):
if key.startswith("<") and key.endswith(">"):
return str(key[1:-1])
return key
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
k = None
if arg is None:
continue
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
kwargs[k] = normalize_value(arg, k)
return kwargs
def main(argv=None):
argv = argv or sys.argv[1:]
if not argv:
print_help()
return 1
conf_path = None
if len(argv) and argv[0] == "--conf":
if len(argv) < 2:
print("No config file specified for --conf option")
print_help()
return 1
conf_path = argv[1]
argv = argv[2:]
method, args = argv[0], argv[1:]
if method in ['help', '--help', '-h']:
if len(args) == 1:
print_help_for_command(args[0])
else:
print_help()
return 0
elif method in ['version', '--version', '-v']:
print(json.dumps(get_platform(get_ip=False), sort_keys=True, indent=2, separators=(',', ': ')))
return 0
elif method == 'start':
sys.exit(daemon_main(args, conf_path))
elif method == 'console':
sys.exit(daemon_console())
elif method not in Daemon.callable_methods:
if method not in Daemon.deprecated_methods:
print('{} is not a valid command.'.format(method))
return 1
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print("{} is permanently deprecated and does not have a replacement command.".format(method))
return 0
print("{} is deprecated, using {}.".format(method, new_method))
method = new_method
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(method, params, conf_path))
return 0
if __name__ == "__main__":
sys.exit(main())
+478 -603
View File
File diff suppressed because it is too large. Load diff
-59
View File
@@ -1,59 +0,0 @@
import platform
import json
import subprocess
import os
from six.moves.urllib import request
from six.moves.urllib.error import URLError
from lbryschema import __version__ as lbryschema_version
from lbrynet import build_type, __version__ as lbrynet_version
from lbrynet.conf import ROOT_DIR
import logging.handlers
log = logging.getLogger(__name__)
def get_lbrynet_version():
if build_type.BUILD == "dev":
try:
with open(os.devnull, 'w') as devnull:
git_dir = ROOT_DIR + '/.git'
return subprocess.check_output(
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
stderr=devnull
).decode().strip().lstrip('v')
except (subprocess.CalledProcessError, OSError):
log.debug("failed to get version from git")
return lbrynet_version
def get_platform(get_ip=True):
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": get_lbrynet_version(),
"lbryschema_version": lbryschema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
try:
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
except ModuleNotFoundError:
pass
# TODO: remove this from get_platform and add a get_external_ip function using treq
if get_ip:
try:
response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
if not response['success']:
raise URLError("failed to get external ip")
p['ip'] = response['data']['ip']
except (URLError, AssertionError):
p['ip'] = "Could not determine IP"
return p
File renamed without changes.
-8
View File
@@ -1,8 +0,0 @@
"""
Classes and functions for dealing with Crypt Streams.
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
metadata is generally stored in a Stream Descriptor File, for example containing a public key
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
may or may not be present.
"""
-106
View File
@@ -1,106 +0,0 @@
import os
import sys
import inspect
import logging
TRACE = 5
####
# This code is copied from logging/__init__.py in the python source code
####
#
# _srcfile is used when walking the stack to check when we've got the first
# caller stack frame.
#
if hasattr(sys, 'frozen'): # support for py2exe
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
_srcfile = __file__[:-4] + '.py'
else:
_srcfile = __file__
_srcfile = os.path.normcase(_srcfile)
def findCaller(srcfile=None):
"""Returns the filename, line number and function name of the caller"""
srcfile = srcfile or _srcfile
f = inspect.currentframe()
# On some versions of IronPython, currentframe() returns None if
# IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
rv = "(unknown file)", 0, "(unknown function)"
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
# ignore any function calls that are in this file
if filename == srcfile:
f = f.f_back
continue
rv = (filename, f.f_lineno, co.co_name)
break
return rv
###
class Logger(logging.Logger):
"""A logger that has an extra `fail` method useful for handling twisted failures."""
def fail(self, callback=None, *args, **kwargs):
"""Returns a function to log a failure from an errback.
The returned function appends the error message and extracts
the traceback from `err`.
Example usage:
d.addErrback(log.fail(), 'This is an error message')
Although odd, making the method call is necessary to extract
out useful filename and line number information; otherwise the
reported values are from inside twisted's deferred handling
code.
Args:
callback: callable to call after making the log. The first argument
will be the `err` from the deferred
args: extra arguments to pass into `callback`
Returns: a function that takes the following arguments:
err: twisted.python.failure.Failure
msg: the message to log, using normal logging string iterpolation.
msg_args: the values to subtitute into `msg`
msg_kwargs: set `level` to change from the default ERROR severity. Other
keywoards are treated as normal log kwargs.
"""
fn, lno, func = findCaller()
def _fail(err, msg, *msg_args, **msg_kwargs):
level = msg_kwargs.pop('level', logging.ERROR)
msg += ": %s"
msg_args += (err.getErrorMessage(),)
exc_info = (err.type, err.value, err.getTracebackObject())
record = self.makeRecord(
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
self.handle(record)
if callback:
try:
return callback(err, *args, **kwargs)
except Exception:
# log.fail is almost always called within an
# errback. If callback fails and we didn't catch
# the exception we would need to attach a second
# errback to deal with that, which we will almost
# never do and then we end up with an unhandled
# error that will get swallowed by twisted
self.exception('Failed to run callback')
return _fail
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(TRACE):
self._log(TRACE, msg, args, **kwargs)
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')
-232
View File
@@ -1,232 +0,0 @@
import sys
import code
import argparse
import asyncio
import logging.handlers
from twisted.internet import defer, reactor, threads
from aiohttp import client_exceptions
from lbrynet import analytics
from lbrynet import conf
from lbrynet.core import utils
from lbrynet.core import log_support
from lbrynet.daemon.auth.client import LBRYAPIClient
from lbrynet.daemon.Daemon import Daemon
log = logging.getLogger(__name__)
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
def color(msg, c="white"):
_colors = {
"normal": (0, 37),
"underlined": (2, 37),
"red": (1, 31),
"green": (1, 32),
"yellow": (1, 33),
"blue": (1, 33),
"magenta": (1, 34),
"cyan": (1, 35),
"white": (1, 36),
"grey": (1, 37)
}
i, j = _colors[c]
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
logo = """\
╓▄█▄ç
,▄█▓▓▀▀▀▓▓▓▌▄,
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
▄█▓▓█▀` ╙▀█▓▓▓▄▄
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
▄█▓▓█▀ ╙▀▓▓
╓▄▓▓▓▀` ▄█▓▓▓▀
▓▓█▀ ,▄▓▓▓▀╙
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
╙▀█▓▓█▓▓▓▀▀
╙▀▀`
"""
else:
def color(msg, c=None):
return msg
logo = """\
'.
++++.
+++,;+++,
:+++ :+++:
+++ ,+++;
'++; .+++'
`+++ `++++
+++. `++++
;+++ ++++
+++ +++
+++: '+
,+++ +++
+++` +++:
`+' ,+++
`+ + +++
`+ +++ '++' :'+++:
`+ ++++ `+++ ++++
`+ ++++ +++. :+++'
`+, ++++ ;+++ +++++
`+++, ++++ +++ +++; +
,+++, ++++ +++: .+++
,+++: '++++++ +++`
,+++: '++ '++'
,+++: `+++
.+++; +++,
.+++; ;+++
.+++; +++
`+++++:
`++
"""
welcometext = """\
For a list of available commands:
>>>help()
To see the documentation for a given command:
>>>help("resolve")
To exit:
>>>exit()
"""
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
welcome += "\n".join(["{:<60}".format(w) for w in welcometext.splitlines()])
welcome += "\n%s" % ("*" * 60)
welcome = color(welcome, "grey")
banner = color(logo, "green") + color(welcome, "grey")
def get_methods(daemon):
locs = {}
def wrapped(name, fn):
client = LBRYAPIClient.get_client()
_fn = getattr(client, name)
_fn.__doc__ = fn.__doc__
return {name: _fn}
for method_name, method in daemon.callable_methods.items():
locs.update(wrapped(method_name, method))
return locs
def run_terminal(callable_methods, started_daemon, quiet=False):
locs = {}
locs.update(callable_methods)
def help(method_name=None):
if not method_name:
print("Available api functions: ")
for name in callable_methods:
print("\t%s" % name)
return
if method_name not in callable_methods:
print("\"%s\" is not a recognized api function")
return
print(callable_methods[method_name].__doc__)
return
locs.update({'help': help})
if started_daemon:
def exit(status=None):
if not quiet:
print("Stopping lbrynet-daemon...")
callable_methods['daemon_stop']()
return sys.exit(status)
locs.update({'exit': exit})
else:
def exit(status=None):
try:
reactor.callLater(0, reactor.stop)
except Exception as err:
print("error stopping reactor: {}".format(err))
return sys.exit(status)
locs.update({'exit': exit})
code.interact(banner if not quiet else "", local=locs)
@defer.inlineCallbacks
def start_server_and_listen(use_auth, analytics_manager, quiet):
log_support.configure_console()
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
logging.getLogger("requests").setLevel(logging.CRITICAL)
analytics_manager.send_server_startup()
yield Daemon().start_listening()
def threaded_terminal(started_daemon, quiet):
callable_methods = get_methods(Daemon)
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
d.addErrback(lambda err: err.trap(SystemExit))
d.addErrback(log.exception)
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
if not utils.check_connection():
print("Not connected to internet, unable to start")
raise Exception("Not connected to internet, unable to start")
if not quiet:
print("Starting lbrynet-console...")
try:
await LBRYAPIClient.get_client().status()
d = defer.succeed(False)
if not quiet:
print("lbrynet-daemon is already running, connecting to it...")
except client_exceptions.ClientConnectorError:
if not use_existing_daemon:
if not quiet:
print("Starting lbrynet-daemon...")
analytics_manager = analytics.Manager.new_instance()
d = start_server_and_listen(useauth, analytics_manager, quiet)
else:
raise Exception("cannot connect to an existing daemon instance, "
"and set to not start a new one")
d.addCallback(threaded_terminal, quiet)
d.addErrback(log.exception)
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
parser.add_argument(
"--use_existing_daemon",
help="Start lbrynet-daemon if it isn't already running",
action="store_true",
default=False,
dest="use_existing_daemon"
)
parser.add_argument(
"--quiet", dest="quiet", action="store_true", default=False
)
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
)
args = parser.parse_args()
loop = asyncio.get_event_loop()
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
reactor.run()
if __name__ == "__main__":
main()
-74
View File
@@ -1,74 +0,0 @@
import os
import sys
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
# This needs to happen before anything else
if 'win' in sys.platform:
import certifi
os.environ['SSL_CERT_FILE'] = certifi.where()
from lbrynet.core import log_support
import argparse
import logging.handlers
from twisted.internet import reactor
from lbrynet import conf
from lbrynet.core import utils, system_info
from lbrynet.daemon.Daemon import Daemon
log = logging.getLogger(__name__)
def test_internet_connection():
return utils.check_connection()
def start(argv=None, conf_path=None):
if conf_path is not None:
conf.conf_file = conf_path
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
)
parser.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
parser.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
parser.add_argument(
'--version', action="store_true",
help='Show daemon version and quit'
)
args = parser.parse_args(argv)
if args.useauth:
conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
if args.version:
version = system_info.get_platform(get_ip=False)
version['installation_id'] = conf.settings.installation_id
print(utils.json_dumps_pretty(version))
return
lbrynet_log = conf.settings.get_log_filename()
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
log_support.configure_loggly_handler()
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
log.info("Starting lbrynet-daemon from command line")
if test_internet_connection():
daemon = Daemon()
daemon.start_listening()
reactor.run()
else:
log.info("Not connected to internet, unable to start")
-1
View File
@@ -1 +0,0 @@
from . import Components # register Component classes
-34
View File
@@ -1,34 +0,0 @@
import logging
from zope.interface import implementer
from twisted.cred import portal, checkers, credentials, error as cred_error
from twisted.internet import defer
from twisted.web import resource
from .keyring import Keyring
log = logging.getLogger(__name__)
@implementer(portal.IRealm)
class HttpPasswordRealm:
def __init__(self, resource):
self.resource = resource
def requestAvatar(self, avatarId, mind, *interfaces):
log.debug("Processing request for %s", avatarId)
if resource.IResource in interfaces:
return (resource.IResource, self.resource, lambda: None)
raise NotImplementedError()
@implementer(checkers.ICredentialsChecker)
class PasswordChecker:
credentialInterfaces = (credentials.IUsernamePassword,)
def __init__(self, keyring: Keyring):
self.api_key = keyring.api_key
def requestAvatarId(self, creds):
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
return defer.succeed(creds.username)
log.warning('Incorrect username or password')
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
-133
View File
@@ -1,133 +0,0 @@
import json
import aiohttp
import logging
from urllib.parse import urlparse
from lbrynet import conf
from lbrynet.daemon.auth.keyring import Keyring, APIKey
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
class UnAuthAPIClient:
def __init__(self, host, port, session):
self.host = host
self.port = port
self.session = session
def __getattr__(self, method):
async def f(*args, **kwargs):
return await self.call(method, [args, kwargs])
return f
@classmethod
async def from_url(cls, url):
url_fragment = urlparse(url)
host = url_fragment.hostname
port = url_fragment.port
connector = aiohttp.TCPConnector(
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
)
session = aiohttp.ClientSession(connector=connector)
return cls(host, port, session)
async def call(self, method, params=None):
message = {'method': method, 'params': params}
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
return await resp.json()
class AuthAPIClient:
def __init__(self, key, session, cookies, url, login_url):
self.session = session
self.__api_key = key
self.__login_url = login_url
self.__id_count = 0
self.__url = url
self.__cookies = cookies
def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'):
raise AttributeError(name)
def f(*args, **kwargs):
return self.call(name, [args, kwargs])
return f
async def call(self, method, params=None):
params = params or {}
self.__id_count += 1
pre_auth_post_data = {
'version': '2',
'method': method,
'params': params,
'id': self.__id_count
}
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
auth_msg = self.__api_key.get_hmac(to_auth).decode()
pre_auth_post_data.update({'hmac': auth_msg})
post_data = json.dumps(pre_auth_post_data)
headers = {
'Host': self.__url.hostname,
'User-Agent': USER_AGENT,
'Content-type': 'application/json'
}
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
if resp is None:
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
resp.raise_for_status()
next_secret = resp.headers.get(LBRY_SECRET, False)
if next_secret:
self.__api_key.secret = next_secret
return await resp.json()
@classmethod
async def get_client(cls, key_name=None):
api_key_name = key_name or "api"
keyring = Keyring.load_from_disk()
api_key = keyring.api_key
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
url = urlparse(login_url)
headers = {
'Host': url.hostname,
'User-Agent': USER_AGENT,
'Content-type': 'application/json'
}
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
session = aiohttp.ClientSession(connector=connector)
async with session.post(login_url, headers=headers) as r:
cookies = r.cookies
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
api_key = APIKey.create(seed=uid.encode())
return cls(api_key, session, cookies, url, login_url)
class LBRYAPIClient:
@staticmethod
def get_client(conf_path=None):
conf.conf_file = conf_path
if not conf.settings:
conf.initialize_settings()
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
-50
View File
@@ -1,50 +0,0 @@
import logging
from twisted.web import server, guard, resource
from twisted.cred import portal
from lbrynet import conf
from .auth import PasswordChecker, HttpPasswordRealm
from ..auth.keyring import Keyring
log = logging.getLogger(__name__)
class HTTPJSONRPCFactory(server.Site):
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
self.use_ssl = False
class HTTPSJSONRPCFactory(server.Site):
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
self.options = keyring.private_certificate.options()
self.use_ssl = True
class AuthJSONRPCResource(resource.Resource):
def __init__(self, protocol):
resource.Resource.__init__(self)
self.putChild(b"", protocol)
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
def getChild(self, name, request):
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
request.setHeader('expires', '0')
return self if name == '' else resource.Resource.getChild(self, name, request)
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
if use_authentication:
log.info("Using authenticated API")
checker = PasswordChecker(keyring)
realm = HttpPasswordRealm(self)
portal_to_realm = portal.Portal(realm, [checker, ])
root = guard.HTTPAuthSessionWrapper(
portal_to_realm, [guard.BasicCredentialFactory('Login to lbrynet api'), ]
)
else:
log.info("Using non-authenticated API")
root = self
return factory_class(root, keyring)
-130
View File
@@ -1,130 +0,0 @@
import os
import datetime
import hmac
import hashlib
import base58
from OpenSSL.crypto import FILETYPE_PEM
from ssl import create_default_context, SSLContext
from cryptography.hazmat.backends import default_backend
from cryptography import x509
from cryptography.hazmat.primitives import hashes
from cryptography.x509.name import NameOID, NameAttribute
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from twisted.internet import ssl
import logging
from lbrynet import conf
log = logging.getLogger(__name__)
def sha(x: bytes) -> str:
h = hashlib.sha256(x).digest()
return base58.b58encode(h).decode()
def generate_key(x: bytes = None) -> str:
if not x:
return sha(os.urandom(256))
else:
return sha(x)
class APIKey:
def __init__(self, secret: str, name: str):
self.secret = secret
self.name = name
@classmethod
def create(cls, seed=None, name=None):
secret = generate_key(seed)
return APIKey(secret, name)
def _raw_key(self) -> str:
return base58.b58decode(self.secret)
def get_hmac(self, message) -> str:
decoded_key = self._raw_key()
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
return base58.b58encode(signature.digest())
def compare_hmac(self, message, token) -> bool:
decoded_token = base58.b58decode(token)
target = base58.b58decode(self.get_hmac(message))
try:
if len(decoded_token) != len(target):
return False
return hmac.compare_digest(decoded_token, target)
except:
return False
class Keyring:
encoding = serialization.Encoding.PEM
filetype = FILETYPE_PEM
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
self.api_key: APIKey = api_key
self.public_certificate: str = public_certificate
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
@classmethod
def load_from_disk(cls):
api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token')
api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem')
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
return
with open(api_key_path, 'rb') as f:
api_key = APIKey(f.read().decode(), "api")
with open(api_ssl_cert_path, 'rb') as f:
public_cert = f.read().decode()
return cls(api_key, public_cert)
@classmethod
def generate_and_save(cls):
dns = conf.settings['api_host']
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=4096,
backend=default_backend()
)
subject = issuer = x509.Name([
NameAttribute(NameOID.COUNTRY_NAME, "US"),
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
])
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
certificate = x509.CertificateBuilder(
subject_name=subject,
issuer_name=issuer,
public_key=private_key.public_key(),
serial_number=x509.random_serial_number(),
not_valid_before=datetime.datetime.utcnow(),
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
).sign(private_key, hashes.SHA256(), default_backend())
public_certificate = certificate.public_bytes(cls.encoding).decode()
private_certificate = ssl.PrivateCertificate.load(
public_certificate,
ssl.KeyPair.load(
private_key.private_bytes(
encoding=cls.encoding,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()
).decode(),
cls.filetype
),
cls.filetype
)
auth_token = APIKey.create(seed=None, name="api")
with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f:
f.write(auth_token.secret.encode())
with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f:
f.write(public_certificate.encode())
return cls(auth_token, public_certificate, private_certificate)
-598
View File
@@ -1,598 +0,0 @@
import logging
from six.moves.urllib import parse as urlparse
import json
import inspect
import signal
from functools import wraps
from twisted.web import server
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.internet.error import ConnectionDone, ConnectionLost
from txjsonrpc import jsonrpclib
from traceback import format_exc
from lbrynet import conf, analytics
from lbrynet.core.Error import InvalidAuthenticationToken
from lbrynet.core import utils
from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
from lbrynet.core.looping_call_manager import LoopingCallManager
from lbrynet.daemon.ComponentManager import ComponentManager
from .keyring import APIKey, Keyring
from .undecorated import undecorated
from .factory import AuthJSONRPCResource
from lbrynet.daemon.json_response_encoder import JSONResponseEncoder
log = logging.getLogger(__name__)
EMPTY_PARAMS = [{}]
LBRY_SECRET = "LBRY_SECRET"
class JSONRPCError:
# http://www.jsonrpc.org/specification#error_object
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
MESSAGES = {
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
CODE_METHOD_NOT_FOUND: "Method Not Found",
CODE_INVALID_PARAMS: "Invalid Params",
CODE_INTERNAL_ERROR: "Internal Error",
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
}
HTTP_CODES = {
CODE_INVALID_REQUEST: 400,
CODE_PARSE_ERROR: 400,
CODE_INVALID_PARAMS: 400,
CODE_METHOD_NOT_FOUND: 404,
CODE_INTERNAL_ERROR: 500,
CODE_APPLICATION_ERROR: 500,
CODE_AUTHENTICATION_ERROR: 401,
}
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
assert isinstance(code, int), "'code' must be an int"
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
self.code = code
if message is None:
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
self.message = message
self.data = {} if data is None else data
self.traceback = []
if traceback is not None:
trace_lines = traceback.split("\n")
for i, t in enumerate(trace_lines):
if "--- <exception caught here> ---" in t:
if len(trace_lines) > i + 1:
self.traceback = [j for j in trace_lines[i+1:] if j]
break
def to_dict(self):
return {
'code': self.code,
'message': self.message,
'data': self.traceback
}
@classmethod
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
return cls(message, code=code, traceback=traceback)
class UnknownAPIMethodError(Exception):
pass
def trap(err, *to_trap):
err.trap(*to_trap)
def jsonrpc_dumps_pretty(obj, **kwargs):
try:
id_ = kwargs.pop("id")
except KeyError:
id_ = None
if isinstance(obj, JSONRPCError):
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
else:
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
class JSONRPCServerType(type):
def __new__(mcs, name, bases, newattrs):
klass = type.__new__(mcs, name, bases, newattrs)
klass.callable_methods = {}
klass.deprecated_methods = {}
for methodname in dir(klass):
if methodname.startswith("jsonrpc_"):
method = getattr(klass, methodname)
if not hasattr(method, '_deprecated'):
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
else:
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
return klass
class AuthorizedBase(metaclass=JSONRPCServerType):
@staticmethod
def deprecated(new_command=None):
def _deprecated_wrapper(f):
f.new_command = new_command
f._deprecated = True
return f
return _deprecated_wrapper
@staticmethod
def requires(*components, **conditions):
if conditions and ["conditions"] != list(conditions.keys()):
raise SyntaxError("invalid conditions argument")
condition_names = conditions.get("conditions", [])
def _wrap(fn):
@defer.inlineCallbacks
@wraps(fn)
def _inner(*args, **kwargs):
component_manager = args[0].component_manager
for condition_name in condition_names:
condition_result, err_msg = yield component_manager.evaluate_condition(condition_name)
if not condition_result:
raise ComponentStartConditionNotMet(err_msg)
if not component_manager.all_components_running(*components):
raise ComponentsNotStarted("the following required components have not yet started: "
"%s" % json.dumps(components))
result = yield fn(*args, **kwargs)
defer.returnValue(result)
return _inner
return _wrap
class AuthJSONRPCServer(AuthorizedBase):
"""
Authorized JSONRPC server used as the base class for the LBRY API
API methods are named with a leading "jsonrpc_"
Attributes:
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
callable_methods (dict): {<api method name>: <api method>}
Authentication:
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
service url will require a username and password.
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
client uses to generate the token for their next request.
"""
#implements(resource.IResource)
isLeaf = True
allowed_during_startup = []
component_attributes = {}
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
to_skip=None, looping_calls=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
self.component_manager = component_manager or ComponentManager(
analytics_manager=self.analytics_manager,
skip_components=to_skip or [],
reactor=reactor
)
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
self._use_authentication = use_authentication or conf.settings['use_auth_http']
self._use_https = use_https or conf.settings['use_https']
self.listening_port = None
self._component_setup_deferred = None
self.announced_startup = False
self.sessions = {}
self.server = None
self.keyring = Keyring.generate_and_save()
@defer.inlineCallbacks
def start_listening(self):
from twisted.internet import reactor, error as tx_error
try:
self.server = self.get_server_factory()
if self.server.use_ssl:
log.info("Using SSL")
self.listening_port = reactor.listenSSL(
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
)
else:
log.info("Not using SSL")
self.listening_port = reactor.listenTCP(
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
)
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
yield self.setup()
self.analytics_manager.send_server_startup_success()
except tx_error.CannotListenError:
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
reactor.fireSystemEvent("shutdown")
except defer.CancelledError:
log.info("shutting down before finished starting")
reactor.fireSystemEvent("shutdown")
except Exception as err:
self.analytics_manager.send_server_startup_error(str(err))
log.exception('Failed to start lbrynet-daemon')
reactor.fireSystemEvent("shutdown")
def setup(self):
from twisted.internet import reactor
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
if not self.analytics_manager.is_started:
self.analytics_manager.start()
self.analytics_manager.send_server_startup()
for lc_name, lc_time in self._looping_call_times.items():
self.looping_call_manager.start(lc_name, lc_time)
def update_attribute(setup_result, component):
setattr(self, self.component_attributes[component.component_name], component.component)
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
self._component_setup_deferred = self.component_manager.setup(**kwargs)
return self._component_setup_deferred
@staticmethod
def _already_shutting_down(sig_num, frame):
log.info("Already shutting down")
def _shutdown(self):
# ignore INT/TERM signals once shutdown has started
signal.signal(signal.SIGINT, self._already_shutting_down)
signal.signal(signal.SIGTERM, self._already_shutting_down)
if self.listening_port:
self.listening_port.stopListening()
self.looping_call_manager.shutdown()
if self.server is not None:
for session in list(self.server.sessions.values()):
session.expire()
if self.analytics_manager:
self.analytics_manager.shutdown()
try:
self._component_setup_deferred.cancel()
except (AttributeError, defer.CancelledError):
pass
if self.component_manager is not None:
d = self.component_manager.stop()
d.addErrback(log.fail(), 'Failure while shutting down')
else:
d = defer.succeed(None)
return d
def get_server_factory(self):
return AuthJSONRPCResource(self).getServerFactory(self.keyring, self._use_authentication, self._use_https)
def _set_headers(self, request, data, update_secret=False):
if conf.settings['allowed_origin']:
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
request.setHeader("Content-Type", "application/json")
request.setHeader("Accept", "application/json-rpc")
request.setHeader("Content-Length", str(len(data)))
if update_secret:
session_id = request.getSession().uid
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
@staticmethod
def _render_message(request, message: str):
request.write(message.encode())
request.finish()
def _render_error(self, failure, request, id_):
if isinstance(failure, JSONRPCError):
error = failure
elif isinstance(failure, Failure):
# maybe failure is JSONRPCError wrapped in a twisted Failure
error = failure.check(JSONRPCError)
if error is None:
# maybe its a twisted Failure with another type of error
if hasattr(failure.type, "code"):
error_code = failure.type.code
else:
error_code = JSONRPCError.CODE_APPLICATION_ERROR
error = JSONRPCError.create_from_exception(
failure.getErrorMessage() or failure.type.__name__,
code=error_code,
traceback=failure.getTraceback()
)
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
log.warning("error processing api request: %s\ntraceback: %s", error.message,
"\n".join(error.traceback))
else:
# last resort, just cast it as a string
error = JSONRPCError(str(failure))
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
self._set_headers(request, response_content)
request.setResponseCode(200)
self._render_message(request, response_content)
@staticmethod
def _handle_dropped_request(result, d, function_name):
if not d.called:
log.warning("Cancelling dropped api request %s", function_name)
d.cancel()
def render(self, request):
try:
return self._render(request)
except BaseException as e:
log.error(e)
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
self._render_error(error, request, None)
return server.NOT_DONE_YET
def _render(self, request):
time_in = utils.now()
# if not self._check_headers(request):
# self._render_error(Failure(InvalidHeaderError()), request, None)
# return server.NOT_DONE_YET
session = request.getSession()
session_id = session.uid
finished_deferred = request.notifyFinish()
if self._use_authentication:
# if this is a new session, send a new secret and set the expiration
# otherwise, session.touch()
if self._initialize_session(session_id):
def expire_session():
self._unregister_user_session(session_id)
session.notifyOnExpire(expire_session)
message = "OK"
request.setResponseCode(200)
self._set_headers(request, message, True)
self._render_message(request, message)
return server.NOT_DONE_YET
else:
session.touch()
request.content.seek(0, 0)
content = request.content.read().decode()
try:
parsed = jsonrpclib.loads(content)
except json.JSONDecodeError:
log.warning("Unable to decode request json")
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
return server.NOT_DONE_YET
request_id = None
try:
function_name = parsed.get('method')
args = parsed.get('params', {})
request_id = parsed.get('id', None)
token = parsed.pop('hmac', None)
except AttributeError as err:
log.warning(err)
self._render_error(
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
)
return server.NOT_DONE_YET
reply_with_next_secret = False
if self._use_authentication:
try:
self._verify_token(session_id, parsed, token)
except InvalidAuthenticationToken as err:
log.warning("API validation failed")
self._render_error(
JSONRPCError.create_from_exception(
str(err),
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
traceback=format_exc()
),
request, request_id
)
return server.NOT_DONE_YET
request.addCookie("TWISTED_SESSION", session_id)
self._update_session_secret(session_id)
reply_with_next_secret = True
try:
fn = self._get_jsonrpc_method(function_name)
except UnknownAPIMethodError as err:
log.warning('Failed to get function %s: %s', function_name, err)
self._render_error(
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
request, request_id
)
return server.NOT_DONE_YET
if args in (EMPTY_PARAMS, []):
_args, _kwargs = (), {}
elif isinstance(args, dict):
_args, _kwargs = (), args
elif len(args) == 1 and isinstance(args[0], dict):
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
# TODO: also delete EMPTY_PARAMS then
_args, _kwargs = (), args[0]
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
_args, _kwargs = args
else:
raise ValueError('invalid args format')
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
if params_error is not None:
params_error_message = '{} for {} command: {}'.format(
params_error, function_name, ', '.join(erroneous_params)
)
log.warning(params_error_message)
self._render_error(
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
request, request_id
)
return server.NOT_DONE_YET
d = defer.maybeDeferred(fn, self, *_args, **_kwargs)
# finished_deferred will callback when the request is finished
# and errback if something went wrong. If the errback is
# called, cancel the deferred stack. This is to prevent
# request.finish() from being called on a closed request.
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
d.addErrback(self._render_error, request, request_id)
d.addBoth(lambda _: log.debug("%s took %f",
function_name,
(utils.now() - time_in).total_seconds()))
return server.NOT_DONE_YET
def _register_user_session(self, session_id):
"""
Add or update a HMAC secret for a session
@param session_id:
@return: secret
"""
log.info("Started new api session")
token = APIKey.create(seed=session_id)
self.sessions.update({session_id: token})
def _unregister_user_session(self, session_id):
log.info("Unregister API session")
del self.sessions[session_id]
def _check_headers(self, request):
return (
self._check_header_source(request, 'Origin') and
self._check_header_source(request, 'Referer')
)
def _check_header_source(self, request, header):
"""Check if the source of the request is allowed based on the header value."""
source = request.getHeader(header)
if not self._check_source_of_request(source):
log.warning("Attempted api call from invalid %s: %s", header, source)
return False
return True
def _check_source_of_request(self, source):
if source is None:
return True
if conf.settings['api_host'] == '0.0.0.0':
return True
server, port = self.get_server_port(source)
return self._check_server_port(server, port)
def _check_server_port(self, server, port):
api = (conf.settings['api_host'], conf.settings['api_port'])
return (server, port) == api or self._is_from_allowed_origin(server, port)
def _is_from_allowed_origin(self, server, port):
allowed_origin = conf.settings['allowed_origin']
if not allowed_origin:
return False
if allowed_origin == '*':
return True
allowed_server, allowed_port = self.get_server_port(allowed_origin)
return (allowed_server, allowed_port) == (server, port)
def get_server_port(self, origin):
parsed = urlparse.urlparse(origin)
server_port = parsed.netloc.split(':')
assert len(server_port) <= 2
if len(server_port) == 2:
return server_port[0], int(server_port[1])
else:
return server_port[0], 80
def _verify_method_is_callable(self, function_path):
if function_path not in self.callable_methods:
raise UnknownAPIMethodError(function_path)
def _get_jsonrpc_method(self, function_path):
if function_path in self.deprecated_methods:
new_command = self.deprecated_methods[function_path].new_command
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
function_path, new_command)
function_path = new_command
self._verify_method_is_callable(function_path)
return self.callable_methods.get(function_path)
@staticmethod
def _check_params(function, args_tup, args_dict):
argspec = inspect.getfullargspec(undecorated(function))
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
duplicate_params = [
duplicate_param
for duplicate_param in argspec.args[1:len(args_tup) + 1]
if duplicate_param in args_dict
]
if duplicate_params:
return 'Duplicate parameters', duplicate_params
missing_required_params = [
required_param
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
if required_param not in args_dict
]
if len(missing_required_params):
return 'Missing required parameters', missing_required_params
extraneous_params = [] if argspec.varkw is not None else [
extra_param
for extra_param in args_dict
if extra_param not in argspec.args[1:]
]
if len(extraneous_params):
return 'Extraneous parameters', extraneous_params
return None, None
def _initialize_session(self, session_id):
if not self.sessions.get(session_id):
self._register_user_session(session_id)
return True
return False
def _verify_token(self, session_id, message, token):
if token is None:
raise InvalidAuthenticationToken('Authentication token not found')
to_auth = json.dumps(message, sort_keys=True)
api_key = self.sessions.get(session_id)
if not api_key.compare_hmac(to_auth, token):
raise InvalidAuthenticationToken('Invalid authentication token')
def _update_session_secret(self, session_id):
self.sessions.update({session_id: APIKey.create(name=session_id)})
def _callback_render(self, result, request, id_, auth_required=False):
try:
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
request.setResponseCode(200)
self._set_headers(request, message, auth_required)
self._render_message(request, message)
except Exception as err:
log.exception("Failed to render API response: %s", result)
self._render_error(err, request, id_)
@staticmethod
def _render_response(result):
return defer.succeed(result)
-57
View File
@@ -1,57 +0,0 @@
from decimal import Decimal
from binascii import hexlify
from datetime import datetime
from json import JSONEncoder
from lbrynet.wallet.transaction import Transaction, Output
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
return float(obj)
if isinstance(obj, bytes):
return obj.decode()
return super().default(obj)
def encode_transaction(self, tx):
return {
'txid': tx.id,
'height': tx.height,
'inputs': [self.encode_input(txo) for txo in tx.inputs],
'outputs': [self.encode_output(txo) for txo in tx.outputs],
'total_input': tx.input_sum,
'total_output': tx.input_sum - tx.fee,
'total_fee': tx.fee,
'hex': hexlify(tx.raw).decode(),
}
def encode_output(self, txo):
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': txo.amount,
'address': txo.get_address(self.ledger),
'is_claim': txo.script.is_claim_name,
'is_support': txo.script.is_support_claim,
'is_update': txo.script.is_update_claim,
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
return output
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
File renamed without changes.
+2 -2
View File
@@ -40,7 +40,7 @@ class _Contact:
self._token = (None, 0) # token, timestamp
def update_token(self, token):
self._token = token, self.getTime()
self._token = token, self.getTime() if token else 0
@property
def token(self):
@@ -67,7 +67,7 @@ class _Contact:
@property
def lastFailed(self):
return self._contactManager._rpc_failures.get((self.address, self.port), [None])[-1]
return (self.failures or [None])[-1]
@property
def failures(self):
+2 -2
View File
@@ -1,5 +1,5 @@
from collections import UserDict
from . import constants
from lbrynet.dht import constants
class DictDataStore(UserDict):
@@ -44,7 +44,7 @@ class DictDataStore(UserDict):
del self[key]
def hasPeersForBlob(self, key):
return True if key in self and len(tuple(self.filter_bad_and_expired_peers(key))) else False
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
if key in self:
+1 -1
View File
@@ -1,4 +1,4 @@
from .error import DecodeError
from lbrynet.dht.error import DecodeError
def bencode(data):
-2
View File
@@ -13,14 +13,12 @@ class DecodeError(Exception):
Should be raised by an C{Encoding} implementation if decode operation
fails
"""
pass
class BucketFull(Exception):
"""
Raised when the bucket is full
"""
pass
class UnknownRemoteException(Exception):
+1 -2
View File
@@ -10,7 +10,6 @@ class IDataStore(Interface):
def keys(self):
""" Return a list of the keys in this data store """
pass
def removeExpiredPeers(self):
pass
@@ -59,7 +58,7 @@ class IRoutingTable(Interface):
@param count: the amount of contacts to return
@type count: int
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
Whatever ID is passed in the paramater will get
Whatever ID is passed in the parameter will get
excluded from the list of returned contacts.
@type _rpcNodeID: str
+7 -7
View File
@@ -1,8 +1,8 @@
import logging
from twisted.internet import defer
from .distance import Distance
from .error import TimeoutError
from . import constants
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants
log = logging.getLogger(__name__)
@@ -67,7 +67,7 @@ class _IterativeFind:
def getContactTriples(self, result):
if self.is_find_value_request:
contact_triples = result['contacts']
contact_triples = result[b'contacts']
else:
contact_triples = result
for contact_tup in contact_triples:
@@ -112,11 +112,11 @@ class _IterativeFind:
# We are looking for a value, and the remote node didn't have it
# - mark it as the closest "empty" node, if it is
# TODO: store to this peer after finding the value as per the kademlia spec
if 'closestNodeNoValue' in self.find_value_result:
if b'closestNodeNoValue' in self.find_value_result:
if self.is_closer(contact):
self.find_value_result['closestNodeNoValue'] = contact
self.find_value_result[b'closestNodeNoValue'] = contact
else:
self.find_value_result['closestNodeNoValue'] = contact
self.find_value_result[b'closestNodeNoValue'] = contact
contactTriples = self.getContactTriples(result)
for contactTriple in contactTriples:
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
+3 -3
View File
@@ -1,8 +1,8 @@
import logging
from . import constants
from .distance import Distance
from .error import BucketFull
from lbrynet.dht import constants
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import BucketFull
log = logging.getLogger(__name__)
+1 -1
View File
@@ -7,7 +7,7 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from . import msgtypes
from lbrynet.dht import msgtypes
class MessageTranslator:
+3 -3
View File
@@ -7,8 +7,8 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from lbrynet.core.utils import generate_id
from . import constants
from lbrynet.utils import generate_id
from lbrynet.dht import constants
class Message:
@@ -48,5 +48,5 @@ class ErrorMessage(ResponseMessage):
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
super().__init__(rpcID, nodeID, errorMessage)
if isinstance(exceptionType, type):
exceptionType = ('%s.%s' % (exceptionType.__module__, exceptionType.__name__)).encode()
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
self.exceptionType = exceptionType
+35 -60
View File
@@ -5,17 +5,12 @@ from functools import reduce
from twisted.internet import defer, error, task
from lbrynet.core.utils import generate_id, DeferredDict
from lbrynet.core.call_later_manager import CallLaterManager
from lbrynet.core.PeerManager import PeerManager
from .error import TimeoutError
from . import constants
from . import routingtable
from . import datastore
from . import protocol
from .peerfinder import DHTPeerFinder
from .contact import ContactManager
from .iterativefind import iterativeFind
from lbrynet.utils import generate_id, DeferredDict
from lbrynet.dht.call_later_manager import CallLaterManager
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants, routingtable, datastore, protocol
from lbrynet.dht.contact import ContactManager
from lbrynet.dht.iterativefind import iterativeFind
log = logging.getLogger(__name__)
@@ -83,8 +78,8 @@ class Node(MockKademliaHelper):
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
routingTableClass=None, networkProtocol=None,
externalIP=None, peerPort=3333, listenUDP=None,
callLater=None, resolve=None, clock=None, peer_finder=None,
peer_manager=None, interface='', externalUDPPort=None):
callLater=None, resolve=None, clock=None,
interface='', externalUDPPort=None):
"""
@param dataStore: The data store to use. This must be class inheriting
from the C{DataStore} interface (or providing the
@@ -124,20 +119,13 @@ class Node(MockKademliaHelper):
else:
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
# Initialize this node's network access mechanisms
if networkProtocol is None:
self._protocol = protocol.KademliaProtocol(self)
else:
self._protocol = networkProtocol
# Initialize the data storage mechanism used by this node
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
self.token_secret = self._generateID()
self.old_token_secret = None
self.externalIP = externalIP
self.peerPort = peerPort
self.externalUDPPort = externalUDPPort or self.port
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
self.peer_manager = peer_manager or PeerManager()
self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
self._join_deferred = None
#def __del__(self):
@@ -220,7 +208,7 @@ class Node(MockKademliaHelper):
if not bootstrap_contacts:
log.warning("no bootstrap contacts to ping")
ping_result = yield _ping_contacts(bootstrap_contacts)
shortlist = ping_result.keys()
shortlist = list(ping_result.keys())
if not shortlist:
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
defer.returnValue(None)
@@ -228,10 +216,10 @@ class Node(MockKademliaHelper):
# find the closest peers to us
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
yield _ping_contacts(closest)
# # query random hashes in our bucket key ranges to fill or split them
# random_ids_in_range = self._routingTable.getRefreshList()
# while random_ids_in_range:
# yield self.iterativeFindNode(random_ids_in_range.pop())
# query random hashes in our bucket key ranges to fill or split them
random_ids_in_range = self._routingTable.getRefreshList()
while random_ids_in_range:
yield self.iterativeFindNode(random_ids_in_range.pop())
defer.returnValue(None)
@defer.inlineCallbacks
@@ -255,7 +243,7 @@ class Node(MockKademliaHelper):
yield _iterative_join()
@defer.inlineCallbacks
def start(self, known_node_addresses=None):
def start(self, known_node_addresses=None, block_on_join=False):
""" Causes the Node to attempt to join the DHT network by contacting the
known DHT nodes. This can be called multiple times if the previous attempt
has failed or if the Node has lost all the contacts.
@@ -270,8 +258,11 @@ class Node(MockKademliaHelper):
self.start_listening()
yield self._protocol._listening
# TODO: Refresh all k-buckets further away than this node's closest neighbour
yield self.joinNetwork(known_node_addresses or [])
self.start_looping_calls()
d = self.joinNetwork(known_node_addresses or [])
d.addCallback(lambda _: self.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
if block_on_join:
yield d
def start_looping_calls(self):
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
@@ -299,25 +290,24 @@ class Node(MockKademliaHelper):
@defer.inlineCallbacks
def storeToContact(self, blob_hash, contact):
try:
token = contact.token
if not token:
find_value_response = yield contact.findValue(blob_hash)
token = find_value_response[b'token']
contact.update_token(token)
res = yield contact.store(blob_hash, token, self.peerPort, self.node_id, 0)
if not contact.token:
yield contact.findValue(blob_hash)
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
if res != b"OK":
raise ValueError(res)
defer.returnValue(True)
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
return True
except protocol.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s",
binascii.hexlify(blob_hash), contact.log_id())
except ValueError as err:
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
contact.update_token(None)
log.error("Unexpected error while storing blob_hash %s at %s: %s",
binascii.hexlify(blob_hash), contact, err)
defer.returnValue(False)
return False
@defer.inlineCallbacks
def announceHaveBlob(self, blob_hash):
@@ -451,20 +441,10 @@ class Node(MockKademliaHelper):
@rtype: twisted.internet.defer.Deferred
"""
try:
contact = self._routingTable.getContact(contactID)
df = defer.Deferred()
df.callback(contact)
df = defer.succeed(self._routingTable.getContact(contactID))
except (ValueError, IndexError):
def parseResults(nodes):
node_ids = [c.id for c in nodes]
if contactID in nodes:
contact = nodes[node_ids.index(contactID)]
return contact
else:
return None
df = self.iterativeFindNode(contactID)
df.addCallback(parseResults)
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
return df
@rpcmethod
@@ -510,7 +490,7 @@ class Node(MockKademliaHelper):
if 0 <= port <= 65536:
compact_port = port.to_bytes(2, 'big')
else:
raise TypeError('Invalid port: {}'.format(port))
raise TypeError(f'Invalid port: {port}')
compact_address = compact_ip + compact_port + rpc_contact.id
now = int(self.clock.seconds())
originallyPublished = now - age
@@ -591,7 +571,7 @@ class Node(MockKademliaHelper):
"""
return generate_id()
# from lbrynet.core.utils import profile_deferred
# from lbrynet.p2p.utils import profile_deferred
# @profile_deferred()
@defer.inlineCallbacks
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
@@ -632,7 +612,7 @@ class Node(MockKademliaHelper):
# # Update the "last accessed" timestamp for the appropriate k-bucket
# self._routingTable.touchKBucket(key)
if len(shortlist) == 0:
log.warning("This node doesnt know any other nodes")
log.warning("This node doesn't know any other nodes")
# This node doesn't know of any other nodes
fakeDf = defer.Deferred()
fakeDf.callback([])
@@ -651,19 +631,14 @@ class Node(MockKademliaHelper):
replication/republishing as necessary """
yield self._refreshRoutingTable()
self._dataStore.removeExpiredPeers()
yield self._refreshStoringPeers()
self._refreshStoringPeers()
defer.returnValue(None)
def _refreshContacts(self):
return defer.DeferredList(
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in self.contacts]
)
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
def _refreshStoringPeers(self):
storing_contacts = self._dataStore.getStoringContacts()
return defer.DeferredList(
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in storing_contacts]
)
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
@defer.inlineCallbacks
def _refreshRoutingTable(self):
-64
View File
@@ -1,64 +0,0 @@
import binascii
import logging
from twisted.internet import defer
from lbrynet import conf
log = logging.getLogger(__name__)
class DummyPeerFinder:
"""This class finds peers which have announced to the DHT that they have certain blobs"""
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
return defer.succeed([])
class DHTPeerFinder(DummyPeerFinder):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
#implements(IPeerFinder)
def __init__(self, dht_node, peer_manager):
"""
dht_node - an instance of dht.Node class
peer_manager - an instance of PeerManager class
"""
self.dht_node = dht_node
self.peer_manager = peer_manager
self.peers = {}
@defer.inlineCallbacks
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
"""
Find peers for blob in the DHT
blob_hash (str): blob hash to look for
timeout (int): seconds to timeout after
filter_self (bool): if True, and if a peer for a blob is itself, filter it
from the result
Returns:
list of peers for the blob
"""
if blob_hash not in self.peers:
self.peers[blob_hash] = [(self.dht_node.externalIP, self.dht_node.peerPort)]
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
timeout = timeout or conf.settings['peer_search_timeout']
if timeout:
finished_deferred.addTimeout(timeout, self.dht_node.clock)
try:
peer_list = yield finished_deferred
except defer.TimeoutError:
log.debug("DHT timed out while looking peers for blob %s after %s seconds",
blob_hash, timeout)
peer_list = []
peers = set(peer_list)
results = []
for node_id, host, port in peers:
if filter_self and (host, port) == (self.dht_node.externalIP, self.dht_node.peerPort):
continue
results.append(self.peer_manager.get_peer(host, port))
self.peers[blob_hash].append((host, port))
defer.returnValue(results)
+59 -68
View File
@@ -1,15 +1,10 @@
import logging
import errno
from binascii import hexlify
from collections import deque
from twisted.internet import protocol, defer
from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
from . import constants
from . import encoding
from . import msgtypes
from . import msgformat
from lbrynet.dht import constants, encoding, msgformat, msgtypes
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
log = logging.getLogger(__name__)
@@ -22,73 +17,53 @@ class PingQueue:
def __init__(self, node):
self._node = node
self._get_time = self._node.clock.seconds
self._queue = deque()
self._enqueued_contacts = {}
self._semaphore = defer.DeferredSemaphore(1)
self._ping_semaphore = defer.DeferredSemaphore(constants.alpha)
self._process_lc = node.get_looping_call(self._semaphore.run, self._process)
self._pending_contacts = {}
self._process_lc = node.get_looping_call(self._process)
def _add_contact(self, contact, delay=None):
if (contact.address, contact.port) in [(c.address, c.port) for c in self._enqueued_contacts]:
return defer.succeed(None)
delay = delay or constants.checkRefreshInterval
self._enqueued_contacts[contact] = self._get_time() + delay
self._queue.append(contact)
return defer.succeed(None)
def enqueue_maybe_ping(self, *contacts, **kwargs):
delay = kwargs.get('delay', constants.checkRefreshInterval)
no_op = (defer.succeed(None), lambda: None)
for contact in contacts:
if delay and contact not in self._enqueued_contacts:
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
else:
self._enqueued_contacts.setdefault(contact, no_op)
@defer.inlineCallbacks
def _process(self):
if not len(self._queue):
defer.returnValue(None)
contact = self._queue.popleft()
now = self._get_time()
# if the oldest contact in the queue isn't old enough to be pinged, add it back to the queue and return
if now < self._enqueued_contacts[contact]:
self._queue.appendleft(contact)
defer.returnValue(None)
pinged = []
checked = []
while now > self._enqueued_contacts[contact]:
checked.append(contact)
if not contact.contact_is_good:
pinged.append(contact)
if not len(self._queue):
break
contact = self._queue.popleft()
if not now > self._enqueued_contacts[contact]:
checked.append(contact)
@defer.inlineCallbacks
def _ping(contact):
try:
yield contact.ping()
except TimeoutError:
pass
except Exception as err:
log.warning("unexpected error: %s", err)
yield defer.DeferredList([_ping(contact) for contact in pinged])
for contact in checked:
if contact in self._enqueued_contacts and contact in pinged:
def _ping(self, contact):
if contact.contact_is_good:
return
try:
yield contact.ping()
except TimeoutError:
pass
except Exception as err:
log.warning("unexpected error: %s", err)
finally:
if contact in self._enqueued_contacts:
del self._enqueued_contacts[contact]
elif contact not in self._queue:
self._queue.appendleft(contact)
defer.returnValue(None)
def _process(self):
# move contacts that are scheduled to join the queue
if self._pending_contacts:
now = self._node.clock.seconds()
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
del self._pending_contacts[contact]
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
# spread pings across 60 seconds to avoid flood and/or false negatives
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
if call.called and not contact.contact_is_good:
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
def start(self):
return self._node.safe_start_looping_call(self._process_lc, 60)
def stop(self):
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
return self._node.safe_stop_looping_call(self._process_lc)
def enqueue_maybe_ping(self, contact, delay=None):
return self._semaphore.run(self._add_contact, contact, delay)
class KademliaProtocol(protocol.DatagramProtocol):
""" Implements all low-level network-related functions of a Kademlia node """
@@ -139,7 +114,20 @@ class KademliaProtocol(protocol.DatagramProtocol):
return args
return args + ({b'protocolVersion': self._protocolVersion},)
@defer.inlineCallbacks
def sendRPC(self, contact, method, args):
for _ in range(constants.rpcAttempts):
try:
response = yield self._sendRPC(contact, method, args)
return response
except TimeoutError:
if contact.contact_is_good:
log.debug("RETRY %s ON %s", method, contact)
continue
else:
raise
def _sendRPC(self, contact, method, args):
"""
Sends an RPC to the specified contact
@@ -174,16 +162,19 @@ class KademliaProtocol(protocol.DatagramProtocol):
df = defer.Deferred()
def _remove_contact(failure): # remove the contact from the routing table and track the failure
contact.update_last_failed()
try:
self._node.removeContact(contact)
if not contact.contact_is_good:
self._node.removeContact(contact)
except (ValueError, IndexError):
pass
contact.update_last_failed()
return failure
def _update_contact(result): # refresh the contact in the routing table
contact.update_last_replied()
if method == b'findValue':
if b'token' in result:
contact.update_token(result[b'token'])
if b'protocolVersion' not in result:
contact.update_protocol_version(0)
else:
@@ -240,7 +231,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
message = self._translator.fromPrimitive(msgPrimitive)
except (encoding.DecodeError, ValueError) as err:
# We received some rubbish here
log.warning("Error decoding datagram %s from %s:%i - %s", datagram.encode('hex'),
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
address[0], address[1], err)
return
except (IndexError, KeyError):
@@ -330,7 +321,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
If the data is spread over multiple UDP datagrams, the packets have the
following structure::
| | | | | |||||||||||| 0x00 |
|Transmision|Total number|Sequence number| RPC ID |Header end|
|Transmission|Total number|Sequence number| RPC ID |Header end|
| type ID | of packets |of this packet | | indicator|
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
| | | | | |||||||||||| |
@@ -357,7 +348,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
while seqNumber < totalPackets:
packetData = data[startPos:startPos + self.msgSizeLimit]
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
txData = '\x00%s%s%s\x00%s' % (encTotalPackets, encSeqNumber, rpcID, packetData)
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
self._scheduleSendNext(txData, address)
startPos += self.msgSizeLimit
@@ -376,7 +367,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
except OSError as err:
if err.errno == errno.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesnt seem to affect anything -grin
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
elif err.errno == errno.ENETUNREACH:
# this should probably try to retransmit when the network connection is back
@@ -441,7 +432,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
else:
result = func()
except Exception as e:
log.exception("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
df.errback(e)
else:
df.callback(result)
+30 -22
View File
@@ -6,14 +6,12 @@
# may be created by processing this file with epydoc: http://epydoc.sf.net
import random
from binascii import unhexlify
from twisted.internet import defer
from . import constants
from . import kbucket
from .error import TimeoutError
from .distance import Distance
import logging
from twisted.internet import defer
from lbrynet.dht import constants, kbucket
from lbrynet.dht.error import TimeoutError
from lbrynet.dht.distance import Distance
log = logging.getLogger(__name__)
@@ -48,6 +46,7 @@ class TreeRoutingTable:
from twisted.internet import reactor
getTime = reactor.seconds
self._getTime = getTime
self._ongoing_replacements = set()
def get_contacts(self):
contacts = []
@@ -134,8 +133,14 @@ class TreeRoutingTable:
to_replace = not_good_contacts[0]
else:
to_replace = self._buckets[bucketIndex]._contacts[0]
df = to_replace.ping()
df.addErrback(replaceContact, to_replace)
if to_replace not in self._ongoing_replacements:
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
self._ongoing_replacements.add(to_replace)
df = to_replace.ping()
df.addErrback(replaceContact, to_replace)
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
else:
df = defer.succeed(None)
return df
else:
self.touchKBucketByIndex(bucketIndex)
@@ -150,7 +155,7 @@ class TreeRoutingTable:
@param count: the amount of contacts to return, default of k (8)
@type count: int
@param sender_node_id: Used during RPC, this is be the sender's Node ID
Whatever ID is passed in the paramater will get
Whatever ID is passed in the parameter will get
excluded from the list of returned contacts.
@type sender_node_id: str
@@ -208,7 +213,7 @@ class TreeRoutingTable:
now = int(self._getTime())
for bucket in self._buckets[startIndex:]:
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
searchID = self._randomIDInBucketRange(bucketIndex)
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
refreshIDs.append(searchID)
bucketIndex += 1
return refreshIDs
@@ -256,22 +261,25 @@ class TreeRoutingTable:
i += 1
return i
def _randomIDInBucketRange(self, bucketIndex):
def random_id_in_bucket_range(self, bucketIndex):
""" Returns a random ID in the specified k-bucket's range
@param bucketIndex: The index of the k-bucket to use
@type bucketIndex: int
"""
idValue = random.randrange(
self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
randomID = hex(idValue)[2:]
if randomID[-1] == 'L':
randomID = randomID[:-1]
if len(randomID) % 2 != 0:
randomID = '0' + randomID
randomID = unhexlify(randomID)
randomID = ((constants.key_bits // 8) - len(randomID)) * b'\x00' + randomID
return randomID
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
return random_id.to_bytes(constants.key_bits // 8, 'big')
def midpoint_id_in_bucket_range(self, bucketIndex):
""" Returns the middle ID in the specified k-bucket's range
@param bucketIndex: The index of the k-bucket to use
@type bucketIndex: int
"""
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
def _splitBucket(self, oldBucketIndex):
""" Splits the specified k-bucket into two new buckets which together
File renamed without changes.
+236
View File
@@ -0,0 +1,236 @@
import sys
import json
import asyncio
import argparse
import logging
from docopt import docopt
from textwrap import dedent
import aiohttp
from lbrynet.extras.compat import force_asyncioreactor_install
force_asyncioreactor_install()
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
from lbrynet.conf import Config, CLIConfig
from lbrynet.utils import check_connection
from lbrynet.extras.daemon.Daemon import Daemon
log = logging.getLogger(lbrynet_name)
log.addHandler(logging.NullHandler())
def display(data):
print(json.dumps(data, indent=2))
async def execute_command(conf, method, params):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
async with session.get(conf.api_connection_url, json=message) as resp:
try:
data = await resp.json()
if 'result' in data:
display(data['result'])
elif 'error' in data:
if 'message' in data['error']:
display(data['error']['message'])
else:
display(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
print("Could not connect to daemon. Are you sure it's running?")
def print_help():
print(dedent("""
NAME
lbrynet - LBRY command line client.
USAGE
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
[--download_dir=<downloads directory>] <command> [<args>]
EXAMPLES
lbrynet start # starts the daemon and listens for jsonrpc commands
lbrynet help # display this message
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
lbrynet commands # list available commands
lbrynet status # get the running status of the daemon
lbrynet resolve what # resolve a name
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
# which can contain a full daemon_settings.yml config file.
# Note: since the daemon is what runs the wallet and
# downloads files, only the --data_dir setting is needed when
# running commands. The wallet_dir and download_dir would only
# by used when starting the daemon.
"""))
def print_help_for_command(command):
fn = Daemon.callable_methods.get(command)
if fn:
print(dedent(fn.__doc__))
else:
print("Invalid command name")
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
if x.lower() == 'false':
return False
if x.isdigit():
return int(x)
return x
def remove_brackets(key):
if key.startswith("<") and key.endswith(">"):
return str(key[1:-1])
return key
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
k = None
if arg is None:
continue
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
kwargs[k] = normalize_value(arg, k)
return kwargs
def get_argument_parser():
main = argparse.ArgumentParser('lbrynet', add_help=False)
main.add_argument(
'--version', dest='cli_version', action="store_true",
help='Show lbrynet CLI version and exit.'
)
main.add_argument(
'-h', '--help', dest='help', action="store_true",
help='Show this help message and exit'
)
CLIConfig.contribute_args(main)
sub = main.add_subparsers(dest='command')
help = sub.add_parser('help', help='Detailed help for remote commands.')
help.add_argument('help_command', nargs='*')
start = sub.add_parser('start', help='Start lbrynet server.')
start.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
start.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
Config.contribute_args(start)
api = Daemon.get_api_definitions()
for group in sorted(api):
group_command = sub.add_parser(group, help=api[group]['doc'])
group_command.set_defaults(group_doc=group_command)
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
continue
commands = group_command.add_subparsers(dest='subcommand')
for command in api[group]['commands']:
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
for deprecated in Daemon.deprecated_methods:
group_command = sub.add_parser(deprecated)
group_command.add_subparsers(dest='subcommand')
return main
def main(argv=None):
argv = argv or sys.argv[1:]
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
return 0
elif args.command == 'start':
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
log.debug('Final Settings: %s', conf.settings_dict)
log.info("Starting lbrynet-daemon from command line")
daemon = Daemon(conf)
if check_connection():
from twisted.internet import reactor
reactor._asyncioEventloop.create_task(daemon.start())
reactor.run()
else:
log.info("Not connected to internet, unable to start")
elif args.command == 'help':
if args.help_command:
method = '_'.join(args.help_command)
else:
parser.print_help()
return 0
if method not in Daemon.callable_methods:
print('Invalid command name: {method}')
return 1
fn = Daemon.callable_methods[method]
print(fn.__doc__)
elif args.command is not None:
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
method = args.command
elif args.subcommand is not None:
method = f'{args.command}_{args.subcommand}'
else:
args.group_doc.print_help()
return 0
if method in Daemon.deprecated_methods:
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print(f"{method} is permanently deprecated and does not have a replacement command.")
return 0
print(f"{method} is deprecated, using {new_method}.")
method = new_method
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, command_args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(conf, method, params))
else:
parser.print_help()
return 0
if __name__ == "__main__":
sys.exit(main())
+25
View File
@@ -0,0 +1,25 @@
import asyncio
def force_asyncioreactor_install():
import sys
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
def d2f(deferred):
return deferred.asFuture(asyncio.get_event_loop())
def f2d(future):
from twisted.internet import defer
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
@@ -1,7 +1,9 @@
import logging
from twisted.internet import defer
from twisted._threads import AlreadyQuit
from .ComponentManager import ComponentManager
from lbrynet.conf import Config
from lbrynet.extras.daemon.ComponentManager import ComponentManager
log = logging.getLogger(__name__)
@@ -26,6 +28,7 @@ class Component(metaclass=ComponentType):
component_name = None
def __init__(self, component_manager):
self.conf: Config = component_manager.conf
self.component_manager = component_manager
self._running = False
@@ -36,37 +39,35 @@ class Component(metaclass=ComponentType):
def running(self):
return self._running
def get_status(self):
async def get_status(self):
return
def start(self):
async def start(self):
raise NotImplementedError()
def stop(self):
async def stop(self):
raise NotImplementedError()
@property
def component(self):
raise NotImplementedError()
@defer.inlineCallbacks
def _setup(self):
async def _setup(self):
try:
result = yield defer.maybeDeferred(self.start)
result = await self.start()
self._running = True
defer.returnValue(result)
return result
except (defer.CancelledError, AlreadyQuit):
pass
except Exception as err:
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
raise err
@defer.inlineCallbacks
def _stop(self):
async def _stop(self):
try:
result = yield defer.maybeDeferred(self.stop)
result = await self.stop()
self._running = False
defer.returnValue(result)
return result
except (defer.CancelledError, AlreadyQuit):
pass
except Exception as err:
@@ -1,7 +1,9 @@
import asyncio
import logging
from twisted.internet import defer
from lbrynet.core.Error import ComponentStartConditionNotMet
from lbrynet.p2p.Error import ComponentStartConditionNotMet
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
from lbrynet.conf import Config
log = logging.getLogger(__name__)
@@ -33,13 +35,16 @@ class RequiredCondition(metaclass=RequiredConditionType):
class ComponentManager:
default_component_classes = {}
def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components):
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
peer_manager=None, peer_finder=None, **override_components):
self.conf = conf
self.skip_components = skip_components or []
self.reactor = reactor
self.component_classes = {}
self.components = set()
self.analytics_manager = analytics_manager
self.peer_manager = peer_manager or PeerManager()
self.peer_finder = peer_finder or DHTPeerFinder(self)
for component_name, component_class in self.default_component_classes.items():
if component_name in override_components:
@@ -52,18 +57,19 @@ class ComponentManager:
for component_class in self.component_classes.values():
self.components.add(component_class(self))
self.daemon = None
@defer.inlineCallbacks
def evaluate_condition(self, condition_name):
if condition_name not in RegisteredConditions.conditions:
raise NameError(condition_name)
condition = RegisteredConditions.conditions[condition_name]
try:
component = self.get_component(condition.component)
result = yield defer.maybeDeferred(condition.evaluate, component)
except Exception as err:
result = condition.evaluate(component)
except Exception:
log.exception('failed to evaluate condition:')
result = False
defer.returnValue((result, "" if result else condition.message))
return result, "" if result else condition.message
def sort_components(self, reverse=False):
"""
@@ -107,33 +113,31 @@ class ComponentManager:
steps.reverse()
return steps
@defer.inlineCallbacks
def setup(self, **callbacks):
"""
Start Components in sequence sorted by requirements
:return: (defer.Deferred)
"""
async def setup(self, **callbacks):
""" Start Components in sequence sorted by requirements """
for component_name, cb in callbacks.items():
if component_name not in self.component_classes:
raise NameError("unknown component: %s" % component_name)
if component_name not in self.skip_components:
raise NameError("unknown component: %s" % component_name)
if not callable(cb):
raise ValueError("%s is not callable" % cb)
def _setup(component):
async def _setup(component):
await component._setup()
if component.component_name in callbacks:
d = component._setup()
d.addCallback(callbacks[component.component_name], component)
return d
return component._setup()
maybe_coro = callbacks[component.component_name](component)
if asyncio.iscoroutine(maybe_coro):
asyncio.create_task(maybe_coro)
stages = self.sort_components()
for stage in stages:
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
needing_start = [
_setup(component) for component in stage if not component.running
]
if needing_start:
await asyncio.wait(needing_start)
@defer.inlineCallbacks
def stop(self):
async def stop(self):
"""
Stop Components in reversed startup order
@@ -141,7 +145,11 @@ class ComponentManager:
"""
stages = self.sort_components(reverse=True)
for stage in stages:
yield defer.DeferredList([component._stop() for component in stage if component.running])
needing_stop = [
component._stop() for component in stage if component.running
]
if needing_stop:
await asyncio.wait(needing_stop)
def all_components_running(self, *component_names):
"""
@@ -1,33 +1,41 @@
import os
import asyncio
import aiohttp
import logging
import treq
import math
import binascii
from hashlib import sha256
from types import SimpleNamespace
from twisted.internet import defer, threads, reactor, error
import lbryschema
from txupnp.upnp import UPnP
from lbrynet import conf
from lbrynet.core.utils import DeferredDict
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.core.RateLimiter import RateLimiter
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.wallet.manager import LbryWalletManager
from lbrynet.wallet.network import Network
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
from lbrynet.daemon.Component import Component
from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
from lbrynet.database.storage import SQLiteStorage
from lbrynet.dht import node, hashannouncer
from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.reflector import ServerFactory as reflector_server_factory
from twisted.internet import defer, reactor, error
from aioupnp import __version__ as aioupnp_version
from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
import lbrynet.schema
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.extras.compat import d2f
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network
from lbrynet.utils import generate_id
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
from lbrynet.core.utils import generate_id
log = logging.getLogger(__name__)
@@ -39,7 +47,6 @@ HEADERS_COMPONENT = "blockchain_headers"
WALLET_COMPONENT = "wallet"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
FILE_MANAGER_COMPONENT = "file_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
REFLECTOR_COMPONENT = "reflector"
@@ -49,53 +56,23 @@ RATE_LIMITER_COMPONENT = "rate_limiter"
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
def get_wallet_config():
wallet_type = GCS('wallet')
if wallet_type == conf.LBRYCRD_WALLET:
raise ValueError('LBRYcrd Wallet is no longer supported')
elif wallet_type != conf.LBRYUM_WALLET:
raise ValueError('Wallet Type {} is not valid'.format(wallet_type))
lbryum_servers = {address: {'t': str(port)}
for address, port in GCS('lbryum_servers')}
config = {
'auto_connect': True,
'chain': GCS('blockchain_name'),
'default_servers': lbryum_servers
}
if 'use_keyring' in conf.settings:
config['use_keyring'] = GCS('use_keyring')
if conf.settings['lbryum_wallet_dir']:
config['lbryum_path'] = GCS('lbryum_wallet_dir')
return config
async def gather_dict(tasks: dict):
async def wait_value(key, value):
return key, await value
return dict(await asyncio.gather(*(
wait_value(*kv) for kv in tasks.items()
)))
class ConfigSettings:
@staticmethod
def get_conf_setting(setting_name):
return conf.settings[setting_name]
@staticmethod
def get_blobfiles_dir():
if conf.settings['BLOBFILES_DIR'] == "blobfiles":
return os.path.join(GCS("data_dir"), "blobfiles")
else:
log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR'])
return conf.settings['BLOBFILES_DIR']
@staticmethod
def get_node_id():
return conf.settings.node_id
@staticmethod
def get_external_ip():
from lbrynet.core.system_info import get_platform
platform = get_platform(get_ip=True)
return platform['ip']
# Shorthand for common ConfigSettings methods
CS = ConfigSettings
GCS = ConfigSettings.get_conf_setting
async def get_external_ip(): # used if upnp is disabled or non-functioning
try:
async with aiohttp.ClientSession() as session:
async with session.get("https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
pass
class DatabaseComponent(Component):
@@ -113,38 +90,24 @@ class DatabaseComponent(Component):
def get_current_db_revision():
return 9
@staticmethod
def get_revision_filename():
return conf.settings.get_db_revision_filename()
@property
def revision_filename(self):
return self.component_manager.daemon.db_revision_file_path
@staticmethod
def _write_db_revision_file(version_num):
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
def _write_db_revision_file(self, version_num):
with open(self.revision_filename, mode='w') as db_revision:
db_revision.write(str(version_num))
@defer.inlineCallbacks
def start(self):
async def start(self):
# check directories exist, create them if they don't
log.info("Loading databases")
if not os.path.exists(GCS('download_directory')):
os.mkdir(GCS('download_directory'))
if not os.path.exists(GCS('data_dir')):
os.mkdir(GCS('data_dir'))
self._write_db_revision_file(self.get_current_db_revision())
log.debug("Created the db revision file: %s", self.get_revision_filename())
if not os.path.exists(CS.get_blobfiles_dir()):
os.mkdir(CS.get_blobfiles_dir())
log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir()))
if not os.path.exists(self.get_revision_filename()):
if not os.path.exists(self.revision_filename):
log.warning("db_revision file not found. Creating it")
self._write_db_revision_file(self.get_current_db_revision())
# check the db migration and run any needed migrations
with open(self.get_revision_filename(), "r") as revision_read_handle:
with open(self.revision_filename, "r") as revision_read_handle:
old_revision = int(revision_read_handle.read().strip())
if old_revision > self.get_current_db_revision():
@@ -152,21 +115,21 @@ class DatabaseComponent(Component):
'Your database is revision %i, expected %i' %
(old_revision, self.get_current_db_revision()))
if old_revision < self.get_current_db_revision():
from lbrynet.database.migrator import dbmigrator
from lbrynet.extras.daemon.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
yield threads.deferToThread(
dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision()
await asyncio.get_event_loop().run_in_executor(
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
# start SQLiteStorage
self.storage = SQLiteStorage(GCS('data_dir'))
yield self.storage.setup()
self.storage = SQLiteStorage(
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
)
await self.storage.open()
@defer.inlineCallbacks
def stop(self):
yield self.storage.stop()
async def stop(self):
await self.storage.close()
self.storage = None
@@ -179,17 +142,17 @@ class HeadersComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.headers_dir = os.path.join(conf.settings['lbryum_wallet_dir'], 'lbc_mainnet')
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
self.headers_file = os.path.join(self.headers_dir, 'headers')
self.old_file = os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
self._downloading_headers = None
self._headers_progress_percent = None
self._headers_progress_percent = 0
@property
def component(self):
return self
def get_status(self):
async def get_status(self):
return {} if not self._downloading_headers else {
'downloading_headers': self._downloading_headers,
'download_progress': self._headers_progress_percent
@@ -204,7 +167,7 @@ class HeadersComponent(Component):
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size()
resume_header = {"Range": "bytes={}-".format(local_header_size)}
resume_header = {"Range": f"bytes={local_header_size}-"}
response = yield treq.get(HEADERS_URL, headers=resume_header)
got_406 = response.code == 406 # our file is bigger
final_size_after_download = response.length + local_header_size
@@ -237,41 +200,40 @@ class HeadersComponent(Component):
return os.stat(self.headers_file).st_size
return 0
@defer.inlineCallbacks
def get_remote_height(self):
async def get_remote_height(self):
ledger = SimpleNamespace()
ledger.config = {
'default_servers': conf.settings['lbryum_servers'],
'data_path': conf.settings['lbryum_wallet_dir']
'default_servers': self.conf.lbryum_servers,
'data_path': self.conf.wallet_dir
}
net = Network(ledger)
net.start()
yield net.on_connected.first
remote_height = yield net.get_server_height()
yield net.stop()
defer.returnValue(remote_height)
first_connection = net.on_connected.first
asyncio.ensure_future(net.start())
await first_connection
remote_height = await net.get_server_height()
await net.stop()
return remote_height
@defer.inlineCallbacks
def should_download_headers_from_s3(self):
if conf.settings['blockchain_name'] != "lbrycrd_main":
defer.returnValue(False)
async def should_download_headers_from_s3(self):
if self.conf.blockchain_name != "lbrycrd_main":
return False
self._check_header_file_integrity()
s3_headers_depth = conf.settings['s3_headers_depth']
s3_headers_depth = self.conf.s3_headers_depth
if not s3_headers_depth:
defer.returnValue(False)
return False
local_height = self.local_header_file_height()
remote_height = yield self.get_remote_height()
log.info("remote height: %i, local height: %s", remote_height, local_height)
remote_height = await self.get_remote_height()
log.info("remote height: %i, local height: %i", remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
defer.returnValue(True)
defer.returnValue(False)
return True
return False
def _check_header_file_integrity(self):
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
if conf.settings['blockchain_name'] != "lbrycrd_main":
if self.conf.blockchain_name != "lbrycrd_main":
return
hashsum = sha256()
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
checksum_length_in_bytes = checksum_height * HEADER_SIZE
if self.local_header_file_size() < checksum_length_in_bytes:
return
@@ -279,7 +241,7 @@ class HeadersComponent(Component):
hashsum.update(headers_file.read(checksum_length_in_bytes))
current_checksum = hashsum.hexdigest()
if current_checksum != checksum:
msg = "Expected checksum {}, got {}".format(checksum, current_checksum)
msg = f"Expected checksum {checksum}, got {current_checksum}"
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
log.warning("Deleting header file so it can be downloaded again.")
os.unlink(self.headers_file)
@@ -288,24 +250,23 @@ class HeadersComponent(Component):
with open(self.headers_file, "rb+") as headers_file:
headers_file.truncate(checksum_length_in_bytes)
@defer.inlineCallbacks
def start(self):
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
self._downloading_headers = yield self.should_download_headers_from_s3()
self._downloading_headers = await self.should_download_headers_from_s3()
if self._downloading_headers:
try:
yield self.fetch_headers_from_s3()
await d2f(self.fetch_headers_from_s3())
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self._downloading_headers = False
def stop(self):
return defer.succeed(None)
async def stop(self):
pass
class WalletComponent(Component):
@@ -320,38 +281,35 @@ class WalletComponent(Component):
def component(self):
return self.wallet_manager
@defer.inlineCallbacks
def get_status(self):
if self.wallet_manager:
async def get_status(self):
if self.wallet_manager and self.running:
local_height = self.wallet_manager.network.get_local_height()
remote_height = self.wallet_manager.network.get_server_height()
best_hash = yield self.wallet_manager.get_best_blockhash()
defer.returnValue({
best_hash = self.wallet_manager.get_best_blockhash()
return {
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
'best_blockhash': best_hash,
'is_encrypted': self.wallet_manager.wallet.use_encryption,
'is_locked': not self.wallet_manager.is_wallet_unlocked,
})
}
@defer.inlineCallbacks
def start(self):
async def start(self):
log.info("Starting torba wallet")
storage = self.component_manager.get_component(DATABASE_COMPONENT)
lbryschema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
self.wallet_manager = LbryWalletManager.from_lbrynet_config(conf.settings, storage)
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
self.wallet_manager.old_db = storage
yield self.wallet_manager.start()
await self.wallet_manager.start()
@defer.inlineCallbacks
def stop(self):
yield self.wallet_manager.stop()
async def stop(self):
await self.wallet_manager.stop()
self.wallet_manager = None
class BlobComponent(Component):
component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
depends_on = [DATABASE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -363,21 +321,22 @@ class BlobComponent(Component):
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
datastore = None
if DHT_COMPONENT not in self.component_manager.skip_components:
dht_node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
datastore = dht_node._dataStore
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
return self.blob_manager.setup()
def stop(self):
return self.blob_manager.stop()
@defer.inlineCallbacks
def get_status(self):
async def get_status(self):
count = 0
if self.blob_manager:
count = yield self.blob_manager.storage.count_finished_blobs()
defer.returnValue({
'finished_blobs': count
})
count = await self.blob_manager.storage.count_finished_blobs()
return {'finished_blobs': count}
class DHTComponent(Component):
@@ -395,39 +354,39 @@ class DHTComponent(Component):
def component(self):
return self.dht_node
def get_status(self):
async def get_status(self):
return {
'node_id': binascii.hexlify(CS.get_node_id()),
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
}
@defer.inlineCallbacks
def start(self):
async def start(self):
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port"))
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port"))
node_id = CS.get_node_id()
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
node_id = self.component_manager.daemon.node_id
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = node.Node(
self.dht_node = Node(
node_id=node_id,
udpPort=GCS('dht_node_port'),
udpPort=self.conf.dht_node_port,
externalUDPPort=self.external_udp_port,
externalIP=self.upnp_component.external_ip,
externalIP=external_ip,
peerPort=self.external_peer_port
)
self.dht_node.start_listening()
yield self.dht_node._protocol._listening
d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
d.addCallback(lambda _: self.dht_node.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
log.info("Started the dht")
@defer.inlineCallbacks
def stop(self):
yield self.dht_node.stop()
return d2f(self.dht_node.stop())
class HashAnnouncerComponent(Component):
@@ -442,18 +401,16 @@ class HashAnnouncerComponent(Component):
def component(self):
return self.hash_announcer
@defer.inlineCallbacks
def start(self):
async def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
yield self.hash_announcer.start()
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
self.hash_announcer.start()
@defer.inlineCallbacks
def stop(self):
yield self.hash_announcer.stop()
self.hash_announcer.stop()
def get_status(self):
async def get_status(self):
return {
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
}
@@ -470,48 +427,11 @@ class RateLimiterComponent(Component):
def component(self):
return self.rate_limiter
def start(self):
async def start(self):
self.rate_limiter.start()
return defer.succeed(None)
def stop(self):
async def stop(self):
self.rate_limiter.stop()
return defer.succeed(None)
class StreamIdentifierComponent(Component):
component_name = STREAM_IDENTIFIER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.sd_identifier = StreamDescriptorIdentifier()
@property
def component(self):
return self.sd_identifier
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
add_lbry_file_to_sd_identifier(self.sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
dht_node.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
GCS('download_directory')
)
yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
def stop(self):
pass
class PaymentRateComponent(Component):
@@ -525,17 +445,17 @@ class PaymentRateComponent(Component):
def component(self):
return self.payment_rate_manager
def start(self):
return defer.succeed(None)
async def start(self):
pass
def stop(self):
return defer.succeed(None)
async def stop(self):
pass
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -545,36 +465,47 @@ class FileManagerComponent(Component):
def component(self):
return self.file_manager
def get_status(self):
async def get_status(self):
if not self.file_manager:
return
return {
'managed_files': len(self.file_manager.lbry_files)
}
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
sd_identifier = StreamDescriptorIdentifier()
add_lbry_file_to_sd_identifier(sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
self.conf,
self.component_manager.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
self.conf.download_dir
)
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager')
self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier)
yield self.file_manager.setup()
log.info('Done setting up file manager')
self.file_manager = EncryptedFileManager(
self.conf, self.component_manager.peer_finder, rate_limiter,
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
)
return self.file_manager.setup()
@defer.inlineCallbacks
def stop(self):
yield self.file_manager.stop()
return d2f(self.file_manager.stop())
class PeerProtocolServerComponent(Component):
component_name = PEER_PROTOCOL_SERVER_COMPONENT
depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
@@ -585,11 +516,10 @@ class PeerProtocolServerComponent(Component):
def component(self):
return self.lbry_server_port
@defer.inlineCallbacks
def start(self):
async def start(self):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
upnp = self.component_manager.get_component(UPNP_COMPONENT)
peer_port = GCS('peer_port')
peer_port = self.conf.peer_port
query_handlers = {
handler.get_primary_query_identifier(): handler for handler in [
BlobRequestHandlerFactory(
@@ -603,13 +533,13 @@ class PeerProtocolServerComponent(Component):
}
server_factory = ServerProtocolFactory(
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
self.component_manager.get_component(DHT_COMPONENT).peer_manager
self.component_manager.peer_manager
)
try:
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
upnp.upnp_redirects.get("TCP", peer_port))
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
@@ -617,47 +547,43 @@ class PeerProtocolServerComponent(Component):
log.error("%s", traceback.format_exc())
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
@defer.inlineCallbacks
def stop(self):
async def stop(self):
if self.lbry_server_port is not None:
self.lbry_server_port, old_port = None, self.lbry_server_port
log.info('Stop listening on port %s', old_port.port)
yield old_port.stopListening()
await d2f(old_port.stopListening())
class ReflectorComponent(Component):
component_name = REFLECTOR_COMPONENT
depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.reflector_server_port = GCS('reflector_port')
self.reflector_server_port = self.conf.reflector_port
self.reflector_server = None
@property
def component(self):
return self.reflector_server
@defer.inlineCallbacks
def start(self):
async def start(self):
log.info("Starting reflector server")
dht_node = self.component_manager.get_component(DHT_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
try:
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
log.info('Started reflector on port %s', self.reflector_server_port)
except error.CannotListenError as e:
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
raise ValueError("{} lbrynet may already be running on your computer.".format(e))
raise ValueError(f"{e} lbrynet may already be running on your computer.")
@defer.inlineCallbacks
def stop(self):
async def stop(self):
if self.reflector_server is not None:
log.info("Stopping reflector server")
self.reflector_server, p = None, self.reflector_server
yield p.stopListening
await d2f(p.stopListening())
class UPnPComponent(Component):
@@ -665,54 +591,131 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = GCS('peer_port')
self._int_dht_node_port = GCS('dht_node_port')
self.use_upnp = GCS('use_upnp')
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
self.external_ip = None
self._maintain_redirects_task = None
@property
def component(self):
return self
@defer.inlineCallbacks
def _setup_redirects(self):
upnp_redirects = yield DeferredDict({
"UDP": self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"),
"TCP": self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
})
self.upnp_redirects.update(upnp_redirects)
async def _repeatedly_maintain_redirects(self, now=True):
while True:
if now:
await self._maintain_redirects()
await asyncio.sleep(360)
@defer.inlineCallbacks
def start(self):
if not self.use_upnp:
self.external_ip = CS.get_external_ip()
return
self.upnp = UPnP(self.component_manager.reactor, try_miniupnpc_fallback=True)
try:
found = yield self.upnp.discover()
except Exception as err:
log.warning("upnp discovery failed: %s", err)
found = False
if found and not self.upnp.miniupnpc_runner:
log.info("set up redirects using txupnp")
elif found and self.upnp.miniupnpc_runner:
log.warning("failed to set up redirect with txupnp, miniupnpc fallback was successful")
if found:
async def _maintain_redirects(self):
# setup the gateway if necessary
if not self.upnp:
try:
self.external_ip = yield self.upnp.get_external_ip()
yield self._setup_redirects()
self.upnp = await UPnP.discover()
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
except Exception as err:
log.warning("error trying to set up upnp: %s", err)
self.external_ip = CS.get_external_ip()
else:
self.external_ip = CS.get_external_ip()
log.warning("upnp discovery failed: %s", err)
self.upnp = None
def stop(self):
return defer.DeferredList(
[self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()]
)
# update the external ip
external_ip = None
if self.upnp:
try:
external_ip = await self.upnp.get_external_ip()
if external_ip != "0.0.0.0" and not self.external_ip:
log.info("got external ip from UPnP: %s", external_ip)
except (asyncio.TimeoutError, UPnPError):
pass
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
external_ip = await get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
self.external_ip = external_ip
assert self.external_ip is not None # TODO: handle going/starting offline
if not self.upnp_redirects and self.upnp: # setup missing redirects
try:
log.info("add UPnP port mappings")
d = {}
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
d["TCP"] = self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
if DHT_COMPONENT not in self.component_manager.skip_components:
d["UDP"] = self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
upnp_redirects = await gather_dict(d)
log.info("set up redirects: %s", upnp_redirects)
self.upnp_redirects.update(upnp_redirects)
except (asyncio.TimeoutError, UPnPError):
self.upnp = None
return self._maintain_redirects()
elif self.upnp: # check existing redirects are still active
found = set()
mappings = await self.upnp.get_redirects()
for mapping in mappings:
proto = mapping['NewProtocol']
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
if mapping['NewInternalClient'] == self.upnp.lan_address:
found.add(proto)
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
try:
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
self.upnp_redirects['UDP'] = udp_port
log.info("refreshed upnp redirect for dht port: %i", udp_port)
except (asyncio.TimeoutError, UPnPError):
del self.upnp_redirects['UDP']
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
try:
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
self.upnp_redirects['TCP'] = tcp_port
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
except (asyncio.TimeoutError, UPnPError):
del self.upnp_redirects['TCP']
if ('TCP' in self.upnp_redirects
and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
if self.upnp_redirects:
log.debug("upnp redirects are still active")
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await get_external_ip()
return
success = False
await self._maintain_redirects()
if self.upnp:
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info)
else:
success = True
if self.upnp_redirects:
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
else:
log.error("failed to setup upnp")
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self):
if self.upnp_redirects:
await asyncio.wait([
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
])
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
self._maintain_redirects_task.cancel()
async def get_status(self):
return {
'aioupnp_version': aioupnp_version,
'redirects': self.upnp_redirects,
'gateway': 'No gateway found' if not self.upnp else self.upnp.gateway.manufacturer_string,
'dht_redirect_set': 'UDP' in self.upnp_redirects,
'peer_redirect_set': 'TCP' in self.upnp_redirects,
'external_ip': self.external_ip
}
class ExchangeRateManagerComponent(Component):
@@ -726,10 +729,8 @@ class ExchangeRateManagerComponent(Component):
def component(self):
return self.exchange_rate_manager
@defer.inlineCallbacks
def start(self):
yield self.exchange_rate_manager.start()
async def start(self):
self.exchange_rate_manager.start()
@defer.inlineCallbacks
def stop(self):
yield self.exchange_rate_manager.stop()
async def stop(self):
self.exchange_rate_manager.stop()
File diff suppressed because it is too large. Load diff
@@ -1,17 +1,17 @@
import logging
import os
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbryschema.fee import Fee
from lbrynet.conf import Config
from lbrynet.schema.fee import Fee
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
from lbrynet.core.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet import conf
from torba.constants import COIN
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from torba.client.constants import COIN
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from lbrynet.extras.compat import f2d
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
@@ -31,14 +31,18 @@ log = logging.getLogger(__name__)
class GetStream:
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None):
self.timeout = timeout or conf.settings['download_timeout']
self.data_rate = data_rate or conf.settings['data_rate']
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
self.download_directory = conf.settings['download_directory']
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
timeout=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self.conf = conf
self.reactor = reactor
self.timeout = timeout or conf.download_timeout
self.data_rate = data_rate or conf.data_rate
self.max_key_fee = max_key_fee or conf.max_key_fee
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
self.download_directory = conf.download_dir
self.timeout_counter = 0
self.code = None
self.sd_hash = None
@@ -51,44 +55,17 @@ class GetStream:
self.sd_identifier = sd_identifier
self.storage = storage
self.downloader = None
self.checker = LoopingCall(self.check_status)
# fired when the download is complete
self.finished_deferred = None
# fired after the metadata and the first data blob have been downloaded
self.data_downloading_deferred = defer.Deferred(None)
self.wrote_data = False
@property
def download_path(self):
return os.path.join(self.download_directory, self.downloader.file_name)
def _check_status(self, status):
if status.num_completed > 0 and not self.data_downloading_deferred.called:
self.data_downloading_deferred.callback(True)
if self.data_downloading_deferred.called:
safe_stop_looping_call(self.checker)
else:
log.debug("Waiting for stream data (%i seconds)", self.timeout_counter)
def check_status(self):
"""
Check if we've got the first data blob in the stream yet
"""
self.timeout_counter += 1
if self.timeout_counter > self.timeout:
if not self.data_downloading_deferred.called:
if self.downloader:
err = DownloadDataTimeout(self.sd_hash)
else:
err = DownloadSDTimeout(self.sd_hash)
self.data_downloading_deferred.errback(err)
safe_stop_looping_call(self.checker)
elif self.downloader:
d = self.downloader.status()
d.addCallback(self._check_status)
else:
log.debug("Waiting for stream descriptor (%i seconds)", self.timeout_counter)
def convert_max_fee(self):
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
@@ -102,11 +79,11 @@ class GetStream:
max_key_fee_amount = self.convert_max_fee()
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
fee.amount)
if converted_fee_amount > (yield self.wallet.default_account.get_balance()):
if converted_fee_amount > (yield f2d(self.wallet.default_account.get_balance())):
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
raise KeyFeeAboveMaxAllowed('Key fee %s above max allowed %s' % (converted_fee_amount,
max_key_fee_amount))
raise KeyFeeAboveMaxAllowed('Key fee {} above max allowed {}'.format(converted_fee_amount,
max_key_fee_amount))
converted_fee = {
'currency': 'LBC',
'amount': converted_fee_amount,
@@ -118,7 +95,7 @@ class GetStream:
for factory in factories:
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
return factory
raise Exception('No suitable factory was found in {}'.format(factories))
raise Exception(f'No suitable factory was found in {factories}')
@defer.inlineCallbacks
def get_downloader(self, factory, stream_metadata, file_name=None):
@@ -134,11 +111,13 @@ class GetStream:
defer.returnValue(downloader)
def _pay_key_fee(self, address, fee_lbc, name):
log.info("Pay key fee %f --> %s", fee_lbc, address)
log.info("Pay key fee %s --> %s", dewies_to_lbc(fee_lbc), address)
reserved_points = self.wallet.reserve_points(address, fee_lbc)
if reserved_points is None:
raise InsufficientFundsError('Unable to pay the key fee of %s for %s' % (fee_lbc, name))
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
raise InsufficientFundsError(
'Unable to pay the key fee of {} for {}'.format(dewies_to_lbc(fee_lbc), name)
)
return f2d(self.wallet.send_points_to_address(reserved_points, fee_lbc))
@defer.inlineCallbacks
def pay_key_fee(self, fee, name):
@@ -147,18 +126,13 @@ class GetStream:
else:
defer.returnValue(None)
@defer.inlineCallbacks
def finish(self, results, name):
self.set_status(DOWNLOAD_STOPPED_CODE, name)
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
self.download_path)
safe_stop_looping_call(self.checker)
status = yield self.downloader.status()
self._check_status(status)
defer.returnValue(self.download_path)
return defer.succeed(self.download_path)
def fail(self, err):
safe_stop_looping_call(self.checker)
raise err
@defer.inlineCallbacks
@@ -180,8 +154,8 @@ class GetStream:
@defer.inlineCallbacks
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, conf.settings['download_mirrors']
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, self.conf.download_mirrors
)
defer.returnValue(sd_blob)
@@ -189,9 +163,11 @@ class GetStream:
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
yield self.pay_key_fee(key_fee, name)
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
self.finished_deferred = self.downloader.start()
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
self.data_downloading_deferred.callback
)
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
@defer.inlineCallbacks
@@ -207,16 +183,19 @@ class GetStream:
"""
self.set_status(INITIALIZING_CODE, name)
key_fee = yield self._initialize(stream_info)
safe_start_looping_call(self.checker, 1)
self.set_status(DOWNLOAD_METADATA_CODE, name)
try:
sd_blob = yield self._download_sd_blob()
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
yield self.data_downloading_deferred
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.data_downloading_deferred.addTimeout(self.timeout, self.reactor)
try:
yield self.data_downloading_deferred
self.wrote_data = True
except defer.TimeoutError:
raise DownloadDataTimeout("data download timed out")
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
safe_stop_looping_call(self.checker)
raise err
defer.returnValue((self.downloader, self.finished_deferred))
@@ -2,28 +2,30 @@ import binascii
import logging
from twisted.internet import defer, task
from lbrynet.core import utils
from lbrynet import conf
from lbrynet.extras.compat import f2d
from lbrynet import utils
from lbrynet.conf import Config
log = logging.getLogger(__name__)
class DHTHashAnnouncer:
def __init__(self, dht_node, storage, concurrent_announcers=None):
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
self.conf = conf
self.dht_node = dht_node
self.storage = storage
self.clock = dht_node.clock
self.peer_port = dht_node.peerPort
self.hash_queue = []
if concurrent_announcers is None:
self.concurrent_announcers = conf.settings['concurrent_announcers']
self.concurrent_announcers = conf.concurrent_announcers
else:
self.concurrent_announcers = concurrent_announcers
self._manage_lc = None
if self.concurrent_announcers:
self._manage_lc = task.LoopingCall(self.manage)
self._manage_lc.clock = self.clock
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.settings['concurrent_announcers'] or 1)
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
def start(self):
if self._manage_lc:
@@ -39,7 +41,7 @@ class DHTHashAnnouncer:
now = self.clock.seconds()
if storing_node_ids:
result = (now, storing_node_ids)
yield self.storage.update_last_announced_blob(blob_hash, now)
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
else:
result = (None, [])
@@ -83,7 +85,7 @@ class DHTHashAnnouncer:
if not self.dht_node.contacts:
log.info("Not ready to start announcing hashes")
return
need_reannouncement = yield self.storage.get_blobs_to_announce()
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
if need_reannouncement:
yield self.immediate_announce(need_reannouncement)
else:
+70
View File
@@ -0,0 +1,70 @@
import binascii
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class DummyPeerFinder:
"""This class finds peers which have announced to the DHT that they have certain blobs"""
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
return defer.succeed([])
class DHTPeerFinder(DummyPeerFinder):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
#implements(IPeerFinder)
def __init__(self, component_manager):
"""
component_manager - an instance of ComponentManager
"""
self.component_manager = component_manager
self.peer_manager = component_manager.peer_manager
self.peers = {}
self._ongoing_searchs = {}
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
"""
Find peers for blob in the DHT
blob_hash (str): blob hash to look for
timeout (int): seconds to timeout after
filter_self (bool): if True, and if a peer for a blob is itself, filter it
from the result
Returns:
list of peers for the blob
"""
if "dht" in self.component_manager.skip_components:
return defer.succeed([])
if not self.component_manager.all_components_running("dht"):
return defer.succeed([])
dht_node = self.component_manager.get_component("dht")
self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
def _filter_self(blob_hash):
my_host, my_port = dht_node.externalIP, dht_node.peerPort
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
@defer.inlineCallbacks
def _execute_peer_search(self, dht_node, blob_hash, timeout):
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
timeout = timeout or self.component_manager.conf.peer_search_timeout
if timeout:
finished_deferred.addTimeout(timeout, dht_node.clock)
try:
peer_list = yield finished_deferred
self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
except defer.TimeoutError:
log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
finally:
del self._ongoing_searchs[blob_hash]
@@ -1,4 +1,4 @@
from lbrynet.core.Peer import Peer
from lbrynet.p2p.Peer import Peer
class PeerManager:
@@ -1,16 +1,21 @@
import asyncio
import logging
import mimetypes
import os
from twisted.internet import defer
from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
from lbrynet.extras.daemon.mime_types import guess_media_type
log = logging.getLogger(__name__)
def d2f(d):
return d.asFuture(asyncio.get_event_loop())
class Publisher:
def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate):
def __init__(self, account, blob_manager, payment_rate_manager, storage,
lbry_file_manager, wallet, certificate):
self.account = account
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.storage = storage
@@ -19,60 +24,54 @@ class Publisher:
self.certificate = certificate
self.lbry_file = None
@defer.inlineCallbacks
def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
async def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
"""Create lbry file and make claim"""
log.info('Starting publish for %s', name)
if not os.path.isfile(file_path):
raise Exception("File {} not found".format(file_path))
raise Exception(f"File {file_path} not found")
if os.path.getsize(file_path) == 0:
raise Exception("Cannot publish empty file {}".format(file_path))
raise Exception(f"Cannot publish empty file {file_path}")
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as read_handle:
self.lbry_file = yield create_lbry_file(
self.lbry_file = await d2f(create_lbry_file(
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
read_handle
)
))
if 'source' not in claim_dict['stream']:
claim_dict['stream']['source'] = {}
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
tx = yield self.wallet.claim_name(
name, bid, claim_dict, self.certificate, holding_address
tx = await self.wallet.claim_name(
self.account, name, bid, claim_dict, self.certificate, holding_address
)
# check if we have a file already for this claim (if this is a publish update with a new stream)
old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
tx.outputs[0].claim_id, self.lbry_file.stream_hash
)
if old_stream_hashes:
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
list(self.lbry_file_manager.lbry_files)):
yield self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False)
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
yield self.storage.save_content_claim(
await self.storage.save_content_claim(
self.lbry_file.stream_hash, tx.outputs[0].id
)
defer.returnValue(tx)
return tx
@defer.inlineCallbacks
def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
"""Make a claim without creating a lbry file"""
tx = yield self.wallet.claim_name(
name, bid, claim_dict, self.certificate, holding_address
tx = await self.wallet.claim_name(
self.account, name, bid, claim_dict, self.certificate, holding_address
)
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
yield self.storage.save_content_claim(
stream_hash.decode(), tx.outputs[0].id
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
defer.returnValue(tx)
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
return tx
+1
View File
@@ -0,0 +1 @@
from lbrynet.extras.daemon import Components # register Component classes
+250
View File
@@ -0,0 +1,250 @@
import asyncio
import collections
import logging
import aiohttp
from lbrynet import utils
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.extras import system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
SERVER_STARTUP_ERROR = 'Server Startup Error'
DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
UPNP_SETUP = "UPnP Setup"
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
log = logging.getLogger(__name__)
class Manager:
def __init__(self, conf: Config, installation_id: str, session_id: str):
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = self._make_context(system_info.get_platform(), 'torba')
self.installation_id = installation_id
self.session_id = session_id
self.task: asyncio.Task = None
@property
def is_started(self):
return self.task is not None
def start(self):
if self._enabled and self.task is None:
self.task = asyncio.create_task(self.run())
log.info("Start")
async def run(self):
while True:
await self._send_heartbeat()
await asyncio.sleep(1800)
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
request_kwargs = {
'method': 'POST',
'url': self.url + endpoint,
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with aiohttp.request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
async def track(self, event):
"""Send a single tracking event"""
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post('/track', event)
async def send_new_download_start(self, download_id, name, claim_dict):
await self._send_new_download_stats("start", download_id, name, claim_dict)
async def send_new_download_success(self, download_id, name, claim_dict):
await self._send_new_download_stats("success", download_id, name, claim_dict)
async def send_new_download_fail(self, download_id, name, claim_dict, e):
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
await self.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
async def send_upnp_setup_success_fail(self, success, status):
await self.track(
self._event(UPNP_SETUP, {
'success': success,
'status': status,
})
)
async def send_server_startup(self):
await self.track(self._event(SERVER_STARTUP))
async def send_server_startup_success(self):
await self.track(self._event(SERVER_STARTUP_SUCCESS))
async def send_server_startup_error(self, message):
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
async def send_download_started(self, id_, name, claim_dict=None):
await self.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
async def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
async def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
async def send_claim_action(self, action):
await self.track(self._event(CLAIM_ACTION, {'action': action}))
async def send_new_channel(self):
await self.track(self._event(NEW_CHANNEL))
async def send_credits_sent(self):
await self.track(self._event(CREDITS_SENT))
async def _send_heartbeat(self):
await self.track(self._event(HEARTBEAT))
async def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
@@ -1,12 +1,10 @@
import asyncio
import aiohttp
import time
import logging
import json
import treq
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.core.Error import InvalidExchangeRateResponse, CurrencyConversionError
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
log = logging.getLogger(__name__)
@@ -38,14 +36,14 @@ class MarketFeed:
REQUESTS_TIMEOUT = 20
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
def __init__(self, market, name, url, params, fee):
def __init__(self, market: str, name: str, url: str, params, fee):
self.market = market
self.name = name
self.url = url
self.params = params
self.fee = fee
self.rate = None
self._updater = LoopingCall(self._update_price)
self._task: asyncio.Task = None
self._online = True
def rate_is_initialized(self):
@@ -54,17 +52,16 @@ class MarketFeed:
def is_online(self):
return self._online
@defer.inlineCallbacks
def _make_request(self):
response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT)
defer.returnValue((yield response.content()))
async def _make_request(self):
async with aiohttp.request('get', self.url, params=self.params) as response:
return (await response.read()).decode()
def _handle_response(self, response):
return NotImplementedError
raise NotImplementedError()
def _subtract_fee(self, from_amount):
# increase amount to account for market fees
return defer.succeed(from_amount / (1.0 - self.fee))
return from_amount / (1.0 - self.fee)
def _save_price(self, price):
log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
@@ -77,21 +74,23 @@ class MarketFeed:
log.debug("Exchange rate error (%s from %s): %s", self.market, self.name, err)
self._online = False
def _update_price(self):
d = self._make_request()
d.addCallback(self._handle_response)
d.addCallback(self._subtract_fee)
d.addCallback(self._save_price)
d.addErrback(self._on_error)
return d
async def _update_price(self):
while True:
try:
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
self._save_price(self._subtract_fee(self._handle_response(response)))
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
self._on_error(err)
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
def start(self):
if not self._updater.running:
self._updater.start(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
if not self._task:
self._task = asyncio.create_task(self._update_price())
def stop(self):
if self._updater.running:
self._updater.stop()
if self._task and not self._task.done():
self._task.cancel()
self._task = None
class BittrexFeed(MarketFeed):
@@ -116,7 +115,7 @@ class BittrexFeed(MarketFeed):
if totals <= 0 or qtys <= 0:
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
vwap = totals / qtys
return defer.succeed(float(1.0 / vwap))
return float(1.0 / vwap)
class LBRYioFeed(MarketFeed):
@@ -133,7 +132,7 @@ class LBRYioFeed(MarketFeed):
json_response = json.loads(response)
if 'data' not in json_response:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(1.0 / json_response['data']['lbc_btc'])
return 1.0 / json_response['data']['lbc_btc']
class LBRYioBTCFeed(MarketFeed):
@@ -153,7 +152,7 @@ class LBRYioBTCFeed(MarketFeed):
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
if 'data' not in json_response:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(1.0 / json_response['data']['btc_usd'])
return 1.0 / json_response['data']['btc_usd']
class CryptonatorBTCFeed(MarketFeed):
@@ -174,7 +173,7 @@ class CryptonatorBTCFeed(MarketFeed):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
'success' not in json_response or json_response['success'] is not True:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(float(json_response['ticker']['price']))
return float(json_response['ticker']['price'])
class CryptonatorFeed(MarketFeed):
@@ -195,7 +194,7 @@ class CryptonatorFeed(MarketFeed):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
'success' not in json_response or json_response['success'] is not True:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(float(json_response['ticker']['price']))
return float(json_response['ticker']['price'])
class ExchangeRateManager:
@@ -234,7 +233,7 @@ class ExchangeRateManager:
return self.convert_currency(
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
raise CurrencyConversionError(
'Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
f'Unable to convert {amount} from {from_currency} to {to_currency}')
def fee_dict(self):
return {market: market.rate.as_dict() for market in self.market_feeds}
@@ -0,0 +1,104 @@
import logging
from decimal import Decimal
from binascii import hexlify
from datetime import datetime
from json import JSONEncoder
from ecdsa import BadSignatureError
from lbrynet.extras.wallet import MainNetLedger
from lbrynet.extras.wallet.transaction import Transaction, Output
from lbrynet.extras.wallet.dewies import dewies_to_lbc
log = logging.getLogger(__name__)
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
return float(obj)
if isinstance(obj, bytes):
return obj.decode()
return super().default(obj)
def encode_transaction(self, tx):
return {
'txid': tx.id,
'height': tx.height,
'inputs': [self.encode_input(txo) for txo in tx.inputs],
'outputs': [self.encode_output(txo) for txo in tx.outputs],
'total_input': dewies_to_lbc(tx.input_sum),
'total_output': dewies_to_lbc(tx.input_sum - tx.fee),
'total_fee': dewies_to_lbc(tx.fee),
'hex': hexlify(tx.raw).decode(),
}
def encode_output(self, txo):
tx_height = txo.tx_ref.height
best_height = self.ledger.headers.height
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_involved:
output.update({
'name': txo.claim_name,
'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url,
})
if txo.script.is_claim_name or txo.script.is_update_claim:
claim = txo.claim
output['value'] = claim.claim_dict
if claim.has_signature:
output['valid_signature'] = None
if txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
try:
output['valid_signature'] = claim.validate_signature(
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
)
except BadSignatureError:
output['valid_signature'] = False
except ValueError:
log.exception(
'txo.id: %s, txo.channel.id:%s, output: %s',
txo.id, txo.channel.id, output
)
output['valid_signature'] = False
if txo.script.is_claim_name:
output['type'] = 'claim'
elif txo.script.is_update_claim:
output['type'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'basic'
return output
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
+64
View File
@@ -0,0 +1,64 @@
import asyncio
import aiohttp
import json
import logging.handlers
import traceback
from lbrynet import utils, __version__
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
async def _emit(self, record):
payload = self.format(record)
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
def emit(self, record):
asyncio.ensure_future(self._emit(record))
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
handler = HTTPSLogglyHandler(loggly_token)
handler.setFormatter(JsonFormatter())
return handler
@@ -3,29 +3,29 @@ import logging
log = logging.getLogger(__name__)
def migrate_db(db_dir, start, end):
def migrate_db(conf, start, end):
current = start
while current < end:
if current == 1:
from lbrynet.database.migrator.migrate1to2 import do_migration
from .migrate1to2 import do_migration
elif current == 2:
from lbrynet.database.migrator.migrate2to3 import do_migration
from .migrate2to3 import do_migration
elif current == 3:
from lbrynet.database.migrator.migrate3to4 import do_migration
from .migrate3to4 import do_migration
elif current == 4:
from lbrynet.database.migrator.migrate4to5 import do_migration
from .migrate4to5 import do_migration
elif current == 5:
from lbrynet.database.migrator.migrate5to6 import do_migration
from .migrate5to6 import do_migration
elif current == 6:
from lbrynet.database.migrator.migrate6to7 import do_migration
from .migrate6to7 import do_migration
elif current == 7:
from lbrynet.database.migrator.migrate7to8 import do_migration
from .migrate7to8 import do_migration
elif current == 8:
from lbrynet.database.migrator.migrate8to9 import do_migration
from .migrate8to9 import do_migration
else:
raise Exception("DB migration of version {} to {} is not available".format(current,
current+1))
do_migration(db_dir)
do_migration(conf)
current += 1
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
return None
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
UNSET_NOUT = -1
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blockchainname_db(db_dir)
migrate_blockchainname_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blockchainname_db(db_dir)
migrate_blockchainname_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blobs_db(db_dir)
migrate_blobs_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
add_lbry_file_metadata(db_dir)
add_lbry_file_metadata(conf.data_dir)
log.info("Migration succeeded")
@@ -2,13 +2,11 @@ import sqlite3
import os
import json
import logging
from lbryschema.decode import smart_decode
from lbrynet import conf
from binascii import hexlify
from lbrynet.schema.decode import smart_decode
log = logging.getLogger(__name__)
download_directory = conf.settings['download_directory']
CREATE_TABLES_QUERY = """
pragma foreign_keys=on;
pragma journal_mode=WAL;
@@ -105,13 +103,13 @@ def verify_sd_blob(sd_hash, blob_dir):
return decoded, sd_length
def do_migration(db_dir):
new_db_path = os.path.join(db_dir, "lbrynet.sqlite")
def do_migration(conf):
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(new_db_path)
metadata_db = sqlite3.connect(os.path.join(db_dir, "blockchainname.db"))
lbryfile_db = sqlite3.connect(os.path.join(db_dir, 'lbryfile_info.db'))
blobs_db = sqlite3.connect(os.path.join(db_dir, 'blobs.db'))
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
name_metadata_cursor = metadata_db.cursor()
lbryfile_cursor = lbryfile_db.cursor()
@@ -187,10 +185,14 @@ def do_migration(db_dir):
(stream_hash, blob_hash, position, iv)
)
download_dir = conf.download_dir
if not isinstance(download_dir, bytes):
download_dir = download_dir.encode()
# insert the file
transaction.execute(
"insert or ignore into file values (?, ?, ?, ?, ?)",
(stream_hash, stream_name, download_directory.encode('hex'),
(stream_hash, stream_name, hexlify(download_dir),
data_rate, status)
)
@@ -275,7 +277,7 @@ def do_migration(db_dir):
# recover damaged streams
if damaged_stream_sds:
blob_dir = os.path.join(db_dir, "blobfiles")
blob_dir = os.path.join(conf.data_dir, "blobfiles")
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
if p in damaged_stream_sds})
for damaged_sd in damaged_sds_on_disk:
@@ -313,7 +315,7 @@ def do_migration(db_dir):
log.warning("detected a failed previous migration to revision 6, repairing it")
connection.close()
os.remove(new_db_path)
return do_migration(db_dir)
return do_migration(conf)
raise err
connection.close()
@@ -2,8 +2,8 @@ import sqlite3
import os
def do_migration(db_dir):
db_path = os.path.join(db_dir, "lbrynet.sqlite")
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
cursor.executescript("alter table blob add last_announced_time integer;")
Loaded 100 of 287 files, more files were not shown because too many files have changed in this diff. Show more