Compare commits

...
1448 Commits
Author SHA1 Message Date
Victor Shyba 3a0affe8ba show status while downloading headers 2019-06-11 02:25:49 -03:00
Jack Robison 06a9770504 Merge pull request #2216 from lbryio/fix-connection-status-startup-error
fix connections status raising an error during startup
2019-06-07 13:55:54 -04:00
Jack Robison 0da43d3e05 fix connections status raising an error during startup 2019-06-07 12:14:05 -04:00
Lex Berezhny 077dbb3138 better handle claims which fail to decode 2019-06-07 08:42:17 -04:00
Jack Robison 3b8fd65fa9 Merge pull request #2212 from lbryio/add-torba-version-to-version-rpc
update `version` docstring, add `torba_version`
2019-06-05 20:59:58 -04:00
Jack Robison d0ddaa5518 update version docstring, add torba_version 2019-06-05 20:33:21 -04:00
Jack Robison a60ab97552 Merge pull request #2211 from lbryio/pin-aioupnp
pin aioupnp requirement to 0.0.12
2019-06-05 19:41:45 -04:00
Jack Robison 078d09976c pin aioupnp requirement 2019-06-05 18:50:36 -04:00
Jack Robison 3770558694 Merge pull request #2210 from lbryio/fix-wine-build
fix torba version on travis
2019-06-05 18:46:24 -04:00
Jack Robison 4409b8c4f8 fix torba version on travis 2019-06-05 18:19:56 -04:00
Lex Berezhny f5b7aaac18 v0.38.0rc6 2019-06-04 22:32:33 -04:00
Lex Berezhny 6dbb6b9079 fix claim expiration calculation 2019-06-04 22:28:35 -04:00
Lex Berezhny 16b774a69e add normalized to claims_in_channel index to make update claim faster 2019-06-04 22:28:35 -04:00
Lex Berezhny 72ea19532e added index specifically for claim_in_channel computation 2019-06-04 22:28:35 -04:00
Lex Berezhny ad24946af3 add improved profiling to validate_channel_signatures 2019-06-04 22:28:35 -04:00
Lex Berezhny b676907fd7 added index for claim expiration_height 2019-06-04 22:28:35 -04:00
Lex Berezhny 6965971613 delete claims and re-calculate winner when claim is expired 2019-06-04 22:28:35 -04:00
Lex Berezhny 6651111522 pylint 2019-06-04 18:44:21 -04:00
Lex Berezhny d8630a02d9 Strings setting works like Server from CLI 2019-06-04 18:44:21 -04:00
Lex Berezhny 69345b4b0e integrationt tests fixed 2019-06-04 16:04:46 -04:00
Lex Berezhny f23aea9951 better handling for claims in abandoned channels 2019-06-04 16:04:46 -04:00
Lex Berezhny f96c46e84a claim search fixes for filtering with valid_channel_signatures 2019-06-04 16:04:46 -04:00
Alex Grintsvayg bae2939943 change a few instances of lbry.io to lbry.com. fixes #2135 2019-06-04 10:23:04 -04:00
Alex Grintsvayg 7063138e3c small improvements from our cli investigation 2019-06-04 09:39:33 -04:00
Lex Berezhny 798ba9acf8 v0.38.0rc5 2019-06-04 01:20:32 -04:00
Jack Robison c8dd1987e6 add connection status 2019-06-04 01:20:05 -04:00
Lex Berezhny 8d2c9e5785 claim_type/stream_type/media_type claim search integration test 2019-06-04 00:51:25 -04:00
Lex Berezhny 9dbf47916b add public_key_id to channel integration tests 2019-06-03 21:56:17 -04:00
Lex Berezhny 565ef5e454 docopt doc string fix 2019-06-03 21:56:17 -04:00
Lex Berezhny 9f1f2b1781 claim search with --public_key_id 2019-06-03 21:56:17 -04:00
Lex Berezhny 76df239d1a start command accepts server lists consistent with other commands 2019-06-03 21:22:24 -04:00
Thomas Zarebczan 86ee6a78f5 add MKV to mime types 2019-06-03 19:51:48 -04:00
Lex Berezhny 4fb6f2bb27 review fixes 2019-06-03 06:51:09 -04:00
Oleg Silkin ec039825e8 Fixes non-subscribtable object error 2019-06-03 06:51:09 -04:00
Oleg Silkin a10f09d169 Updates signature return field 2019-06-03 06:51:09 -04:00
Oleg Silkin 76409600e1 Fixes stuff up 2019-06-03 06:51:09 -04:00
Oleg Silkin d06a0f876a Patches comment server interface in the SDK 2019-06-03 06:51:09 -04:00
Oleg Silkin 8e857de62a Fixes non-subscribtable object error 2019-06-03 06:51:09 -04:00
Oleg Silkin 3d37f7c735 Updates signature return field 2019-06-03 06:51:09 -04:00
Oleg Silkin 552b94bc65 Fixes stuff up 2019-06-03 06:51:09 -04:00
Oleg Silkin 17164601a1 Patches comment server interface in the SDK 2019-06-03 06:51:09 -04:00
Lex Berezhny 26dd6d2ac7 claim search test fix 2019-05-30 21:57:49 -04:00
Lex Berezhny 5d5e36a2c2 short_url and canonical_url returned from wallet server use original claim name instead of normalized 2019-05-30 21:32:35 -04:00
Lex Berezhny 77afbcded6 wallet server, skip table join for count(*) query 2019-05-30 21:07:07 -04:00
Lex Berezhny ae11d72f66 added claim.effective_amount index on wallet server 2019-05-30 16:13:46 -04:00
Lex Berezhny d2773f9713 v0.38.0rc4 2019-05-29 22:48:28 -04:00
Lex Berezhny a7fba569cd update tests to use valid_channel_signatures argument 2019-05-29 22:47:33 -04:00
Lex Berezhny 0559c64038 fixed claim search bug related to signed claims 2019-05-29 22:47:33 -04:00
Lex Berezhny 5ab2754bc6 shutdown daemon during tests teardown 2019-05-29 19:27:05 -04:00
Lex Berezhny 6bfb8384e7 channel import/export working tests 2019-05-29 19:27:05 -04:00
Lex Berezhny 0156aa8b00 channel import/export wip 2019-05-29 19:27:05 -04:00
Lex Berezhny ad411cc157 trying to fix rebase 2019-05-29 19:27:05 -04:00
hackrush 6d462ad6d8 channel export import 2019-05-29 19:27:05 -04:00
Lex Berezhny 1b35cef77a fix wallet.server.test_sqldb test 2019-05-28 20:51:22 -04:00
Lex Berezhny a357208a77 cleaned up with passing tests 2019-05-28 20:51:22 -04:00
hackrush 7e1f8a56d5 Changed txo.private_key to be an object 2019-05-28 20:51:22 -04:00
hackrush 37665f2df7 Review Fixes 2019-05-28 20:51:22 -04:00
hackrush 8f3b596385 Add consolidated field to migration result 2019-05-28 20:51:22 -04:00
hackrush e45c6fe42c Only store pubkey-private key pairs 2019-05-28 20:51:22 -04:00
hackrush 56edf62cb9 Temporary fix for github.com/PyCQA/pylint/issues/2820 2019-05-28 20:51:22 -04:00
hackrush d367ff6ac1 Update tests and code 2019-05-28 20:51:22 -04:00
hackrush 7a78d49e13 Store public-private mappings in wallet 2019-05-28 20:51:22 -04:00
Lex Berezhny 4dae1ae05f regenerate docs 2019-05-27 23:18:26 -04:00
Lex Berezhny 9004f240e6 bug fixes 2019-05-27 23:18:26 -04:00
Lex Berezhny 1848462ad9 v0.38.0rc3 2019-05-26 14:04:45 -04:00
Lex Berezhny 786efa06dd fixed resolve to work with partial claim_ids 2019-05-26 14:01:06 -04:00
Lex Berezhny b6811d55cf no longer log skipped claim insertions on wallet server, happens too often 2019-05-26 14:01:06 -04:00
Lex Berezhny 42dc0eccd6 wallet server channel key change error 2019-05-26 14:01:06 -04:00
Lex Berezhny 29bd936181 wallet server tracks claim signature 2019-05-26 14:01:06 -04:00
Lex Berezhny 9d97b90ff4 pylint 2019-05-26 14:01:06 -04:00
Lex Berezhny 370b34f860 canonical_url takes into account whether channel signature is valid 2019-05-26 14:01:06 -04:00
Lex Berezhny affa46e0f6 initial import of canonical url via the new sql wallet server 2019-05-26 14:01:06 -04:00
Jack Robison 6b93705599 Merge pull request #2181 from lbryio/improve-set-build
improve set_build.py, pin torba version
2019-05-24 19:00:38 -04:00
Jack Robison 925e39c5a8 improve set_build.py, pin torba version
- adds BUILD_COMMIT to lbrynet.build_type
2019-05-24 18:02:55 -04:00
Jack Robison 1efa2c9406 Merge pull request #2180 from lbryio/fix-completed-false-negative
fix false negative with file `completed` field
2019-05-24 17:18:04 -04:00
Jack Robison 4f8bcfcd18 fix false negative with file completed field 2019-05-24 16:47:25 -04:00
Jack Robison 3ed72a215e Merge pull request #2179 from lbryio/fix-range-request-start-offset
start returning bytes for a range request at the requested starting position
2019-05-24 16:46:21 -04:00
Jack Robison 85d94d4ca3 start returning range request bytes at the requested offset 2019-05-24 15:32:40 -04:00
Jack Robison b922ea8e7e Merge pull request #2178 from lbryio/lru-cached-read-blob
add a lru cache for decrypted blobs to minimize redownloading them, add `blob_lru_cache_size` to the config to set the cache size
2019-05-24 13:56:59 -04:00
Jack Robison cb6ddaf276 add blob_lru_cache_size config setting, defaults to 32 2019-05-24 12:02:58 -04:00
Lex Berezhny 1c2029cddf fix startup error when single key account present in wallet 2019-05-22 18:31:49 -04:00
Alex Grintsvayg d8d136f1d8 uniform spacing on docstring 2019-05-22 16:51:14 -04:00
Victor Shyba fedc0e5bc3 add checktrie.py script 2019-05-22 12:20:33 -04:00
Alex Grintsvayg 4fbccb9336 uniform spacing on docstring 2019-05-22 09:23:31 -04:00
Lex Berezhny ef5dd5593f v0.38.0rc2 2019-05-20 21:39:16 -04:00
Lex Berezhny baed974be0 several minor wallet server fixes 2019-05-20 21:35:20 -04:00
Lex Berezhny b2b3509b61 regenerate protobufs 2019-05-20 15:19:14 -04:00
Lex Berezhny 51720d9b73 add MAX() to claimtrie GROUP BY algo to make results consistent across SQLite versions 2019-05-20 15:19:14 -04:00
Lex Berezhny 6abb569ec8 v0.38.0rc1 2019-05-19 23:03:25 -04:00
Lex Berezhny c2fbbadc69 start trending calculation as wallet server sync nears best height 2019-05-19 21:16:19 -04:00
Lex Berezhny b8897223ec added zscore trending in wallet server 2019-05-19 21:16:19 -04:00
hackrush c2c184b4ef Updated to include the environment variables needed in mac 2019-05-18 21:44:49 -04:00
Lex Berezhny 07dbdef3fe dont run test_resolve_lru_cache_doesnt_persist_errors 2019-05-18 21:43:46 -04:00
Lex Berezhny b841fb2f0c fix docopt parser issue 2019-05-18 21:43:46 -04:00
Lex Berezhny 2b75982446 disable trending until its fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 021ca3efd2 updated doc strings and fixed wallet server claimtrie implementation 2019-05-18 21:43:46 -04:00
Lex Berezhny e56b72cf04 pylint and use torba master 2019-05-18 21:43:46 -04:00
Lex Berezhny 342f485ddf good stuff 2019-05-18 21:43:46 -04:00
Lex Berezhny 8fe16fbfc9 performance 2019-05-18 21:43:46 -04:00
Lex Berezhny 1e3035029e timer fix 2019-05-18 21:43:46 -04:00
Lex Berezhny 5f4c02f836 wallet server sync performance improvements 2019-05-18 21:43:46 -04:00
Lex Berezhny 03a1185e0f pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny 0e1705d19e instrumented most wallet server claim related functions with timer 2019-05-18 21:43:46 -04:00
Lex Berezhny 82c739daa6 abstract _apply_constraints_for_array_attributes for wallet server 2019-05-18 21:43:46 -04:00
Lex Berezhny 4702bea9f0 claim_search --not_tags feature + integration tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 75211b7e84 updated claim_search doc string 2019-05-18 21:43:46 -04:00
Lex Berezhny 9f9273bd82 working claim_search with tags 2019-05-18 21:43:46 -04:00
Lex Berezhny d2608bdf43 fixed wallet/server/test_sqldb.py tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 091bef1ead sequence/amount_order resolve 2019-05-18 21:43:46 -04:00
Lex Berezhny 998ee9feba resolve integration tests fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 587f0b2ea8 integration tests for jsonrpc_transaction_show fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 048aa07e80 test_file_commands integration tests fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 7ca01511f4 fixing integration tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 8dee83aac0 disable cache for integration tests while using custom torba branch 2019-05-18 21:43:46 -04:00
Lex Berezhny 879ae5cc72 .travis.yml 2019-05-18 21:43:46 -04:00
Lex Berezhny 74a9e17178 .travis.yml 2019-05-18 21:43:46 -04:00
Lex Berezhny 93d2173fad chris45 integration test passing 2019-05-18 21:43:46 -04:00
Lex Berezhny a459ee34be test_url unit test fix 2019-05-18 21:43:46 -04:00
Lex Berezhny 2e6297f834 pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny 9accbfcf8b move exception handling into is_signed_by 2019-05-18 21:43:46 -04:00
Lex Berezhny 6946860521 added result schema types 2019-05-18 21:43:46 -04:00
Lex Berezhny 9b8be8682c massive progress 2019-05-18 21:43:46 -04:00
Lex Berezhny 03455310ae rewrote URL parser 2019-05-18 21:43:46 -04:00
Lex Berezhny e5edaed677 removed test code from wallet server daemon.py 2019-05-18 21:43:46 -04:00
Lex Berezhny 5e97215eaa integration tests should clone sqltrie torba branch 2019-05-18 21:43:46 -04:00
Lex Berezhny 266f94bc95 pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny d26caded53 use sqltrie branch of torba 2019-05-18 21:43:46 -04:00
Lex Berezhny 2260608bb6 working sql claimtrie 2019-05-18 21:43:46 -04:00
Lex Berezhny bb1f42057e v0.37.1 2019-05-17 23:01:47 -04:00
Lex Berezhny 695f6658a0 v0.37.1rc2 2019-05-17 21:43:37 -04:00
Jack Robison eca677a720 Merge pull request #2148 from lbryio/lru-cache-concurrent
Fix resolve internals caching and persisting errors
2019-05-17 19:26:22 -04:00
Jack Robison 51f301785f fix test, add lru_cache_concurrent decorator 2019-05-17 18:57:07 -04:00
Jack Robison 41a6e288aa add test_resolve_lru_cache_doesnt_persist_errors 2019-05-17 18:56:28 -04:00
Jack Robison 97d8d0ef4c Merge pull request #2133 from lbryio/more-wallets
Add missing wallet server
2019-05-17 15:50:10 -04:00
Niko Storni 5baeb5f538 fix examples 2019-05-17 15:19:11 -04:00
Niko Storni cc49c8407c add missing wallet server
switch to .com domain
2019-05-17 15:19:11 -04:00
Jack Robison 3f37963f7f Merge pull request #2139 from lbryio/android-platform-name
fix system name on android
2019-05-17 14:39:20 -04:00
Jack Robison 12eb83f773 fix system name on android 2019-05-17 14:00:25 -04:00
Jack Robison ca1d135ba4 Merge pull request #2138 from lbryio/fix-file-save-doc
fix save_file doc
2019-05-17 13:44:22 -04:00
Jack Robison 107e57e3ff fix docs 2019-05-17 13:11:04 -04:00
Jack Robison 928689a10d Merge pull request #2130 from lbryio/fix-datastore
Fix updating refreshed announcements in the datastore
2019-05-17 12:58:11 -04:00
Jack Robison 6ecc22f2c7 fix refreshing peers in the datastore 2019-05-17 12:22:47 -04:00
Lex Berezhny 079c81f298 v0.37.1rc1 2019-05-17 08:59:10 -04:00
Victor Shyba 8afe178d62 remove supports from resolve 2019-05-17 08:55:39 -04:00
Jack Robison 8777dcee7c v0.37.0 2019-05-13 23:39:22 -04:00
Jack Robison 1cca4a91dd Merge pull request #2128 from lbryio/add-streaming-server-settings
add `streaming_server` and `streaming_get` settings to configure the interface/port of the media server and if it supports the `/get` endpoint
2019-05-13 22:51:53 -04:00
Jack Robison 41abfbdd9b add streaming_server and streaming_get settings
-split rpc runner from range request runner
2019-05-13 22:27:48 -04:00
Jack Robison 46ee65c8f2 Merge pull request #2127 from lbryio/add-streaming-url
add `streaming_url` field to `File` api response type
2019-05-13 21:05:52 -04:00
Jack Robison fd19daad9e add streaming_url to file api response type 2019-05-13 19:05:26 -04:00
Jack Robison 26d183cbab Merge pull request #2125 from lbryio/dht_wip
Remove search junctions, make search retry, some refactoring
2019-05-13 18:02:56 -04:00
Victor Shyba 71a4be02dd use loop to create task 2019-05-13 15:57:03 -03:00
Victor Shyba 124fdb7d31 fixes from review 2019-05-13 14:57:58 -03:00
Victor Shyba 2439743804 probe peer instead of scheduling ping for removal 2019-05-13 14:51:03 -03:00
Victor Shyba 6f79875135 fix test 2019-05-13 14:27:49 -03:00
Victor Shyba b91d2190f4 disable infinite peer search, cleanup logging, tune scores to slow connections 2019-05-13 14:24:13 -03:00
Victor Shyba d024433d1b simplify accumulate task 2019-05-13 13:34:49 -03:00
Victor Shyba 7c4eb416d6 dont clearbanned if there are working peers, dont remove node immediately during search 2019-05-13 13:14:08 -03:00
Victor Shyba fb457c820a reorganizing iterative find code 2019-05-13 04:34:39 -03:00
Victor Shyba 18af2dcd4e remove peer immediately instead of probing when its bad during search 2019-05-13 02:52:24 -03:00
Victor Shyba 0075dcc2c0 iterative find probes bad peers + test case for edge scenario Tom found 2019-05-13 02:40:04 -03:00
Victor Shyba 6e327d729b fix test 2019-05-12 04:28:58 -03:00
Victor Shyba 20022d49c1 less verbose dht tests 2019-05-12 04:13:58 -03:00
Victor Shyba 32d1563885 fix handling of invalid token 2019-05-12 04:12:12 -03:00
Victor Shyba f5cef14d8d test get token on announce 2019-05-12 03:55:07 -03:00
Victor Shyba bac7d99b8a add ability to re-join network on disconnect + tests 2019-05-12 03:39:11 -03:00
Victor Shyba 6f06026511 announcer goes idle if no peers in routing table 2019-05-12 01:10:05 -03:00
Victor Shyba f02df86709 fix announce loop when there are no peers to announce 2019-05-12 00:42:19 -03:00
Victor Shyba b7d76fd09f add integration tests, fix bug that made refresh not exclude bad peers 2019-05-11 23:30:05 -03:00
Victor Shyba e675f1387c remove peer junction and some refactoring 2019-05-11 04:58:56 -03:00
Victor Shyba 0da2827c78 fix dht deadlock 2019-05-10 19:54:59 -03:00
Jack Robison 309cfc840c v0.37.0rc7 2019-05-10 15:34:38 -04:00
Jack Robison 68ee31b62a Merge pull request #2123 from lbryio/stop-timed-out-downloads
Stop timed out downloads
2019-05-10 15:29:33 -04:00
Jack Robison ad2da24d64 improve test 2019-05-10 14:57:04 -04:00
Jack Robison 648dbcc36b add timeout to blob downloads after the head blob, stop timed out streams 2019-05-10 14:50:01 -04:00
Victor Shyba e59e96190e fix peer search using inexistent variable, fix test to cover it 2019-05-10 13:49:35 -04:00
Jack Robison 7d2f592bb5 Merge pull request #2107 from lbryio/fix-update-file-we-dont-have
fix stream_update failing on metadata only updates for files we don't have
2019-05-10 12:28:53 -04:00
Jack Robison 1a182bc9bd bytes/str 2019-05-10 11:50:20 -04:00
Jack Robison 324cbdcdb2 only save the content claim if stream_hash isn't None
-stream_hash being none means we don't have the file in the database
2019-05-10 11:50:20 -04:00
Lex Berezhny 6c0c0ee322 added logging to publish command 2019-05-10 11:43:34 -04:00
Jack Robison 1e54271d8a Merge pull request #2105 from lbryio/fix-store-content-fee
fix store content fee
2019-05-09 18:47:30 -04:00
Jack Robison 54b9ed6cb2 fix store content fee 2019-05-09 18:14:54 -04:00
Lex Berezhny 558a02d187 correctly format lat/long and fee amounts in JSON RPC response 2019-05-09 16:19:49 -04:00
Jack Robison 3567b331c0 v0.37.0rc6 2019-05-09 11:52:34 -04:00
Jack Robison 9e4cf3c995 Merge pull request #2048 from lbryio/timeout_settings
Adjust default max_connections_per_download and peer_connect_timeout settings
2019-05-09 11:14:59 -04:00
Thomas Zarebczan db1d547b19 adjust the default max_connections_per_download and peer_connect_timeout 2019-05-09 10:38:32 -04:00
Jack Robison c31efd780b docs 2019-05-09 10:32:07 -04:00
Jack Robison 7cdb7987e3 Merge pull request #2102 from lbryio/shutdown_fast
Set a shutdown timeout for TCPSite to avoid hanging on exit
2019-05-09 10:24:13 -04:00
Victor Shyba ec07037e88 set shutdown_timeout on TCPSite 2019-05-09 01:01:07 -03:00
Lex Berezhny 9bb7b0571e fix integrat tests 2019-05-09 00:00:39 -04:00
Lex Berezhny d98162a118 stream_update --replace also clears the channel 2019-05-09 00:00:39 -04:00
Jack Robison c5ae59bff7 Merge pull request #2100 from lbryio/junction_fixes
DHT junction fixes for cases that can lock up
2019-05-08 23:52:54 -04:00
Victor Shyba efbf2f49a9 fix generator junction 2019-05-08 23:02:38 -03:00
Victor Shyba 77c69f661d add locking cases scenarios 2019-05-08 23:02:38 -03:00
Victor Shyba 2e09d2adfa close search on transport close 2019-05-08 23:02:38 -03:00
Jack Robison 8620a0e424 Merge pull request #2095 from lbryio/add-content-fee-and-saved-file-to-db
add saved_file and content_fee columns to file table
2019-05-08 20:37:56 -04:00
Jack Robison e2c123456b write decrypted blob data from run_in_executor 2019-05-08 16:07:32 -04:00
Jack Robison c9be79f062 prevent re-migrating 10 to 11 2019-05-08 15:13:07 -04:00
Jack Robison a2a1ef34ab fix tests 2019-05-08 14:55:09 -04:00
Jack Robison 761a2b235c set written_bytes, download_path, file_name, and download_directory to None if the file doesn't exist 2019-05-08 14:11:44 -04:00
Jack Robison 3ca2fee592 use stream size from claim if available 2019-05-08 14:11:39 -04:00
Jack Robison a480b2d25f fix race when closing blob client protocol after getting a response 2019-05-08 12:21:11 -04:00
Jack Robison d7032b12d7 add saved_file and content_fee columns to file table
-drop not null constraints for file_name and download_directory

-add migrator
2019-05-08 10:26:06 -04:00
Jack Robison 84381ff76c Merge pull request #2093 from lbryio/fix_announce_on_storage
Refactor announcer + fixups for last db migrator code
2019-05-07 10:46:12 -04:00
Victor Shyba 007dd43861 re-raise cancelled errors 2019-05-07 11:10:49 -03:00
Victor Shyba 1bc4e4b702 typo on logging, announcements logging is info, not warning 2019-05-07 11:10:49 -03:00
Victor Shyba b91f27219d use fetchall as there is an update while iterating 2019-05-07 11:10:49 -03:00
Victor Shyba dc4560cc9a refactor announcer 2019-05-07 11:10:49 -03:00
Victor Shyba 84b471d486 limit batch size on announcer test so it covers batching logic 2019-05-07 11:10:49 -03:00
Victor Shyba c6a3c05a0c add missing migrator line 2019-05-07 11:10:49 -03:00
Jack Robison dd665a758d Merge pull request #2065 from lbryio/streaming-bug-fixes
Streaming bug fixes
2019-05-07 10:10:39 -04:00
Jack Robison 32dc1297ec fix infinite loop 2019-05-06 18:46:26 -04:00
Jack Robison bef244ba77 dont make duplicate files 2019-05-06 18:46:26 -04:00
Jack Robison 3c231d6215 handle get with file_name/download_directory/save_file arg for a stream that is already running 2019-05-06 18:46:26 -04:00
Jack Robison a212cf6ba4 set content_fee attribute 2019-05-06 18:46:26 -04:00
Jack Robison bc060f1378 pay fee after downloading the sd blob/adding to the db 2019-05-06 18:46:26 -04:00
Jack Robison 22c701fd50 fix completed field for an item in file list 2019-05-06 18:46:26 -04:00
Jack Robison 4f0e8fce6e add optional download_directory argument to get 2019-05-06 18:46:26 -04:00
Jack Robison f642cfe9dd fix resuming save tasks on startup 2019-05-06 18:46:26 -04:00
Jack Robison 24e073680b add connection id workaround 2019-05-06 18:46:26 -04:00
Jack Robison b2f63a1545 fix tests 2019-05-06 18:46:26 -04:00
Jack Robison f506b3e6d4 tests 2019-05-06 18:46:26 -04:00
Jack Robison f7412514b3 fix partial content 0 padding 2019-05-06 18:46:26 -04:00
Jack Robison bd6a609b30 fix logging 2019-05-06 18:46:26 -04:00
Jack Robison 4e32b69d1d don't set running streams as stopped on startup 2019-05-06 18:46:26 -04:00
Jack Robison 1116c7f29e logging 2019-05-06 18:46:26 -04:00
Jack Robison d80c671cf2 look victor 2019-05-06 18:46:26 -04:00
Jack Robison 14b12cbea2 block /stream and /get requests on the stream manager starting 2019-05-06 18:46:26 -04:00
Jack Robison 1f7feafb67 force close open streaming requests and api calls on shutdown 2019-05-06 18:46:26 -04:00
Jack Robison cbe689ea7f logging 2019-05-06 18:46:26 -04:00
Jack Robison 9099ee2e8e fix/refactor starting and stopping files
-move partial content handling into ManagedStream
-add delayed stop test
2019-05-06 18:46:26 -04:00
Jack Robison b134e0c9c9 fix blob_get 2019-05-06 18:46:26 -04:00
Jack Robison f8c0e80cfc add __slots__ to BlobInfo, StreamDescriptor, and KademliaPeer 2019-05-06 18:46:26 -04:00
Jack Robison 1f266ebdad add logging for streams we already have 2019-05-06 18:46:26 -04:00
Jack Robison b09e1f8825 fix written_bytes never being 0 2019-05-06 18:46:26 -04:00
Jack Robison c496278a5b Merge pull request #2092 from lbryio/fix_announce_on_storage
Set should_announce=1 to head and descriptor blobs when saving a stream to database
2019-05-06 18:45:36 -04:00
Victor Shyba cacc65d45e fix edge case from test for single blob stream 2019-05-06 18:18:12 -03:00
Victor Shyba 2247b354b3 integration test for announcing both on publish and download 2019-05-06 17:27:12 -03:00
Victor Shyba 43111f0497 bump db revision 2019-05-06 17:19:40 -03:00
Victor Shyba 6c142b4477 add migrator for previous bugs 2019-05-06 17:12:04 -03:00
Victor Shyba e8683a24bd fix should_announce being set on stream creation 2019-05-06 16:49:52 -03:00
Victor Shyba a4d17d1705 re-enable tests for announcing on stream creation 2019-05-06 16:49:03 -03:00
Baltazar Gomez a030e13548 fix incorrect media-type of .cbr archives 2019-05-05 20:57:36 -04:00
Jack Robison 32b8601c38 Merge pull request #2083 from lbryio/fix_get_params
make save_file a parameter instead of a flag so it obeys conf
2019-05-05 19:44:29 -04:00
Victor Shyba f5bd28bfdb make save_file a parameter instead of a flag so it obeys conf 2019-05-05 19:46:42 -03:00
Lex Berezhny 6cbbe23c7c account_list defaults to 0 confirmations for balance 2019-05-04 18:08:33 -04:00
Lex Berezhny d7c2e4e9b3 fixed bug where lbrynet failed to start with single-key accounts in the wallet 2019-05-04 17:56:39 -04:00
Jack Robison d61ddbb950 Merge pull request #2076 from lbryio/download_improv
make ttfb configurable, fix race condition on writers, ban on time scales, remove idle code for getting unstuck on long downloads, fix foreign key error and improve downloader loop checks
2019-05-02 17:18:16 -04:00
Victor Shyba cf7bb6a391 fix tests and stream being deleted on data timeout 2019-05-02 16:41:48 -03:00
Victor Shyba 68d7328918 always close the write handle 2019-05-02 12:20:49 -03:00
Victor Shyba ff2985cc80 close protocol if transport reuse brings responses before requests 2019-05-02 12:13:33 -03:00
Victor Shyba ac69fcbc5f fix possible division by zero 2019-05-02 11:52:09 -03:00
Victor Shyba 609cf42868 cleanup downloader code 2019-05-01 18:04:45 -03:00
Victor Shyba e43b29fcd1 if the current writer is closed, its fine to open a new one 2019-05-01 15:09:11 -03:00
Victor Shyba 6594b7c50c test concurrency through races on getting a writer 2019-05-01 15:08:47 -03:00
Victor Shyba 971f2a1cef log download speed only if downloading full streams 2019-05-01 14:31:11 -03:00
Victor Shyba fc88261cc6 add head_blob_only to ttfb 2019-05-01 13:59:45 -03:00
Victor Shyba 895b3992c9 set type for stall download timeout 2019-05-01 13:26:51 -03:00
Victor Shyba 41951d8ec2 improve ttfb script confs 2019-05-01 13:17:13 -03:00
Victor Shyba ae2121c5c4 fix tests and new test 2019-05-01 13:16:45 -03:00
Victor Shyba 771fe935f4 test get-delete-get respects existing files 2019-05-01 13:16:32 -03:00
Victor Shyba 60f7766cf7 small adjustment to new_peer_or_finished 2019-05-01 02:55:51 -03:00
Victor Shyba 63bd983012 pylint 2019-05-01 00:42:51 -03:00
Victor Shyba 7ffce1040c fix foreign key error 2019-04-30 19:51:19 -03:00
Victor Shyba ac6835fa60 dont get stuck 2019-04-30 19:51:02 -03:00
Victor Shyba 3cf5c536c0 improve ban timing and downloader loop friction 2019-04-30 17:56:29 -03:00
Lex Berezhny 85cc1a4763 v0.37.0rc5 2019-04-29 17:22:47 -04:00
Lex Berezhny 180cfcfb72 fix chris45 tests 2019-04-29 16:01:59 -04:00
Lex Berezhny 864c1dd901 fix test_claim_commands integration tests 2019-04-29 16:01:59 -04:00
Lex Berezhny aeaa7ba986 switched from sha256 for unencrypted file hash to sha384 2019-04-29 16:01:59 -04:00
Lex Berezhny b8325c1f89 regenerated docs 2019-04-29 09:49:39 -04:00
Lex Berezhny a36936b5ff v0.37.0rc4 2019-04-29 00:27:53 -04:00
Victor Shyba 7cf245a0e9 encode timestamp and confirmations into file list 2019-04-29 00:26:14 -04:00
Victor Shyba 9f83cce574 test confirmations and timestamp on file list, being updated on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 30b113d64a depth -> confirmations 2019-04-29 00:26:14 -04:00
Victor Shyba ccd2023d37 add timestamp field to resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 3127cd11cf test for timestamp field on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 6e483cba8d tests: add confirm parameters, like the other calls 2019-04-28 22:53:43 -04:00
Victor Shyba 46c03c0cb2 refactor: move generate to blockchain call 2019-04-28 22:53:43 -04:00
Victor Shyba c2ca0a9467 tests: replace sleep with the proper generate call 2019-04-28 22:53:43 -04:00
Victor Shyba b350e74aca fix stream update callback to update the main dict 2019-04-28 22:53:43 -04:00
Victor Shyba de9473f75a test resolve updates file list 2019-04-28 22:53:43 -04:00
Lex Berezhny bfc107fdb3 removed blockchain.block.get_block from wallet server API 2019-04-27 18:03:35 -04:00
Lex Berezhny 6a2efa54ab dropped block_show command from API, users should run a full node if they need access to blocks 2019-04-27 18:03:35 -04:00
Lex Berezhny 72f0e1ca14 added timestamp field to transaction output results from JSON RPC calls 2019-04-27 17:10:49 -04:00
Lex Berezhny 9a1262d557 regenerate docs 2019-04-27 13:13:20 -04:00
Lex Berezhny 706a8e52dc v0.37.0rc3 2019-04-27 13:12:27 -04:00
Lex Berezhny f3c8776878 fixed --preview mode on stream_create/update 2019-04-27 13:11:32 -04:00
Lex Berezhny 539ea04039 fixed signing streams using --channel_name argument 2019-04-27 11:52:29 -04:00
Lex Berezhny 8f96b07559 updating stream claim with new file now correctly updates file list removing old file 2019-04-27 10:55:20 -04:00
Lex Berezhny 0a1e97fe23 cli test fix 2019-04-26 23:37:10 -04:00
Lex Berezhny da637b9c0e added --replace option to stream_update and channel_update, publish defaults to --replace 2019-04-26 23:37:10 -04:00
Lex Berezhny 3618de1227 added --clear_fee ability to stream update command 2019-04-26 17:52:29 -04:00
Victor Shyba 3cb8a63bf9 add channel_name parameter on claim search 2019-04-26 17:30:21 -04:00
Victor Shyba bad99de5b3 add tests for channel_name, which should work for urls too 2019-04-26 17:30:21 -04:00
Lex Berezhny 3d1efbc9bb v0.37.0rc2 2019-04-25 20:40:07 -04:00
Victor Shyba 7bdef3146b fix ttfb 2019-04-25 20:38:33 -04:00
Victor Shyba 2145bfcabf install uvloop on wallet server docker image and default to it 2019-04-25 20:07:10 -04:00
Lex Berezhny b88832d0a9 regenerated docs/api.json 2019-04-24 23:03:30 -04:00
Lex Berezhny db08882f20 v0.37.0rc1 2019-04-24 21:35:59 -04:00
Jack Robison 6f52d36b22 Merge pull request #2020 from lbryio/download-range-requests
Support HTTP 206 partial content requests for streaming downloads
2019-04-24 14:21:41 -04:00
Jack Robison 056a629e87 remove unused property 2019-04-24 13:39:06 -04:00
Jack Robison 1ffb99f9f9 rename streaming_only setting -> save_files 2019-04-24 12:44:12 -04:00
Jack Robison a8e612773d test 2019-04-24 09:48:04 -04:00
Jack Robison 67088b2ca2 test_blob_writers_concurrency 2019-04-24 09:48:04 -04:00
Jack Robison dab49e812f set writer result or error before calling close_handle 2019-04-24 09:48:04 -04:00
Jack Robison 0f3f38d8a3 pylint 2019-04-24 09:48:04 -04:00
Jack Robison 7ba433c2e5 fix setting blob verified event and canceling blob writers
-raise OSError if multiple writers for one peer are opened for a blob
2019-04-24 09:48:04 -04:00
Jack Robison bb7649e8ad test file_save stopped before finished writing with streaming_only on/off 2019-04-24 09:48:04 -04:00
Jack Robison 832537a5cf set deleted downloads as streaming mode on startup 2019-04-24 09:48:04 -04:00
Jack Robison 3234d70270 add peer address/port to hash blob writer 2019-04-24 09:48:04 -04:00
Jack Robison 432fe444f8 fix cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 861c25716e add unit tests for cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Victor Shyba 9ba6653475 raise proper error when sd is not available 2019-04-24 09:48:04 -04:00
Jack Robison 8107e97304 remove debugging /streams endpoint 2019-04-24 09:48:04 -04:00
Jack Robison e2663f6d57 remove unnecessary except/raise 2019-04-24 09:48:04 -04:00
Jack Robison 35479dcbd2 add file_save integration tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a997277aa reduce the amount of Blob objects created 2019-04-24 09:48:04 -04:00
Jack Robison a7d94f4514 add file_save 2019-04-24 09:48:04 -04:00
Jack Robison a0e3338177 only re-reflect streams if the sd blob is readable 2019-04-24 09:48:04 -04:00
Jack Robison d5a353066c missing await 2019-04-24 09:48:04 -04:00
Jack Robison bc29fc4465 type 2019-04-24 09:48:04 -04:00
Jack Robison 987f175fcc logging 2019-04-24 09:48:04 -04:00
Jack Robison 5b1398b588 rename test_streaming.py 2019-04-24 09:48:04 -04:00
Jack Robison 873b58efc9 update example_daemon_settings.yml 2019-04-24 09:48:04 -04:00
Jack Robison 01a81fceed more streaming_only + save_blobs integration testing 2019-04-24 09:48:04 -04:00
Jack Robison a14b0aa573 tests 2019-04-24 09:48:04 -04:00
Jack Robison a1abb7c8fb test range requests 2019-04-24 09:48:04 -04:00
Jack Robison 46b166952b non async blob_completed callback, tests 2019-04-24 09:48:04 -04:00
Jack Robison 1375153fda fix publish updates not being saved to the db, don't query the db for the stream hash 2019-04-24 09:48:04 -04:00
Jack Robison 1a14157d37 fix unawaited task 2019-04-24 09:48:04 -04:00
Jack Robison 40b9fa38b0 update mock dht 2019-04-24 09:48:04 -04:00
Jack Robison 4d0e9e0428 relative http redirect from /get -> /stream 2019-04-24 09:48:04 -04:00
Jack Robison 3a53ef5690 catch TransportNotConnected 2019-04-24 09:48:04 -04:00
Jack Robison 33a68b5cef handle dht is_closing 2019-04-24 09:48:04 -04:00
Jack Robison 63bf084a6a cancel dht response future 2019-04-24 09:48:04 -04:00
Jack Robison a882f2e8c5 merge conflicts 2019-04-24 09:48:04 -04:00
Jack Robison 119e0ec408 cancelled error 2019-04-24 09:48:04 -04:00
Jack Robison 7d782d516a track and close blob readers 2019-04-24 09:48:04 -04:00
Jack Robison d3eac48035 tests, add __slots__ to AbstractBlob 2019-04-24 09:48:04 -04:00
Jack Robison 60c9ae64b4 endpoint to call get and redirect to the partial content stream 2019-04-24 09:48:04 -04:00
Jack Robison 2d314dce60 add save_file argument to get 2019-04-24 09:48:04 -04:00
Jack Robison 287b89db66 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 7a160f7335 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 04ddee7d89 blob tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a916a8e8e tests 2019-04-24 09:48:04 -04:00
Jack Robison f125468ebf fix claim not being saved to the db 2019-04-24 09:48:04 -04:00
Jack Robison f0e17cff9a support streaming downloads / range requests 2019-04-24 09:48:04 -04:00
Jack Robison 93267efe0b remove StreamAssembler, refactor ManagedStream and StreamDownloader (WIP) 2019-04-24 09:48:04 -04:00
Jack Robison c663e5a3cf add cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 676f0015aa refactor BlobFile into AbstractBlob, BlobFile, and BlobBuffer classes 2019-04-24 09:48:04 -04:00
Jack Robison d44a79ada2 cache resolve_host, add async_timed_cache decorator 2019-04-24 09:48:04 -04:00
Jack Robison d7ed8ffaa6 add streaming_only and save_blobs settings 2019-04-24 09:48:04 -04:00
Jack Robison b230d8fbcb rename BlobFileManager -> BlobManager 2019-04-24 09:48:04 -04:00
Lex Berezhny 7d2428fd17 regenerate api.json 2019-04-23 23:49:39 -04:00
Lex Berezhny 4157045a41 block_show command example 2019-04-23 23:49:39 -04:00
Lex Berezhny 58987857cf v0.36.0 2019-04-22 18:47:58 -04:00
Lex Berezhny fa8858325c add lbry:// to permanent_url response field 2019-04-22 18:41:11 -04:00
Lex Berezhny 3357a419ab ability to overide the file name, hash and size when updating a stream claim 2019-04-22 18:05:49 -04:00
Lex Berezhny a782adea75 regenerate api doc 2019-04-22 14:06:42 -04:00
Lex Berezhny e8991e6d52 improve account locking/unlocing CLI docs 2019-04-22 14:06:42 -04:00
Lex Berezhny 13bd81c2b6 v0.36.0rc4 2019-04-21 23:11:23 -04:00
Victor Shyba 21241b98a4 workaround for having new encoder fields on resolve so app can use them 2019-04-21 22:51:48 -04:00
Lex Berezhny bf467e06c5 added create stream image detection test 2019-04-21 22:12:02 -04:00
Lex Berezhny 886ed6415a added examples which include all metadata passed to create stream and create channel 2019-04-21 21:34:13 -04:00
Lex Berezhny 97db241109 v0.36.0rc3 2019-04-21 20:20:53 -04:00
Lex Berezhny 216379918b renamed txo sub_type to value_type in txo json encoder output 2019-04-21 17:40:40 -04:00
Lex Berezhny fcab4eb9bd make sure to use python3 when running set_build.py 2019-04-21 16:31:46 -04:00
Lex Berezhny 19f300cc45 added protobuf to get and file_list commands 2019-04-21 14:28:43 -04:00
Lex Berezhny 26fec098e4 fix stream unit tests 2019-04-21 14:28:43 -04:00
Lex Berezhny 7824de655e updated stream manager to pull protobuf instead of hex from resolve response 2019-04-21 14:28:43 -04:00
Lex Berezhny 6ef1b1dd59 resolve command also returns protobuf 2019-04-21 14:28:43 -04:00
Lex Berezhny 9fae8f5153 added --include_protobuf to add hexlified protobuf to RPC responses 2019-04-21 14:28:43 -04:00
Lex Berezhny f660691e78 v0.36.0rc2 2019-04-21 00:50:27 -04:00
Lex Berezhny bf60683ba6 update to protobuf change where ClaimList defaults to collection list type 2019-04-21 00:45:51 -04:00
Lex Berezhny 68215699f7 fix chri45 test 2019-04-21 00:45:51 -04:00
Lex Berezhny 1644ffbf22 regenerated docs 2019-04-21 00:45:51 -04:00
Lex Berezhny 71f5061848 exposed new protobuf fields in APIs and updated tests 2019-04-21 00:45:51 -04:00
Lex Berezhny 61cadb5cbe v0.36.0rc1 2019-04-20 02:34:27 -04:00
Lex Berezhny 2034104cfc test fixes 2019-04-20 02:33:41 -04:00
Lex Berezhny 633f725634 stream manager test fix 2019-04-20 02:33:41 -04:00
Lex Berezhny 8bf2e41c95 --contact_email -> --email and --homepage_url -> --website_url 2019-04-20 02:33:41 -04:00
Lex Berezhny fa9a4a75c6 update to an even newer protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny 90bef98bc3 use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 35c43647aa use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 449a583286 default to claimType == 1 when it is missing for old claim protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny b1f813782a regenerated protobuf python code 2019-04-20 02:33:41 -04:00
Lex Berezhny 60080cbd19 updating to new protobufs 2019-04-20 02:33:41 -04:00
Victor Shyba 665dba62c7 simplify save from resolve, deduplicate items 2019-04-20 01:19:40 -04:00
Victor Shyba 76dea9e89b batch save supports 2019-04-20 01:19:40 -04:00
Victor Shyba ebc9ed01c5 make lru_cache a power of two, following py docs 2019-04-20 01:19:40 -04:00
Victor Shyba 55116763e9 cache transactions 2019-04-20 01:19:40 -04:00
Victor Shyba e1a6b4e3d6 fix dict handling mistake 2019-04-20 01:19:40 -04:00
Victor Shyba 911ca8c37e cache and share txs fetches while validating concurrently on batches 2019-04-20 01:19:40 -04:00
jessop 7f5deaf6c8 adds api examples to docs 2019-04-16 14:18:23 -04:00
Lex Berezhny 0200756218 put generated commands in groups 2019-04-12 14:24:16 -04:00
ポール ウェッブ a63442f843 Fixes and updated gitignore 2019-04-12 11:53:37 -04:00
Oleg Silkin 74b3da79aa Adds integration tests 2019-04-07 15:27:59 -04:00
Oleg Silkin b6a4589da5 Adds comment group to lbrynet with commands list and create 2019-04-07 15:27:59 -04:00
Oleg Silkin fbbb0a9931 Adds server URL to configuration settings 2019-04-07 15:27:59 -04:00
Oleg Silkin a94d65d522 Adds utility functions for making batch jsonrpc requests to comment server 2019-04-07 15:27:59 -04:00
Oleg Silkin fa2ead002e Fixes incompatible attrs issue when running pip install -e . 2019-04-06 18:43:34 -04:00
Thomas Zarebczan 8dfee4867a Update python3 > python, add mac instructions 2019-04-06 17:10:32 -04:00
Thomas Zarebczan 6614ab8e79 updated with new venv command 2019-04-06 17:10:32 -04:00
Thomas Zarebczan d32c1dfbb3 update install for windowz
Didn't need to install any build tools
Also, venv commands are a bit different.
2019-04-06 17:10:32 -04:00
Lex Berezhny 9f42b958fa run examples fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 574a912f7d run examples as part of ci 2019-04-06 17:08:15 -04:00
Lex Berezhny 2cfcc3cd45 test fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 9af625e627 test fixes 2019-04-06 17:08:15 -04:00
Lex Berezhny 1871c94003 new examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 93220c3874 daemon json rpc return type docs 2019-04-06 17:08:15 -04:00
Lex Berezhny 362ed3f1c8 remove unused import 2019-04-06 17:08:15 -04:00
Lex Berezhny 62a182b8bb automated example generator 2019-04-06 17:08:15 -04:00
Lex Berezhny 6f76d9e24a moved integration testcase.py into lbrynet so that it can be used from scripts 2019-04-06 17:08:15 -04:00
Lex Berezhny 55e83fe05c regenerate api.json 2019-04-06 17:08:15 -04:00
Lex Berezhny 2f0c7d6a3f added title to examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 59e1df9efa generate api examples 2019-04-06 17:08:15 -04:00
Victor Shyba fd56dc68cd fix depth on resolve, add tests 2019-04-05 09:40:26 -04:00
Victor Shyba a0a810edc3 make permanent url name#claim_id on torba too 2019-04-05 09:40:26 -04:00
Victor Shyba c1d3b5984d fix old hash reverse confusion 2019-04-05 09:40:26 -04:00
Victor Shyba 1fb719dd61 fix valid_at_height not being shown for name resolutions, which fixes tests 2019-04-05 09:40:26 -04:00
Victor Shyba b87316fb02 make permanent url name#id only for now 2019-04-05 09:40:26 -04:00
Victor Shyba 65f3d78084 fix invalid signatures for missing certificates 2019-04-05 09:40:26 -04:00
Victor Shyba 47bc1526f7 test invalid signature to be False, not accepting None 2019-04-05 09:40:26 -04:00
Victor Shyba 3acabec4bb fixes from review 2019-04-03 00:51:05 -04:00
Victor Shyba 36a62d3756 port orchstr8 fix to our test case as its lbrynet specific 2019-04-03 00:51:05 -04:00
Victor Shyba 2e22473ce3 pylint and smaller fixes 2019-04-03 00:51:05 -04:00
Victor Shyba 0b0fd64642 further refactor resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba ca17d8597d move resolve from ledger to resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba d16e2716c8 handle invalid data 2019-04-03 00:51:05 -04:00
Victor Shyba 89c80cedfd test resolve for invalid data 2019-04-03 00:51:05 -04:00
Thomas Zarebczan 021e30ffde fix: extension check
disregard the first period, otherwise the `filename.` test fails.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan eb2cd8fec1 stream type mapping
Mapping extensions to stream types. I went with document / model / binary for the ones we were missing. This can be adjusted later on if not correct, but at least we'll have the main ones. Added extensions for lbry, cbz, cbr also.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan 6356818557 fix: time to first byte script
Fix with for metadata
2019-04-01 16:14:53 -04:00
YULIUS KURNIAWAN KRISTIANTO 8bf724a78b Update CONTRIBUTING.md 2019-04-01 16:12:49 -04:00
YULIUS KURNIAWAN KRISTIANTO 65d0a924c1 Update README.md 2019-04-01 16:12:49 -04:00
Lex Berezhny ef7bd00f22 round USD fees to nearest penny 2019-03-31 18:14:13 -04:00
Lex Berezhny 3c1fdb5114 v0.35.0 2019-03-29 21:51:05 -04:00
Lex Berezhny 83411acfbd split claim_abandon into stream_abandon and channel_abandon 2019-03-29 21:48:55 -04:00
Lex Berezhny 4230812f82 added integration test to make sure publishing the same stream re-signs it with the original channel 2019-03-29 21:14:20 -04:00
Lex Berezhny 7b57ba96dd fix docopt parsing and stream manager tests 2019-03-29 21:14:20 -04:00
Lex Berezhny 668840921f typo fix 2019-03-29 21:14:20 -04:00
Lex Berezhny a3bd335f7f --languages, --locations fully support all attributes, add BTC 2019-03-29 21:14:20 -04:00
Lex Berezhny 4e6bca1389 support_abandon sends new --keep support to same address as the first support being abandoned 2019-03-29 21:14:20 -04:00
Lex Berezhny c4b15920ff dont show supports to foreign address if the inputs are also foreign 2019-03-29 21:14:20 -04:00
Lex Berezhny 2a55e583e7 added to docstring that a channel_id is a channel claim id 2019-03-29 21:14:20 -04:00
Lex Berezhny 2c4fde06c5 --new-signing-key -> --new_signing_key 2019-03-29 21:14:20 -04:00
Lex Berezhny eca83d7f47 --clear-tags -> --clear_tags 2019-03-29 21:14:20 -04:00
Lex Berezhny 87e818ac39 v0.35.0rc2 2019-03-27 22:20:09 -04:00
Lex Berezhny adab7bd7b1 fix file command tests 2019-03-27 21:57:22 -04:00
Lex Berezhny 750c0e4f4f fix cli test 2019-03-27 21:57:22 -04:00
Lex Berezhny 5f8482bdb7 hash -> sd_hash, encoding fixes, update to new protobufs 2019-03-27 21:57:22 -04:00
Lex Berezhny cd266477fb stop regenerating v1 protobufs 2019-03-27 21:57:22 -04:00
Lex Berezhny 7860b956ff fixes and tests for publish command 2019-03-27 21:57:22 -04:00
Thomas Zarebczan 148481a5ca doh 2019-03-27 21:54:19 -04:00
Thomas Zarebczan 93faa276a5 update BIP32 2019-03-27 21:54:19 -04:00
Victor Shyba 53e67a6e81 fixes from rebase 2019-03-27 10:31:37 -04:00
Victor Shyba 3ad6808d87 remove unused claim_id_hash 2019-03-27 10:31:37 -04:00
Victor Shyba 071ab32ab7 fix hash references 2019-03-27 10:31:37 -04:00
Victor Shyba ab87588250 use torba parsing for addresses too, erase old code 2019-03-27 10:31:37 -04:00
Victor Shyba 22db29ee1d use torba tx parsing on wallet server 2019-03-27 10:31:37 -04:00
Lex Berezhny 66bf6c61f0 v0.35.0rc1 2019-03-25 23:57:59 -04:00
Lex Berezhny 9abf40d21f stream_type fix 2019-03-25 23:45:10 -04:00
Lex Berezhny ea4444edbe fix internal transaction api test 2019-03-25 22:54:08 -04:00
Lex Berezhny 51bb8fd283 pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 183e3ac1c7 claim search 2019-03-25 22:54:08 -04:00
Lex Berezhny 8087b457c6 reorganized commands and finished support_abandon with --keep 2019-03-25 22:54:08 -04:00
Lex Berezhny fedb7c23d0 bring back blocks_behind in wallet status 2019-03-25 22:54:08 -04:00
Lex Berezhny e2964f9728 test cli fix, take 2 2019-03-25 22:54:08 -04:00
Lex Berezhny e0cea60c87 fix cli test 2019-03-25 22:54:08 -04:00
Lex Berezhny c708c1db07 audio/video/image stream types 2019-03-25 22:54:08 -04:00
Lex Berezhny 1cb696fb5c fix test_claim_show 2019-03-25 22:54:08 -04:00
Lex Berezhny 7cd0213eb9 fixed chris45 again 2019-03-25 22:54:08 -04:00
Lex Berezhny 2007c48af3 fixed tests for setting claim values 2019-03-25 22:54:08 -04:00
Lex Berezhny 50721786d2 skip test_claim_list for now 2019-03-25 22:54:08 -04:00
Lex Berezhny 6d7eb259cb test normalization fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny cc5852a9b9 update pyyaml due to security alert 2019-03-25 22:54:08 -04:00
Lex Berezhny 0592cee567 claim integration test fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny 814eaca524 cli tests fixed 2019-03-25 22:54:08 -04:00
Lex Berezhny e06916f5ae fixing unit tests 2019-03-25 22:54:08 -04:00
Victor Shyba d61e1e9d8c fix proto v1 integration test 2019-03-25 22:54:08 -04:00
Lex Berezhny 25436893bc pylint and a few other fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny 5b9d41fd5c regeneate script after modifying protobufs 2019-03-25 22:54:08 -04:00
Victor Shyba b1720d1659 fix setting sd hash 2019-03-25 22:54:08 -04:00
Lex Berezhny 4b3a6ab875 big support tests passes 2019-03-25 22:54:08 -04:00
Lex Berezhny 67dcede2d8 lots of stuff deleted 2019-03-25 22:54:08 -04:00
Victor Shyba 0872a6fea3 partial test_claim_list_by_channel 2019-03-25 22:54:08 -04:00
Victor Shyba a3aed3219f fix test_abandoned_channel_with_signed_claims 2019-03-25 22:54:08 -04:00
Lex Berezhny af73e1a2f5 chris45 passing again 2019-03-25 22:54:08 -04:00
Lex Berezhny 55bbf5aa74 progress on chris45 2019-03-25 22:54:08 -04:00
Lex Berezhny c1fbb02bfe dont validate inputs in json_response_encoder 2019-03-25 22:54:08 -04:00
Lex Berezhny d47575e8e0 progress 2019-03-25 22:54:08 -04:00
Victor Shyba 8fc4f4aa73 cleanup recent test changes 2019-03-25 22:54:08 -04:00
Victor Shyba fe9aa666e9 fix name is not part of old sigs 2019-03-25 22:54:08 -04:00
Victor Shyba a9e0eeba7e add tests and fix verification of old signatures 2019-03-25 22:54:08 -04:00
Victor Shyba 0830917afb delete unused test method 2019-03-25 22:54:08 -04:00
Victor Shyba 1ed086e90f bring back test for picking oldest on dupe claims 2019-03-25 22:54:08 -04:00
Victor Shyba 56db631626 fix signed channel tests, WIP on tampering 2019-03-25 22:54:08 -04:00
Victor Shyba aa029955ee fix tx api tests 2019-03-25 22:54:08 -04:00
Victor Shyba f2236eb7a4 resolve: check if there is a cert before fetching a tx 2019-03-25 22:54:08 -04:00
Lex Berezhny df649341cd json output formatting fix for Claim 2019-03-25 22:54:08 -04:00
Lex Berezhny a3c3143868 fixed remaining references to lbrynet.extras.wallet -> lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny d0201fe7c7 base integration test class should default to VERBOSITY=WARN, not DEBUG 2019-03-25 22:54:08 -04:00
Lex Berezhny 664a2a324d fix import error on stream unit test 2019-03-25 22:54:08 -04:00
Lex Berezhny ea09aca175 pylint fixes and ClaimDict->Claim 2019-03-25 22:54:08 -04:00
Lex Berezhny ca92fddecf pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 8259d7c680 txo.claim_hash and other fixes related to proto3 refactor 2019-03-25 22:54:08 -04:00
Victor Shyba 6653da2973 all 2019-03-25 22:54:08 -04:00
Lex Berezhny e74998209d for victor 2019-03-25 22:54:08 -04:00
Lex Berezhny 2bea0f697a pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 1d68bef6f2 client side pretty much done 2019-03-25 22:54:08 -04:00
Lex Berezhny 22bacc907b moved test_channel_signing.py from tests/unit/schema to tests/unit/wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 1e8ad900c7 renamed test_signing.py to test_channel_signing.py 2019-03-25 22:54:08 -04:00
Lex Berezhny cd2b535afb moved claim signing to TXO 2019-03-25 22:54:08 -04:00
Lex Berezhny 9dfe7bbcf6 imports updated after moving lbrynet.extras.wallet to lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 120618ffd5 updated imports due to lbrynet.extras.wallet -> lbrynet.wallet move 2019-03-25 22:54:08 -04:00
Lex Berezhny e8abcbb06e moved lbrynet.extras.wallet -> lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 1ec8f0b0b4 wip 2019-03-25 22:54:08 -04:00
Lex Berezhny 323694fb12 removed address.py 2019-03-25 22:54:08 -04:00
Lex Berezhny d5e37acd8f minor Fee changes 2019-03-25 22:54:08 -04:00
Lex Berezhny 129a886f2b + fee.amount 2019-03-25 22:54:08 -04:00
Lex Berezhny cd15230a92 getting close to a nice model 2019-03-25 22:54:08 -04:00
Lex Berezhny c1b7fdc68e deleted bunch of stuff 2019-03-25 22:54:08 -04:00
Lex Berezhny 9fd67aaad6 removed legacy_schema_v1 2019-03-25 22:54:08 -04:00
Lex Berezhny ce9dc1836e moved schema/current_schema/claim.py into schema/claim.py 2019-03-25 22:54:08 -04:00
Lex Berezhny 2e557d3a36 converted everyting in schema/legacy/* into schema/jsonschema.py 2019-03-25 22:54:08 -04:00
Lex Berezhny 5df05cb37e make schema.types a module 2019-03-25 22:54:08 -04:00
Lex Berezhny 326b62dc57 moved proto2/proto3 to types/v1 and types/v2 respectively 2019-03-25 22:54:08 -04:00
Lex Berezhny 25cc7ba284 updated schema to latest types: Claim uses oneof type for Channel/Stream 2019-03-25 22:54:08 -04:00
Victor Shyba 550f064302 baseschema 2019-03-25 22:54:08 -04:00
Victor Shyba 11086b93f1 migrate other fields 2019-03-25 22:54:08 -04:00
Victor Shyba df5662136d migrate cert + adapt to latest types repo 2019-03-25 22:54:08 -04:00
Victor Shyba 291d65070d move common schema values to constants.py 2019-03-25 22:54:08 -04:00
Victor Shyba 9b64ee33dc fix imports: schema.schema -> schema.legacy_schema_v1 2019-03-25 22:54:08 -04:00
Victor Shyba 5404d26f2f schema.schema -> schema.legacy_schema_v1 2019-03-25 22:54:08 -04:00
Victor Shyba 26ade1f228 bump merged proto3 2019-03-25 22:54:08 -04:00
Victor Shyba b49f1f1b54 fix imports 2019-03-25 22:54:08 -04:00
Victor Shyba 3bafa2ad22 regenerate protos 2019-03-25 22:54:08 -04:00
Victor Shyba 378ec44f3c update make to generate proto2 and proto3 2019-03-25 22:54:08 -04:00
Victor Shyba 9d0d17b422 rename proto proto2 2019-03-25 22:54:08 -04:00
Victor Shyba f7c8145283 fix travis not to build wallet on PR 2019-03-19 12:22:00 -04:00
Lex Berezhny 0de17730ca v0.34.0 2019-03-15 16:21:29 -04:00
Jack Robison 58743ba19b Merge pull request #1987 from lbryio/more-dht-tests
improve dht unit tests
2019-03-15 15:38:51 -04:00
Jack Robison f2fefbe287 add lbrynet.dht.blob_announcer unit tests 2019-03-15 14:29:02 -04:00
Jack Robison 60a24f0e6e add lbrynet.dht.protocol.data_store unit tests 2019-03-15 12:44:55 -04:00
Jack Robison 664f91bfab add lbrynet.dht.protocol.distance unit tests 2019-03-15 12:44:41 -04:00
Jack Robison 44f92a271f move dht tests to reflect the real directory structure 2019-03-15 12:01:21 -04:00
Jack Robison 6565ca8558 improve lbrynet.dht.serialization unit tests 2019-03-14 18:45:56 -04:00
Jack Robison 3b0ba3e534 v0.34.0rc2 2019-03-14 16:03:39 -04:00
Jack Robison 8106eb1fa5 Merge pull request #1986 from lbryio/fix-timeout-no-peers
fix default download timeout not being set
2019-03-14 15:47:12 -04:00
Jack Robison c9b2829227 fix/test fixed_peer_delay and added_fixed_peers 2019-03-14 15:08:26 -04:00
Jack Robison 4f2f8b16f3 fix pending downloads not being removed 2019-03-14 13:52:00 -04:00
Jack Robison 2ab2b846b1 fix default download timeout not being set 2019-03-14 13:07:11 -04:00
Jack Robison 2df7284c29 Merge pull request #1985 from lbryio/fix-ttfb-analytics
fix the time to the head blob including the time to the sd blob in analytics
2019-03-14 11:17:37 -04:00
Jack Robison ab5d71ff61 fix the time to the head blob including the time to the sd blob in analytics 2019-03-13 16:24:23 -04:00
Lex Berezhny 99b289f135 v0.34.0rc1 2019-03-12 20:29:10 -04:00
Lex Berezhny 6db02e4362 show build type setting when building 2019-03-12 20:23:10 -04:00
Lex Berezhny 98d8b7d463 sync uses new apply APIs in torba 2019-03-12 20:22:41 -04:00
Lex Berezhny 96f1883fc4 check modified_on 2019-03-12 20:22:41 -04:00
Lex Berezhny a914de155a working sync 2019-03-12 20:22:41 -04:00
Lex Berezhny 2dee0ff0bf wip account manifest command 2019-03-12 20:22:41 -04:00
Jack Robison 993cb43e5b fix tests
-fix cleanup of a failed download
2019-03-12 20:22:01 -04:00
Jack Robison 5d212a0f82 time to first bytes analytics 2019-03-12 20:22:01 -04:00
Jack Robison 0cc602ca53 Merge pull request #1978 from lbryio/sigint
handle sigint
2019-03-12 15:06:08 -04:00
Victor Shyba f390ff144d handle sigint 2019-03-12 13:43:12 -04:00
Jack Robison b266cde511 Merge pull request #1980 from lbryio/batched-channel-name-initialization
use _batched_select to populate channel names when initializing files
2019-03-12 13:41:58 -04:00
Jack Robison d0a1541d0a fix test 2019-03-12 13:00:43 -04:00
Jack Robison 4cbde1b08f use _batched_select to populate channel names when initializing files 2019-03-12 09:58:19 -04:00
Lex Berezhny c6740d8d70 v0.33.0 2019-03-08 20:00:45 -05:00
Lex Berezhny 46358142ed v0.33.0rc1 2019-03-06 22:18:04 -05:00
Lex Berezhny f0f651fc77 rely on torba cffi explicit version 2019-03-05 09:32:43 -05:00
Lex Berezhny 5d428138ed try older cffi on windows build 2019-03-05 09:32:43 -05:00
Brannon King 8d9da61c7d reverted temporary code for full local stack from 2019-03-05 09:32:43 -05:00
Brannon King c326a9382c changed to pull down the newer lbrycrd 2019-03-05 09:32:43 -05:00
Brannon King 5bdd87c904 removed name-to-claim lookup table
added unit test for Greek and made it work


pylint: revert bad move


revert claim_sequence changes


fixed broken test
2019-03-05 09:32:43 -05:00
Jack Robison ad134fe8bb v0.32.4 2019-03-01 17:24:55 -05:00
Jack Robison 2f3c616acd v0.32.4rc1 2019-03-01 16:33:59 -05:00
Jack Robison 3caa2a5039 Merge pull request #1968 from lbryio/analytics
restore old download analytics
2019-03-01 16:08:18 -05:00
Jack Robison e38a0f7a95 logging 2019-03-01 15:26:45 -05:00
Jack Robison 9de3f3ff09 don't block on sending download analytics events 2019-03-01 15:22:19 -05:00
Jack Robison df8934d472 restore old download analytics and update tests 2019-03-01 14:50:06 -05:00
Jack Robison be4a96c1b3 Merge pull request #1965 from lbryio/certifi
fix ssl errors, use certifi to make ssl contexts
2019-02-28 13:14:17 -05:00
Jack Robison d97d978961 use certifi for all operating systems 2019-02-28 12:45:56 -05:00
Jack Robison f52e44b56a typing 2019-02-28 12:42:23 -05:00
Jack Robison 10d73fe480 use certifi 2019-02-28 12:40:11 -05:00
Jack Robison 4fd2d994f2 Merge pull request #1964 from lbryio/update-aiohttp
update aiohttp and pylint
2019-02-28 12:38:17 -05:00
Jack Robison d95903d219 update aiohttp and pylint 2019-02-28 12:04:07 -05:00
Jack Robison 591e66bfbb Merge pull request #1952 from lbryio/no_trap
stop trapping CancelledError
2019-02-27 11:06:02 -05:00
Victor Shyba e047748961 log dropped api request instead of stack trace 2019-02-26 17:42:45 -05:00
Victor Shyba 9f071bbe36 pylint 2019-02-26 17:42:45 -05:00
Victor Shyba 2d8ebe25ed refactor so we can stop trapping CancelledError everywhere 2019-02-26 17:42:45 -05:00
Victor Shyba 7a96e742f2 stop trapping CancelledError 2019-02-26 17:42:45 -05:00
Jack Robison 13c7026b20 Merge pull request #1961 from lbryio/pin-requirements
Pin requirements
2019-02-26 17:42:13 -05:00
Jack Robison 876a3c68dd pin requirements 2019-02-26 17:20:16 -05:00
Jack Robison 573617d6b6 Merge pull request #1957 from lbryio/dht_memory
add LRU to KademliaPeer creation
2019-02-26 10:10:33 -05:00
Victor Shyba e8c9abb944 add LRU to KademliaPeer creation 2019-02-26 09:42:32 -05:00
Jack Robison 8cee627dfa Merge pull request #1959 from lbryio/pin-jsonschema
pin jsonschema version to 2.6.0
2019-02-26 09:41:20 -05:00
Jack Robison ab4a5cee01 pin jsonschema version 2019-02-26 09:23:41 -05:00
Lex Berezhny 5bb0a07fae pip with git does not appear to work in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 5ca594b9fd upgrade pip in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 3910964df1 plyvel no longer installed unless platform is explicitely linux in torba/setup.py 2019-02-22 20:52:42 -05:00
Jack Robison faa1935203 Merge pull request #1946 from lbryio/file-type-fix
lowercase/stripped comparison for known types
2019-02-22 13:07:09 -05:00
Jack Robison ba518e2a0e update test 2019-02-22 11:12:31 -05:00
Thomas Zarebczan 2a82ff7cc5 Update mime_types.py 2019-02-22 11:11:10 -05:00
Jack Robison 673b958020 v0.32.3 2019-02-21 19:38:14 -05:00
Jack Robison 3f4ed0a896 Merge pull request #1951 from lbryio/fix_invalid_blob_decode_startup
Fix invalid blob decode startup
2019-02-21 19:36:02 -05:00
Victor Shyba f49578cdbc fix integration test 2019-02-21 21:14:51 -03:00
Victor Shyba b834f29634 nullify length on blob delete 2019-02-21 21:00:28 -03:00
Victor Shyba 61132312dd test nullify length on blob delete 2019-02-21 21:00:20 -03:00
Victor Shyba fcdd9295b4 delete upon failing to decode JSON from sd blob 2019-02-21 20:53:21 -03:00
Victor Shyba edb5083a5e test that corrupted sd blobs gets delete if fails to decode 2019-02-21 20:53:04 -03:00
Victor Shyba 4e7d88311f raise proper exception on undecode-able blobs 2019-02-21 20:42:51 -03:00
Victor Shyba aa40337ded test that invalid blobs raise proper exception 2019-02-21 20:42:36 -03:00
Jack Robison 690e2e7e0c v0.32.3rc1 2019-02-21 18:37:19 -05:00
Jack Robison 0c67469321 Merge pull request #1950 from lbryio/fix-slow-ping-queue-shutdown
Fix slow ping queue shutdown
2019-02-21 18:12:22 -05:00
Jack Robison fc64d8d1a8 fix slow ping queue shutdown 2019-02-21 17:44:02 -05:00
Jack Robison 7fe9513857 v0.32.2 2019-02-21 14:46:36 -05:00
Jack Robison e76107c318 v0.32.2rc3 2019-02-21 14:28:13 -05:00
Jack Robison 3868dea819 don't join buckets if there is only one 2019-02-21 14:18:48 -05:00
Jack Robison c52541197f Merge pull request #1949 from lbryio/fix-join-buckets
fix cases where buckets could be unjoined after removing a peer
2019-02-21 14:08:34 -05:00
Jack Robison 03cd6b7200 fix cases where buckets could be unjoined after removing a peer 2019-02-21 13:53:35 -05:00
Jack Robison 6915a7a553 v0.32.2rc2 2019-02-21 13:16:32 -05:00
Jack Robison 7ad29b4bfb Merge pull request #1948 from lbryio/duplicates-in-routing
Test and fix edge cases for adding peers to routing
2019-02-21 13:11:34 -05:00
Jack Robison 4294bf742d simplify/fix ping queue 2019-02-21 13:00:23 -05:00
Jack Robison 57ad9f1c52 test_add_peer_after_handle_request 2019-02-21 11:34:25 -05:00
Jack Robison e260de59f1 fix updating/setting contact triple 2019-02-21 10:50:11 -05:00
Jack Robison 154702e6dd fix KBucket.add_peer 2019-02-21 10:50:11 -05:00
Jack Robison d06d998c81 improve KBucket test_add_peer 2019-02-21 10:50:11 -05:00
Jack Robison 8aff860bfe Merge pull request #1945 from lbryio/fix_delete_log
Fix delete log
2019-02-21 10:14:29 -05:00
Victor Shyba d7ba90bbea update ttfb script 2019-02-20 17:10:36 -03:00
Victor Shyba 59325ad8c1 fix delete log 2019-02-20 17:04:08 -03:00
Jack Robison 1f72e27272 improve time_to_first_byte.py 2019-02-20 17:04:08 -03:00
Jack Robison aa41b8b558 Merge pull request #1944 from lbryio/fix-seed-node-startup
fix seed node startup
2019-02-20 15:03:47 -05:00
Jack Robison 2496e7ed44 improve time_to_first_byte.py 2019-02-20 14:43:40 -05:00
Jack Robison ee12985135 dont ping outselves 2019-02-20 14:15:31 -05:00
Jack Robison 4c7f357441 v0.32.2rc1 2019-02-20 12:12:04 -05:00
Jack Robison 8944a36291 Merge pull request #1942 from lbryio/split_buckets_under_index
add split_buckets_under_index config setting for seed nodes
2019-02-20 12:00:46 -05:00
Jack Robison cb5d428d12 add split_buckets_under_index config setting for seed nodes 2019-02-20 11:25:50 -05:00
Jack Robison f175e8826c Merge pull request #1939 from lbryio/missing_await
refactor file_name and full_path as it can be None, fix missing await, stop using blob_manager.storage on stream_manager and use your own
2019-02-20 10:44:35 -05:00
Victor Shyba 38ed212c01 refactor, dont call storage from blob_manager + missing await 2019-02-19 21:45:14 -03:00
Victor Shyba d035c13883 file_path can be None, add helper to check if stream output file exists instead 2019-02-19 21:45:14 -03:00
Jack Robison 82ce2ccee6 update docs 2019-02-19 17:27:31 -05:00
Jack Robison 2cf9b2dd9e Merge pull request #1938 from lbryio/fix-kbucket-indexing
fix KBucket.key_in_range
2019-02-19 15:03:16 -05:00
Jack Robison fcbdaa336f fix KBucket.key_in_range
The range_min and range_max of a KBucket where mistakenly bounding the node id of a peer to be added rather than the distance from the local node id to the peer node id
2019-02-19 13:57:11 -05:00
Lex Berezhny 3ae6137edb v0.32.1 2019-02-18 22:56:35 -05:00
Lex Berezhny 0fdd2ae178 added tx to JSON response from get 2019-02-18 22:53:11 -05:00
Victor Shyba 8c5007fe07 assert amount transferred to correct account 2019-02-18 22:53:11 -05:00
Victor Shyba d2150cf142 fix balance check 2019-02-18 22:53:11 -05:00
Victor Shyba ce2bb22929 add tests for paid downloads 2019-02-18 22:53:11 -05:00
Lex Berezhny 1ce3adb6a0 reduce integration test verbosity 2019-02-18 21:45:37 -05:00
Lex Berezhny c2dc1f3c1d v0.32.1rc1 2019-02-18 17:17:20 -05:00
Victor Shyba b36c22e2f4 unban after 10 seconds, give up after 60 2019-02-18 17:16:10 -05:00
Victor Shyba 71b66ab337 test stall unban 2019-02-18 17:16:10 -05:00
Lex Berezhny 461c3c8408 downloading header from S3 correctly tracks progress 2019-02-18 17:15:40 -05:00
Thomas Zarebczan 03396c1993 remove invalid reflector_component test config 2019-02-18 17:10:42 -05:00
Jack Robison fa7e9923f4 Merge pull request #1932 from lbryio/fix-1930
Fix blob_list for sd_hash including stream terminator
2019-02-18 14:41:00 -05:00
Jack Robison 3f7e466c97 return failure message for invalid blob hash in blob_delete 2019-02-18 12:18:56 -05:00
Jack Robison 4483380352 fix terminator blob hash (None) being included when running blob_list for an sd hash 2019-02-18 12:12:14 -05:00
Lex Berezhny 4a0a6b0243 v0.32.0 2019-02-16 20:17:41 -05:00
Thomas Zarebczan a003e794ba listen on internal port instead of upnp redirect 2019-02-16 18:55:58 -05:00
Lex Berezhny 051fabc180 v0.32.0rc3 2019-02-16 15:07:07 -05:00
Jack Robison ca686041e7 Merge pull request #1925 from lbryio/fix-extension
fix application/x-ext mime types
2019-02-15 21:11:05 -05:00
Jack Robison fc450eb164 add TestMimeTypes, pylint 2019-02-15 20:42:55 -05:00
Thomas Zarebczan 652d7cbd24 consider . extension 2019-02-15 20:40:55 -05:00
Thomas Zarebczan 43b372c0fe fix extension - remove dot
The rest of our extensions don't have a dot. Also, strip off any characters and lowercase it to be safe.
2019-02-15 20:40:55 -05:00
Jack Robison eaacf7e034 Merge pull request #1926 from lbryio/fix_unfair_ban
Fix p2p ignoring good peers
2019-02-15 20:39:49 -05:00
Victor Shyba 576da1bc30 do not keep trying when blob is being written 2019-02-15 21:20:06 -03:00
Victor Shyba 5c217f1544 resovle reflector IP before adding it 2019-02-15 21:20:06 -03:00
Jack Robison cfbf0dbe0f Merge pull request #1919 from lbryio/sync-blobs-on-startup
Sync blobs on startup
2019-02-15 18:40:33 -05:00
Jack Robison fd081496c0 fix blocking on data being written for resumed streams 2019-02-15 17:35:46 -05:00
Jack Robison 32b4405a4c logging
fix https://github.com/lbryio/lbry/issues/1881
2019-02-15 17:26:28 -05:00
Jack Robison 0d558bd552 fix default file list sorting 2019-02-15 17:26:24 -05:00
Jack Robison ca835f3f80 fix 1883 2019-02-15 16:26:13 -05:00
Jack Robison dbb6ba6241 recover streams with missing sd blobs, handle previous sd blob bugs
-test download and recover stream with old key sorting
2019-02-15 15:21:29 -05:00
Jack Robison a228d20137 move store_stream and store_file to standalone functions 2019-02-15 15:21:07 -05:00
Jack Robison f1b60e3ef2 fix get_blobs_for_stream 2019-02-15 15:21:07 -05:00
Jack Robison 39737c790f sync blob files in database with those in directory on startup 2019-02-15 15:21:07 -05:00
Jack Robison dba4a09d38 Merge pull request #1922 from lbryio/supress_invalid_token_error_to_debug
supress invalid token error to debug
2019-02-15 15:19:58 -05:00
Victor Shyba 918122e7cf supress invalid token error to debug 2019-02-15 16:31:47 -03:00
Victor Shyba a49f479a5f move wait to where its needed on assembler 2019-02-15 10:26:05 -05:00
Victor Shyba 320aabbd70 delete duplicated method accident from rebase 2019-02-15 10:26:05 -05:00
Jack Robison 19fdcad5af Merge pull request #1905 from lbryio/faster-file-delete
Faster file delete
2019-02-14 17:23:55 -05:00
Jack Robison 1734e08297 fix test 2019-02-14 16:34:37 -05:00
Jack Robison 9d89f9b834 use executemany instead of batched_operation 2019-02-14 15:57:24 -05:00
Jack Robison 56c41b2fea fix integration test
-cancel running reflector uploads from publishes upon shutdown
-refactor blob.delete to be non-async
-delete blobs synchronously
2019-02-14 15:43:26 -05:00
Jack Robison 9c8593d88e fix file_delete being slow 2019-02-14 12:33:48 -05:00
Jack Robison 85f41887fd don't create BlobFile object when deleting if it doesn't already exist 2019-02-14 12:28:42 -05:00
Mark Michon 8eede6dfc7 Update LICENSE
Updating license to 2019
2019-02-14 08:47:26 -05:00
Victor Shyba 0d8c37a63b resolve uri->url on ttfb 2019-02-14 00:30:24 -05:00
Lex Berezhny d2ba3fdcfb v0.32.0rc2 2019-02-14 00:18:21 -05:00
Victor Shyba be5ab0f307 test get retry 2019-02-14 00:10:44 -05:00
Victor Shyba 04f70d1f31 stop when data timeout 2019-02-14 00:10:44 -05:00
Victor Shyba 60ba07cc17 add tests for incomplete files, update existing one 2019-02-14 00:10:44 -05:00
Victor Shyba dbc8cfe5ae delete incomplete files on assembler stop 2019-02-14 00:10:44 -05:00
Victor Shyba d91e8f5996 revert import to original one + remove print 2019-02-13 23:33:49 -05:00
Victor Shyba b6df2b9d28 missing None for optional field + pylint 2019-02-13 23:33:49 -05:00
Victor Shyba 5c1ab42277 resume should put the stream into wait as done usually on get calls 2019-02-13 23:33:49 -05:00
Victor Shyba bbdc96e762 file_name as a property of managed stream, to reflect internal changes 2019-02-13 23:33:49 -05:00
Victor Shyba fc0c949187 properly update file name on downloader/assembler 2019-02-13 23:33:49 -05:00
Victor Shyba 3fce8e4023 tests resuming downloads with conflicting file names 2019-02-13 23:33:49 -05:00
Victor Shyba 37bb765a2e resume downloads if DHT disabled 2019-02-13 23:33:49 -05:00
hackrush 21ca4f60f7 Stop the daemon on calling lbrynet stop 2019-02-13 14:22:23 -05:00
Victor Shyba 119d498bce apply a timeout on test wait so if it hangs it fails fast 2019-02-13 13:10:27 -05:00
Victor Shyba 8108aa39c1 verify all blobs got transferred 2019-02-13 13:10:27 -05:00
Victor Shyba 0a068011d8 fix chunked response parsing 2019-02-13 13:10:27 -05:00
Victor Shyba 1193d2cea4 adds tests for chunked responses 2019-02-13 13:10:27 -05:00
Lex Berezhny 76ecb5d50d renamed resolve argument url to urls 2019-02-13 11:55:12 -05:00
Lex Berezhny dbc1164562 v0.32.0rc1 2019-02-12 21:35:41 -05:00
Victor Shyba e2170ec83d fix download timeout types 2019-02-12 20:58:31 -05:00
Victor Shyba 266e777655 add integration test for different download errors 2019-02-12 20:58:31 -05:00
Lex Berezhny 68905d82b9 pass conf instead of data_dir to migrate_db, fixes #1898 2019-02-12 18:10:34 -05:00
Lex Berezhny 396c49f7c6 passing get, file list and file delete tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 5e509c9fd6 testing get 2019-02-12 11:54:00 -05:00
Lex Berezhny e6b6a3f55e dedicated transaction commands tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 019af119d9 moved test_cli.py up one level 2019-02-12 11:54:00 -05:00
Lex Berezhny 58da2f16c2 use absolute imports instead of relative 2019-02-12 11:54:00 -05:00
Lex Berezhny 8aa50aa14f cleaned up resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny 65c6c83b51 reusable command testcase 2019-02-12 11:54:00 -05:00
Lex Berezhny f497989871 split out claim commands 2019-02-12 11:54:00 -05:00
Lex Berezhny 5366779b1b moved all other tests out of test_chris45.py and into dedicated files 2019-02-12 11:54:00 -05:00
Lex Berezhny a3f59f041a dedicated integration tests for account commands 2019-02-12 11:54:00 -05:00
Lex Berezhny d560b0c1cd dedicated integration tests for resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny e94229121a explicitly set LEDGER 2019-02-12 11:54:00 -05:00
Lex Berezhny 097291a498 moved wallet integration tests up one level to simplify 2019-02-12 11:54:00 -05:00
Jack Robison afdc44ca13 Merge pull request #1894 from lbryio/fix-max-key-fee
fix max_key_fee not being followed
2019-02-11 16:27:29 -05:00
Jack Robison 6732b302bb logging 2019-02-11 15:31:31 -05:00
Jack Robison c9d54bb049 fix max_key_fee 2019-02-11 15:26:13 -05:00
Alex Grintsvayg f65e70df2f better resolve description 2019-02-11 12:34:57 -05:00
Jack Robison 4541e754dd Merge pull request #1892 from lbryio/batch-query-files
Improve startup time
2019-02-08 20:43:52 -05:00
Jack Robison 9f89c62e8a download full streams in time_to_first_byte.py 2019-02-08 20:14:10 -05:00
Jack Robison 6386815a29 fix reflect_streams not being followed 2019-02-08 20:13:26 -05:00
Jack Robison 0c93090d2c faster is_valid_hashcharacter 2019-02-08 19:57:26 -05:00
Jack Robison 45404b6ad4 faster is_valid_blobhash 2019-02-08 19:54:59 -05:00
Jack Robison aeb2891baa faster population of channel names 2019-02-08 19:51:13 -05:00
Jack Robison 8abfdc82b2 faster blob manager startup 2019-02-08 17:37:35 -05:00
Jack Robison da9b4e317a limit blobs to announce 2019-02-08 16:29:55 -05:00
Jack Robison e414cc5c48 whitespace 2019-02-08 16:11:56 -05:00
Jack Robison 939edb2f18 logging
-call loop.set_debug when using --verbose
2019-02-08 16:06:38 -05:00
Jack Robison 868110a6f2 populate stream manager with a single batch query 2019-02-08 16:06:35 -05:00
Jack Robison 869a8b712b Merge pull request #1889 from lbryio/download_stuc
Refactor peer scoring and maintain connections to fix stuck downloads, remove locking from ping queue
2019-02-08 15:10:13 -05:00
Victor Shyba 9ecec52412 check lock acquired to prevent races 2019-02-08 04:39:28 -03:00
Victor Shyba 8b25acff6e do not wait blob to be written, log errors from races 2019-02-08 04:32:53 -03:00
Victor Shyba e2b06677b5 defer db insert during downloads 2019-02-08 03:45:23 -03:00
Victor Shyba af1619ebfb disable race limiting for now 2019-02-08 03:32:38 -03:00
Victor Shyba 3352e0e4f4 score peers by speed 2019-02-08 03:08:08 -03:00
Victor Shyba 141d68a2cd fix write race condition 2019-02-08 02:39:29 -03:00
Victor Shyba c06ec6cd69 manage connections, reusing them when possible 2019-02-08 02:27:58 -03:00
Victor Shyba 1be5dce30e stop racing during long streams 2019-02-08 01:28:03 -03:00
Victor Shyba 73da223ae1 conf: give up first contact earlier, but let the transfer go longer 2019-02-08 01:05:58 -03:00
Victor Shyba 2d7eb83518 change score calculation, wait for active peers too, simplify peer sorting/keeping 2019-02-08 01:04:38 -03:00
Victor Shyba ad03f91d24 let cancelled errors flow 2019-02-08 01:03:13 -03:00
Victor Shyba 5586a226c2 bypass parser during download 2019-02-07 21:19:31 -03:00
Victor Shyba a616582733 wait on connection tasks 2019-02-07 20:11:57 -03:00
Victor Shyba 43ac928f0b remove dht locks 2019-02-07 20:11:57 -03:00
Lex Berezhny a121110743 v0.31.0 2019-02-06 14:28:40 -05:00
Jack Robison 8bc5bb4e14 Merge pull request #1873 from lbryio/fix_incomplete_downloads_assembly
test incomplete streams, respect real length, delete on incomplete as…
2019-02-06 14:26:40 -05:00
Victor Shyba 3a15ff4bcd test incomplete streams, respect real length, delete on incomplete assembly 2019-02-06 15:56:06 -03:00
Jack Robison 3508da4993 Merge pull request #1866 from lbryio/non-async-close-blob
refactor blob.close to be non-async, speed up deleting blobs and streams
2019-02-06 13:52:37 -05:00
Jack Robison dfdc8eda00 improve test, fix stream_finished_event 2019-02-06 13:32:50 -05:00
Jack Robison 58f6cb71c6 test stream manager 2019-02-06 13:08:57 -05:00
Jack Robison ae11c5bb4b fix setting finished event 2019-02-06 12:18:24 -05:00
Jack Robison f22b5da170 fix raised error for a peer address mismatch 2019-02-06 12:11:22 -05:00
Jack Robison c1c6d5bc99 fix deleting partial downloads when stopped and previous streams when updating a publish 2019-02-06 11:43:05 -05:00
Jack Robison 0e972ec2ae refactor BlobFile.close to be non async 2019-02-06 11:00:57 -05:00
Jack Robison f8dc0f298e Merge pull request #1868 from lbryio/fix_error_response
format error dict on get
2019-02-06 10:37:31 -05:00
Victor Shyba 8b5803c945 format error dict on get 2019-02-06 11:55:12 -03:00
Lex Berezhny 1314f75219 release script shows PRs that did not get included in changelog 2019-02-06 09:51:09 -05:00
Jack Robison be90ec51e4 release-text 2019-02-06 09:00:59 -05:00
Jack Robison e06c903731 Merge pull request #1867 from lbryio/stream-descriptor-validation
Add stream descriptor tests
2019-02-05 21:43:45 -05:00
Jack Robison 76c8447a56 add stream descriptor tests 2019-02-05 21:25:31 -05:00
Jack Robison 82b53e108c Merge pull request #1865 from lbryio/fix-delete-previous-stream
Fix deleting incomplete downloads when they are stopped
2019-02-05 15:43:55 -05:00
Jack Robison 7c8ad0fe30 fix deleting incomplete downloads upon stop
fix deleting previous streams (not assembled/published files) for a claim being updated
2019-02-05 15:33:08 -05:00
Jack Robison 073ffdd049 proto and type args 2019-02-05 13:36:25 -05:00
Jack Robison 7fe211d134 fix resolve_host 2019-02-05 13:31:57 -05:00
Jack Robison 43d424978d Merge pull request #1862 from lbryio/fix_announce
adds announce tests on stream creation/assemble, refactor calls to set announce
2019-02-05 12:58:56 -05:00
Victor Shyba a4ced4dbd2 set verified after callback 2019-02-05 13:17:00 -03:00
Victor Shyba c6b910a5f3 use time_getter on storage 2019-02-05 13:14:59 -03:00
Victor Shyba 63d1ec3c28 fixes from jack comments 2019-02-05 13:01:12 -03:00
Victor Shyba c31e61586d tests: fix value, compare to blobs len 2019-02-05 13:01:12 -03:00
Victor Shyba 7b8f42e6d0 remove dead set_should_announce code 2019-02-05 13:01:12 -03:00
Victor Shyba 2c275efa48 remove redundant calls to should announce 2019-02-05 13:01:12 -03:00
Victor Shyba 30846f932b test that creating a stream marks sd and head to announce 2019-02-05 13:01:12 -03:00
Victor Shyba 6ead932ccb announces naturally, removes redundant calls 2019-02-05 13:01:12 -03:00
Victor Shyba e9b58577dd tests: verify completed and to announce after assembling streams 2019-02-05 13:01:12 -03:00
Victor Shyba f766dbefe0 announce and set completed after inserts 2019-02-05 13:01:12 -03:00
Jack Robison f8a7d2ebe2 add release_texts to release.py 2019-02-05 09:58:42 -05:00
Jack Robison af43b21ef6 Merge pull request #1860 from lbryio/fix-announce
Fix setting blob completed and next announce time
2019-02-04 18:39:10 -05:00
Jack Robison 0685bc5689 include torba in log file 2019-02-04 18:25:47 -05:00
Jack Robison 8cda3143f2 fix setting should_announce and blob status 2019-02-04 18:22:44 -05:00
Jack Robison 5af351a234 reflector logging 2019-02-04 17:33:25 -05:00
Jack Robison b9934a7eb7 Merge pull request #1859 from lbryio/fix_invalid_data_client
handle and log if downloading from bad sendfile implementations
2019-02-04 17:19:57 -05:00
Victor Shyba b82d2c3120 handle and log if downloading from bad sendfile implementations 2019-02-04 19:13:31 -03:00
Jack Robison 8e607fcc2a Merge pull request #1858 from lbryio/fix-blob-sendfile
Fix blob sendfile
2019-02-04 17:12:50 -05:00
Jack Robison 2bce7cecfa fix sendfile on windows 2019-02-04 16:48:48 -05:00
Jack Robison e141cc9e22 Merge pull request #1857 from lbryio/handle-invalid-data-and-fix-cancel-blob-request
Handle invalid data and fix cancel blob request
2019-02-04 16:47:35 -05:00
Jack Robison 8883587cc7 logging 2019-02-04 16:19:27 -05:00
Jack Robison c85991704d catch invalid data error 2019-02-04 16:19:27 -05:00
Jack Robison 377442d9f0 fix request_blob cancellation 2019-02-04 16:19:24 -05:00
Jack Robison ad30b7df6d Merge pull request #1855 from lbryio/findvalue-always-include-close-contacts
Always return close peers from findValue
2019-02-04 15:30:17 -05:00
Jack Robison f0b49fc153 always return close peers from findValue 2019-02-04 15:07:42 -05:00
Jack Robison 6033399f05 Merge pull request #1854 from lbryio/no-self-queries
fix trying downloading from oneself, prune querying peer from findValue results
2019-02-04 15:00:13 -05:00
Jack Robison ae1162f5b0 don't try downloading from oneself
-don't return the querying peer in results for findValue
2019-02-04 14:50:53 -05:00
Jack Robison 7d7d4c35c3 Merge pull request #1852 from lbryio/close_protocol_reflector
refactor reflect to close protocol
2019-02-04 14:42:18 -05:00
Victor Shyba ad71a9a89d refactor reflect to close protocol 2019-02-04 13:34:18 -03:00
Jack Robison 13b3e4ba8e v0.31.0rc5 2019-02-04 10:22:43 -05:00
Jack Robison 43799168ba Merge pull request #1850 from lbryio/resume
Properly resume streams and fixup with statement
2019-02-04 10:18:31 -05:00
Victor Shyba 736a949cb5 resume streams on startup 2019-02-03 20:49:09 -03:00
Victor Shyba 11bb1cc7ee use with clause to avoid leaking fds 2019-02-03 20:41:47 -03:00
Jack Robison dbce49e567 Merge pull request #1849 from lbryio/fix-connection-status
Fix connection status
2019-02-03 17:00:14 -05:00
Jack Robison dcd78bd86e always delete file from downloads if download was not complete 2019-02-03 16:43:40 -05:00
Jack Robison e5af041627 fix connection status message 2019-02-03 16:20:52 -05:00
Jack Robison 73f4e954e5 Merge pull request #1848 from lbryio/fix-announce-time
Fix setting blob announce time
2019-02-03 16:18:33 -05:00
Jack Robison 0a027cf2de logging 2019-02-03 15:47:19 -05:00
Jack Robison 3f98b16800 fix blob announce time, use time.time instead of loop.time 2019-02-03 15:47:15 -05:00
Jack Robison bbadece75d remove unused packages from extension-pkg-whitelist in pylintrc 2019-02-03 15:02:43 -05:00
Jack Robison 8f5a8ab8ba fix peer_ping 2019-02-02 02:02:19 -05:00
Jack Robison 91e581422c v0.31.0rc4 2019-02-02 00:30:57 -05:00
Jack Robison b54d7185a1 Merge pull request #1847 from lbryio/re-reflect
add re-reflect task to stream manager, add concurrent_reflector_uploads to config
2019-02-01 23:11:06 -05:00
Jack Robison f2968aab22 add re-reflect task to stream manager, add concurrent_reflector_uploads to config 2019-02-01 22:59:41 -05:00
Jack Robison 4e27eaf8e6 Merge pull request #1846 from lbryio/create-directories-if-dont-exist
create data. download, and wallet directories if they don't exist on startup
2019-02-01 22:57:27 -05:00
Jack Robison ccbe8aa0eb create data. download, and wallet directories if they don't exist on startup 2019-02-01 22:45:28 -05:00
Jack Robison 478f79353a fix bytes error 2019-02-01 22:20:11 -05:00
Jack Robison c3b0151943 fix dht status error during startup 2019-02-01 22:12:07 -05:00
Jack Robison ba818354af Merge pull request #1844 from lbryio/reassemble-when-deleted-from-downloads
Reassemble when deleted from downloads
2019-02-01 21:59:36 -05:00
Jack Robison e96b75a0d0 fix file/download paths
move download-from-uri logic into stream manager
2019-02-01 21:40:35 -05:00
Jack Robison efe4afd09e fix blob announce time 2019-02-01 21:28:59 -05:00
Jack Robison c7cb6822cc logging 2019-02-01 21:28:59 -05:00
Jack Robison c75665d3f0 update file_list docs, add blobs_remaining to file list args and to file dict result 2019-02-01 21:28:59 -05:00
Jack Robison 744375b2c0 re-assemble file / resume downloads 2019-02-01 21:28:59 -05:00
Jack Robison c1b4a012ec Merge pull request #1845 from lbryio/stuck_peers_fixup
fix active peers not being removed after closing + parsing of data stream
2019-02-01 21:28:20 -05:00
Victor Shyba 40da5b6cb6 rework data parser 2019-02-01 20:09:14 -03:00
Victor Shyba b36cf3e36c remove closing peers from active connections 2019-02-01 20:09:14 -03:00
Jack Robison 4e3b529aec Merge pull request #1840 from lbryio/close-transport-on-sendfile-fail
close blob server transport on sendfile failing
2019-02-01 15:13:07 -05:00
Jack Robison 0a08f445a4 close blob server transport on sendfile failing 2019-02-01 15:11:37 -05:00
Jack Robison 3eac99c509 Merge pull request #1841 from lbryio/general_fixes_downloader
General fixes downloader
2019-02-01 15:10:36 -05:00
Victor Shyba f5bf1463af fixes from review 2019-02-01 17:09:37 -03:00
Victor Shyba ca4a55ef28 set verified event earlier and remove stop awaits 2019-02-01 16:05:14 -03:00
Victor Shyba 832be0758b let cancel propagate 2019-02-01 16:05:14 -03:00
Victor Shyba 25050fdeeb fixes for writers (check inside lock and dont hold verified bytes) 2019-02-01 16:05:14 -03:00
Jack Robison df0635103e Merge pull request #1839 from lbryio/faster-stream-manager-startup
faster stream manager startup
2019-02-01 14:00:19 -05:00
Jack Robison dabd9cbe1e faster stream manager startup 2019-02-01 13:53:03 -05:00
Jack Robison 42d7fbdbb3 Merge pull request #1838 from lbryio/fix-update-claim-missing-stream
Fix publishing an update where we don't have the previous stream
2019-02-01 13:52:34 -05:00
Jack Robison 2336015f9a fix publish
fixes https://github.com/lbryio/lbry/issues/1826
- remove `sources` argument from `publish`
- only add/update files we have
- delete existing stream if updating with a new one
2019-02-01 13:35:18 -05:00
Jack Robison 5a28128b69 Merge pull request #1835 from lbryio/remove-peer-manager-lock
Remove peer manager lock
2019-01-31 21:04:21 -05:00
Jack Robison 31445c7797 remove lock from PeerManager 2019-01-31 20:57:48 -05:00
Jack Robison 16d0ff8376 Merge pull request #1833 from lbryio/fix-publish-update
Fix publish when making an update with an existing stream
2019-01-31 20:56:34 -05:00
Jack Robison 87f5d1bd4a fix updating claim with existing stream 2019-01-31 20:49:19 -05:00
Jack Robison be6e8744b0 Merge pull request #1834 from lbryio/remove-blob-request-lock
remove blob request asyncio.Lock
2019-01-31 20:46:01 -05:00
Jack Robison 2e9acd1baa remove blob request asyncio.Lock 2019-01-31 20:19:50 -05:00
Jack Robison 607ccefe90 Merge pull request #1831 from lbryio/cleanup-downloader
Download and publish fixes
2019-01-31 18:17:26 -05:00
Jack Robison df86937f4e fix deleting a file mid download 2019-01-31 17:59:42 -05:00
Jack Robison 957d0290df add torba logging 2019-01-31 16:53:54 -05:00
Jack Robison 40b90ce79b fix hanging download 2019-01-31 16:53:27 -05:00
Jack Robison aa7f0845cc catch sendfile errors 2019-01-31 16:10:27 -05:00
Jack Robison 6e69d306de tests 2019-01-31 15:39:23 -05:00
Jack Robison d85d3dfaa4 increase default max_connections_per_download 2019-01-31 15:39:20 -05:00
Jack Robison 2b035009ef handle both urls and ip addresses for fixed nodes and reflector servers 2019-01-31 15:39:20 -05:00
Jack Robison f9fd62c214 add fixed peers immediately if the dht has no peers 2019-01-31 15:39:20 -05:00
Jack Robison e8a1953815 add fixed peers immediately if dht is skipped 2019-01-31 15:39:20 -05:00
Jack Robison 34776e7eb9 fix test 2019-01-31 15:39:20 -05:00
Jack Robison a4419f562f pylint 2019-01-31 15:39:20 -05:00
Jack Robison c0160ab4e9 update time_to_first_byte.py 2019-01-31 15:39:20 -05:00
Jack Robison e60e5b3919 fix updating content claim for a file
-fix file name and download dir not being hex encoded during publish, add repair script
2019-01-31 15:39:20 -05:00
Jack Robison 3589cc9977 call update_reflected_stream upon reflector finishing 2019-01-31 15:39:20 -05:00
Jack Robison 2e978c00b2 prioritize peers to download from 2019-01-31 15:39:20 -05:00
Jack Robison 2335243de9 pylint 2019-01-31 15:39:20 -05:00
Jack Robison 6ad68eabd9 tests 2019-01-31 15:39:20 -05:00
Jack Robison ca5c638124 tests 2019-01-31 15:39:20 -05:00
Jack Robison 262b9a624b update file_reflect 2019-01-31 15:39:20 -05:00
Jack Robison 05915c540a set fully reflected if the server indicates it had all of the stream 2019-01-31 15:39:20 -05:00
Jack Robison 6aef6a80b7 refactor downloader
split peer accumulation out, use Queues instead of locks
2019-01-31 15:39:20 -05:00
Jack Robison 16efe9ba95 add fixed_peer_delay setting 2019-01-31 15:39:20 -05:00
Jack Robison fcda9500fe add accumulate_peers to Node
simplifies collecting peers during a download by giving a queue of hashes to search for and a queue or peers as they are found
2019-01-31 15:39:20 -05:00
Alex Grin 64dec821b7 Update README.md 2019-01-31 14:24:44 -05:00
Jack Robison d5f6e58aad Update README.md 2019-01-31 14:15:38 -05:00
Jack Robison 087b65522e Merge pull request #1830 from lbryio/dht_fixes
Dht fixes
2019-01-31 13:52:00 -05:00
Victor Shyba 5ee4b2173a fix probes bugs, partial DHT refactor 2019-01-31 01:49:52 -03:00
Victor Shyba 3e1c3d1e4b use node_id parameter instead of your own 2019-01-31 01:11:09 -03:00
Victor Shyba 21f2b2dcd7 cast timestamp to integer before querying 2019-01-31 01:09:57 -03:00
Jack Robison ffc7fc0793 fix uncaught exception in BlobServerProtocol when client disconnects during a blob transfer 2019-01-30 12:03:05 -05:00
Jack Robison 15dbe5e54e Merge pull request #1828 from lbryio/fix_chunked
fix chunked requests for server
2019-01-30 10:25:30 -05:00
Victor Shyba 2e633c0ee6 fix chunked requests for server 2019-01-30 12:11:05 -03:00
Jack Robison 933d58d49b logging 2019-01-29 20:47:02 -05:00
Jack Robison 8f7cf0b38f verbose logging 2019-01-29 13:08:11 -05:00
Jack Robison adf378b6e9 Merge pull request #1825 from lbryio/fix-hanging-download
Fix hanging download
2019-01-28 23:44:42 -05:00
Jack Robison 1a4f083e76 skip test 2019-01-28 22:50:06 -05:00
Jack Robison 7bde09dcf8 add download_blob_from_peer.py 2019-01-28 22:41:16 -05:00
Jack Robison 4630fd38aa fix async generator junction tests 2019-01-28 22:39:36 -05:00
Jack Robison 418def56ac fix log 2019-01-28 22:38:17 -05:00
Jack Robison cefe3eb520 fix file_delete 2019-01-28 22:01:43 -05:00
Jack Robison 64dffa306f logging, cleanup 2019-01-28 22:01:43 -05:00
Jack Robison 7d33b4f1f3 fix unawaited task 2019-01-28 22:01:43 -05:00
Jack Robison f507d95198 always add fixed peers if configured after 2s 2019-01-28 22:01:43 -05:00
Jack Robison 2a7d80f8b4 logging 2019-01-28 22:01:43 -05:00
Jack Robison 305034ffb2 Merge pull request #1824 from lbryio/fix-ping-timeout
fix timeout error
2019-01-28 21:53:32 -05:00
Jack Robison bfaebf0085 fix timeout error 2019-01-28 17:20:14 -05:00
Jack Robison a94cf04e87 Merge pull request #1822 from lbryio/update-settings
Update settings
2019-01-28 17:17:56 -05:00
Jack Robison 71f9f8ae9c fix race condition in reflector server 2019-01-28 16:51:11 -05:00
Jack Robison 330862e487 fix https://github.com/lbryio/lbry/issues/1297 2019-01-28 16:00:37 -05:00
Jack Robison 778d3826ab clean up settings and use them 2019-01-28 15:51:27 -05:00
Jack Robison 4a11cf007f remove unused settings and constants 2019-01-27 19:51:44 -05:00
Jack Robison f6e7700c4c Merge pull request #1818 from lbryio/fix-find-value
fix error encoding find value response
2019-01-26 17:26:23 -05:00
Jack Robison 573285a5bb fix error encoding find value response 2019-01-26 17:04:11 -05:00
Lex Berezhny 60ed4e503f refactored settings_set to support all settings 2019-01-26 00:04:36 -05:00
Lex Berezhny aa1405887e use float instead of decimal for now 2019-01-25 23:20:43 -05:00
Lex Berezhny cced217e43 update doc format for max_key_fee 2019-01-25 23:20:43 -05:00
Lex Berezhny 8a3b8d2df1 max_key_fee in yaml as null 2019-01-25 23:20:43 -05:00
Lex Berezhny be0bd3bdea refactored --max-key-fee to be more ergonomic 2019-01-25 23:20:43 -05:00
Lex Berezhny e01c73a7f8 pylint 2019-01-25 21:00:20 -05:00
Lex Berezhny 134338db4b dropped help, commands and resolve_name commands 2019-01-25 21:00:20 -05:00
Lex Berezhny f47a685734 help message for start command 2019-01-25 20:28:47 -05:00
Lex Berezhny 6e29f7e99b user is shown help when typing lbrynet help 2019-01-25 20:28:47 -05:00
Lex Berezhny 026d93e92a Update .travis.yml
use `stage: build` for the docker builds to get some parallelism.
2019-01-25 17:28:06 -05:00
Lex Berezhny a55802c1de v0.31.0rc3 2019-01-25 17:05:38 -05:00
Jack Robison 894397a327 Merge pull request #1810 from lbryio/async-reflector
async reflector
2019-01-25 16:41:37 -05:00
Jack Robison 3e3f6c0e22 fix test 2019-01-25 16:20:19 -05:00
Jack Robison a7610e3d34 async reflector 2019-01-25 16:10:17 -05:00
Jack Robison d0e6e74e1b Merge pull request #1809 from lbryio/fix-blob-server
fix blob server protocol factory
2019-01-25 16:07:07 -05:00
Jack Robison 126d71978b disable slow test 2019-01-25 15:38:38 -05:00
Jack Robison 69bcb052de fix blob server protocol factory 2019-01-25 13:10:40 -05:00
Victor Shyba 071af12d1a release wallet server master tag every master merge 2019-01-25 12:20:42 -05:00
Lex Berezhny b7d62ca084 v0.31.0rc2 2019-01-25 10:46:20 -05:00
Jack Robison fa97f7bd38 fix written_bytes error 2019-01-25 10:23:03 -05:00
Lex Berezhny 4a35247016 fixes for toggle config, passing servers as list, etc 2019-01-25 10:19:24 -05:00
Lex Berezhny d0526ad7d5 bring --config back to start command 2019-01-25 10:19:24 -05:00
Lex Berezhny 19b529cf38 removed unused import 2019-01-25 10:19:24 -05:00
Lex Berezhny 143353be44 fix cli unit test 2019-01-25 10:19:24 -05:00
Lex Berezhny 0373196707 max_key_fee is plain YAML now 2019-01-25 10:19:24 -05:00
Lex Berezhny 7a601db23f cli fixup 2019-01-25 10:19:24 -05:00
Victor Shyba ed61c7feec add step to build wallet server images on release 2019-01-24 19:20:23 -05:00
Jack Robison 77ece96356 fix config error if $HOME is None
-not seen on travis due to HOME being manually set to /tmp
2019-01-24 12:42:55 -05:00
Lex Berezhny 80440fd6bd v0.31.0rc1 2019-01-24 00:13:55 -05:00
Lex Berezhny ed6537db79 integration test fixes 2019-01-24 00:13:15 -05:00
Lex Berezhny 79e4767f69 unused imports 2019-01-24 00:13:15 -05:00
Lex Berezhny 0ee279bfd7 fix startup/shutdown and file_list --full-status argument 2019-01-24 00:13:15 -05:00
Lex Berezhny 5249d2a3ca removed unused methods 2019-01-24 00:13:15 -05:00
Lex Berezhny 4d5823f473 distro package is in setup.py, does not make sense to wrap it in try/except ModuleNotFoundError 2019-01-24 00:13:15 -05:00
Lex Berezhny c2b9909397 moved tests/unit/lbrynet_daemon/test_docs.py into tests/unit/test_cli.py 2019-01-24 00:13:15 -05:00
Lex Berezhny afff362544 component tests need AsyncioTestCase 2019-01-24 00:13:15 -05:00
Lex Berezhny 64eb7dbc49 renamed test_Component_Manager.py to test_component_manager.py 2019-01-24 00:13:15 -05:00
Lex Berezhny 2dcd187d69 fixed component manager tests 2019-01-24 00:13:15 -05:00
Lex Berezhny 3d9ea92118 remove unnecessary logger configuration for BitcoinRPC 2019-01-24 00:13:15 -05:00
Lex Berezhny 13a18d422e dropped envparse 2019-01-24 00:13:15 -05:00
Lex Berezhny 1bbbbd289c TestCase -> AsyncioTestCase 2019-01-24 00:13:15 -05:00
Lex Berezhny e33b74f60c import mock from unittest m0ar 2019-01-24 00:13:15 -05:00
Lex Berezhny cd76e0c296 import mock from unittest 2019-01-24 00:13:15 -05:00
Lex Berezhny 3b61222e5a pylint 2019-01-24 00:13:15 -05:00
Lex Berezhny 2ca438c36e passing integration tests 2019-01-24 00:13:15 -05:00
Jack Robison 8948d5be86 don't shut down immediately after startup 2019-01-24 00:13:15 -05:00
Jack Robison a3e64f9cef remove dht requirement from stream manager component 2019-01-24 00:13:15 -05:00
Jack Robison 09afec1961 import 2019-01-24 00:13:15 -05:00
Jack Robison f9c465a636 f string 2019-01-24 00:13:15 -05:00
Jack Robison 89efe3d1b9 minimal changes to cli.py 2019-01-24 00:13:15 -05:00
Jack Robison 94cec1b86d fix 2019-01-24 00:13:15 -05:00
Jack Robison d62ac0b608 KeyboardInterrupt 2019-01-24 00:13:15 -05:00
Jack Robison 5596eac930 merge conflicts 2019-01-24 00:13:15 -05:00
Jack Robison 59f5381b0a update tests 2019-01-24 00:13:15 -05:00
Jack Robison 65ae1b3f08 update setup.py, travis, and wine_build 2019-01-24 00:13:15 -05:00
Jack Robison 977a7b34f4 update pylintrc 2019-01-24 00:13:15 -05:00
Jack Robison c01728a6a7 update and add scripts 2019-01-24 00:13:15 -05:00
Jack Robison 5bb94d744e storage 2019-01-24 00:13:15 -05:00
Jack Robison 494917158c update daemon and components 2019-01-24 00:13:15 -05:00
Jack Robison ea15674e62 delete old dht functional tests 2019-01-24 00:13:15 -05:00
Jack Robison 34a409ade5 delete log_support, looping_call_manager, and compat 2019-01-24 00:13:15 -05:00
Jack Robison 273e614715 delete old reflector code 2019-01-24 00:13:15 -05:00
Jack Robison 34a725f7ad fix conf 2019-01-24 00:13:15 -05:00
Jack Robison bb6112276f delete old tests 2019-01-24 00:13:15 -05:00
Jack Robison db75f0baef update SQLiteStorage 2019-01-24 00:13:15 -05:00
Jack Robison 30689bbf8d update/add config
remove the following settings:
  peer_search_timeout
  sd_download_timeout
  run_reflector_server
  download_mirrors
  pointtrader_server

add the following settings:
  blob_download_timeout
  peer_connect_timeout
  node_rpc_timeout
2019-01-24 00:13:15 -05:00
Jack Robison 4bf7742c19 async lbrynet.stream 2019-01-24 00:13:15 -05:00
Jack Robison 69b259c285 async lbrynet.blob_exchange 2019-01-24 00:13:15 -05:00
Jack Robison 2fa5233796 async lbrynet.dht 2019-01-24 00:13:15 -05:00
Jack Robison a5524d490c async lbrynet.blob 2019-01-24 00:13:15 -05:00
Jack Robison c713fac2d9 lbrynet.error 2019-01-24 00:13:15 -05:00
Lex Berezhny 7a038bbb98 cli bug fixes 2019-01-23 13:25:10 -05:00
Victor Shyba eba88c1df7 fix resolve for undecodeable claims 2019-01-23 13:22:23 -05:00
Victor Shyba 881e5aba68 simplify claim_list 2019-01-23 13:22:23 -05:00
Victor Shyba 368f0683ec fix claim_list for channels 2019-01-23 09:36:27 -05:00
Victor Shyba 94dc8c8520 add tests for claim_list 2019-01-23 09:36:27 -05:00
Lex Berezhny 8593cd362a check darwin before windows because darwin has win, face plam! 2019-01-22 18:04:17 -05:00
Lex Berezhny 2dbf96da81 blob downer test fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny 83c7797187 pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny 6606deafa5 fixed after testing with desktop app 2019-01-22 18:04:17 -05:00
Lex Berezhny 19b0d59159 config options api_host and api_port changed to just api 2019-01-22 18:04:17 -05:00
Lex Berezhny 9ccb3fa2a3 unit/test_cli tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 48533df523 fix integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 1ae1137005 fixing integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny d5f2832b2f pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny d0230b4893 command line fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny a6cd53b97c test_Daemon.py uses /tmp for data_dir 2019-01-22 18:04:17 -05:00
Lex Berezhny 7a28171a72 conf no longer global 2019-01-22 18:04:17 -05:00
Lex Berezhny 150bcb1116 new configuration manager, first pass 2019-01-22 18:04:17 -05:00
Jack Robison 6fe9c5bf00 Merge pull request #1794 from lbryio/async-exchange-rate-manager
refactor exchange rate manager to use asyncio
2019-01-22 12:05:07 -05:00
Jack Robison 3a86fc6680 refactor exchange rate manager to use asyncio 2019-01-22 11:34:49 -05:00
Jack Robison 2805cbb331 refactor loggly handler to use asyncio 2019-01-22 10:52:50 -05:00
Jack Robison 68d7a7014c Merge pull request #1792 from lbryio/delete-unused-scripts
delete unused scripts
2019-01-22 09:53:24 -05:00
Jack Robison 883fa267a5 delete unused scripts 2019-01-22 09:33:42 -05:00
Lex Berezhny 5f2c3ac965 dont run customlogger tests 2019-01-19 16:32:11 -05:00
Lex Berezhny bf8e32ebd2 remove custom logging 2019-01-19 16:30:19 -05:00
Lex Berezhny 5971be5bef v0.30.5 2019-01-18 16:14:48 -05:00
Lex Berezhny 677d76a3e4 v0.30.5rc4 2019-01-17 23:26:12 -05:00
Victor Shyba 6ed3207cb7 phase1: legacy claim format, ability to decode new format 2019-01-17 23:24:58 -05:00
Victor Shyba be6448b74f fix str/bytes confusion from resolve/claimdict on certificate_id 2019-01-17 23:24:58 -05:00
Victor Shyba b707ee4844 encode and decode detached signatures from claim dict 2019-01-17 23:24:58 -05:00
Victor Shyba 0e9888be3b fixes from review 2019-01-17 23:24:58 -05:00
Victor Shyba 68b56b7858 adds test for claim_show 2019-01-17 23:24:58 -05:00
Victor Shyba d32697f611 adjust assertions for fee changes 2019-01-17 23:24:58 -05:00
Victor Shyba 3c54b2bcd7 activate detached sigs 2019-01-17 23:24:58 -05:00
Victor Shyba 899b182811 fix getclaimsfortx in server 2019-01-17 23:24:58 -05:00
Victor Shyba 0dfb5e008d add integration test with unserializable claim 2019-01-17 23:24:58 -05:00
Victor Shyba abf94357eb tests and fixes for sig validation on undecode-able claims 2019-01-17 23:24:58 -05:00
Victor Shyba 36aded3830 store original payload for signing and verifying 2019-01-17 23:24:58 -05:00
Victor Shyba d7ebf50602 detached as a parameter instead of class attribute 2019-01-17 23:24:58 -05:00
Victor Shyba 523c4c86a9 change default curve to SECP256k1, pass name parameter 2019-01-17 23:24:58 -05:00
Victor Shyba 74cccdbfc7 simplify detached signing and verifying 2019-01-17 23:24:58 -05:00
Victor Shyba 70471eebfa wallet server and resolve working, functional test 2019-01-17 23:24:58 -05:00
Victor Shyba 6339224980 tests for detached signatures 2019-01-17 23:24:58 -05:00
Victor Shyba fe97f7a119 adds a detached signature attribute to ClaimDict 2019-01-17 23:24:58 -05:00
Victor Shyba f8ed605da2 adds a way to verify detached signatures 2019-01-17 23:24:58 -05:00
Victor Shyba e38ef2af1f adds a way to sign detached 2019-01-17 23:24:58 -05:00
Victor Shyba 5d8e627d02 adds a separate representation for a detached signature 2019-01-17 23:24:58 -05:00
Lex Berezhny c9d4ad5450 pylint unused import 2019-01-17 19:04:52 -05:00
Lex Berezhny 31586bf588 error out when user tries --http-auth 2019-01-17 19:04:52 -05:00
Lex Berezhny 974d3c83e3 keep wallet_balance to test deprecation with, until we find better way to do this 2019-01-17 17:35:15 -05:00
Lex Berezhny 8dc8ca382b removed previously deprecated commands 2019-01-17 17:35:15 -05:00
Lex Berezhny 1dbb250413 fix bytes vs str error in migrate5to6 script 2019-01-15 21:44:16 -05:00
Lex Berezhny 9f9ec692c7 orchstr8 docker for lbry 2019-01-15 17:15:24 -05:00
Thomas Zarebczan 2c1b54c76c Add note about wallet needing to be unlocked (grin was confused about this) 2019-01-15 15:59:45 -05:00
Thomas Zarebczan 7314494fb9 add m4v
It's a container type very similar to mp4. Someone recently added one in the community top bids section.
2019-01-15 15:55:59 -05:00
Jack Robison e74a5e8ef3 cleanup 2019-01-11 16:36:55 -05:00
Jack Robison b9af999739 rename -> time_to_first_byte.py 2019-01-11 16:36:55 -05:00
hackrush 2de155f90f Profile get using rpc calls 2019-01-11 16:36:55 -05:00
hackrush ab3db2e626 Get paths from args 2019-01-11 16:36:55 -05:00
hackrush f9d26b04d8 Get averages! 2019-01-11 16:36:55 -05:00
hackrush 24e0673198 Profile downloading of sd blobs 2019-01-11 16:36:55 -05:00
hackrush 833261801f Profile downloading of sd blobs 2019-01-11 16:36:55 -05:00
hackrush 6c8cbc1b9e added additional fields to uri parser 2019-01-11 15:10:22 -05:00
Lex Berezhny 6ae4e68d1c looping_call shouldnt need to mess with cancellation 2019-01-11 14:33:45 -05:00
Oleg Silkin d22a2dee33 Removes await from shutdown call 2019-01-11 14:33:45 -05:00
Oleg Silkin 0f1ed589a5 Modifies the unit tests for the analytics manager to have async routines
Signed-off-by: Oleg Silkin <o.silkin98@gmail.com>
2019-01-11 14:33:45 -05:00
Oleg Silkin b032e99dd1 Removes the twisted dependency from the analytics file and replaces it
with `asyncio` and `aiohttp`. Also removes the calling loop from the
analytics in favor of a `dict` that stores the names of the methods
and their coroutines that are wrapped inside a looping task. The tasks
are canceled when the analytics manager is asked to shutdown

Signed-off-by: Oleg Silkin <o.silkin98@gmail.com>
2019-01-11 14:33:45 -05:00
Lex Berezhny fd7811f38d v0.30.5rc3 2019-01-11 01:17:43 -05:00
Lex Berezhny 071db3a73d release.py script now extracts backwards-incompatible: value from PR description and adds it to changelog under Backwards Incompatible Changes 2019-01-11 01:16:59 -05:00
Lex Berezhny 950c122850 fix blob announce command argument constraints 2019-01-11 00:54:15 -05:00
Thomas Zarebczan e0386ea9dc rename lbnyum_wallet_dir to wallet_dir 2019-01-11 00:02:52 -05:00
Thomas Zarebczan 6c3eee0f0e Update conf.py 2019-01-10 23:50:59 -05:00
Thomas Zarebczan 4c23add193 fix unavailable download directory
If an uploaded file had a directory that was removed, or if the old download directory doesn't exist, use the default.
2019-01-10 23:05:43 -05:00
Lex Berezhny 248d1a18c7 pylint 2019-01-10 22:38:27 -05:00
Lex Berezhny 17d1b85423 run brew upgrade twice and ignore result 2019-01-10 22:38:27 -05:00
Lex Berezhny b0deb63129 blob_reflect fix 2019-01-10 22:38:27 -05:00
Lex Berezhny ebdb33bd11 blob_availability fix 2019-01-10 22:38:27 -05:00
Lex Berezhny 69ccbda168 fixed error handling when calling commands 2019-01-10 22:38:27 -05:00
Lex Berezhny a38643cca6 remove d2f from storage function call 2019-01-10 22:38:27 -05:00
Lex Berezhny 712b19605e Revert "fix breaking integration tests after fees changed due to content_type modification"
This reverts commit 0613cf2d59.
2019-01-10 10:22:37 -05:00
Alex Grintsvayg 8a5f3842fc fix media type for files without extension 2019-01-10 10:22:37 -05:00
Lex Berezhny db3e578bcd v0.30.5rc2 2019-01-08 23:56:07 -05:00
Lex Berezhny 7e22adff7c added contributor username to each PR listed in changelog 2019-01-08 23:55:34 -05:00
Lex Berezhny 76fbaac658 do not log aiohttp.access 2019-01-08 23:54:32 -05:00
hackrush c4fac29ecb actually get the certificate_id 2019-01-08 23:45:20 -05:00
Lex Berezhny 33392b3351 migration fix 2019-01-08 23:15:25 -05:00
Lex Berezhny 0613cf2d59 fix breaking integration tests after fees changed due to content_type modification 2019-01-08 17:54:40 -05:00
Alex Grintsvayg a930ca0631 if media type is not listed, default to a type that includes the file extension 2019-01-08 16:06:44 -05:00
Lex Berezhny 63a7271008 v0.30.5rc1 2019-01-07 22:15:37 -05:00
Lex Berezhny 5ca0ebcbb2 store loop on storage 2019-01-07 21:56:05 -05:00
Lex Berezhny 2c84b7a7dd get_running_loop() -> get_event_loop() 2019-01-07 21:56:05 -05:00
Lex Berezhny 71a0c704bd storage.clock.seconds() -> asyncio.get_running_loop().time() 2019-01-07 21:56:05 -05:00
Lex Berezhny d30e7cd29e use loop.time() instead of reactor.seconds() in SQLite storage 2019-01-07 21:56:05 -05:00
Lex Berezhny aeb7c6fe76 pylint, removed unused import 2019-01-07 21:56:05 -05:00
Lex Berezhny 134c20da93 stream cost estimate command refactor 2019-01-07 21:56:05 -05:00
Lex Berezhny 893b4fc3ca fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 9b27e81940 fix logging 2019-01-07 21:56:05 -05:00
Lex Berezhny 72c0acf269 test_HTTPBlobDownloader 2019-01-07 21:56:05 -05:00
Lex Berezhny 7be53ad692 more bug fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 16e596ec87 lots o bugs fixed discovered while using desktop app 2019-01-07 21:56:05 -05:00
Lex Berezhny 3fdcb80960 minor test fixes 2019-01-07 21:56:05 -05:00
Lex Berezhny 26a3221bac hash announcer fix 2019-01-07 21:56:05 -05:00
Lex Berezhny eb16cb0aa8 fix test_HashBlob 2019-01-07 21:56:05 -05:00
Lex Berezhny b5340505e5 fix blobmanager tests and dont run conf tests 2019-01-07 21:56:05 -05:00
Lex Berezhny 4bb8129bb7 fixing tests to work with asyncio daemon 2019-01-07 21:56:05 -05:00
Lex Berezhny 7f838d3a31 all integration tests run with plain unittest now 2019-01-07 21:56:05 -05:00
Lex Berezhny c186c31bf7 simplified component setup 2019-01-07 21:56:05 -05:00
Lex Berezhny 86fb99a37c fully asyncio daemon api 2019-01-07 21:56:05 -05:00
Lex Berezhny 6c52450858 temporary module providing d2f and f2d for converting between twisted and asyncio 2019-01-07 21:56:05 -05:00
Lex Berezhny fb4e94b04a asyncifying minimal amount of the old blob/p2p code 2019-01-07 21:56:05 -05:00
Lex Berezhny 248baf58b4 asyncio daemon 2019-01-07 21:56:05 -05:00
Lex Berezhny 0845d65f4e rebase master 2019-01-07 21:56:05 -05:00
Lex Berezhny dcc5415648 v0.30.4 2019-01-04 22:03:39 -05:00
Lex Berezhny 3c38a88b22 v0.30.4rc2 2019-01-04 11:53:54 -05:00
Victor Shyba 187b7670b1 adds a dockerfile for wallet server 2019-01-04 11:10:52 -05:00
Victor Shyba 049c17ff08 adds testenet to wallet server 2019-01-04 11:10:52 -05:00
Thomas Zarebczan 8de97bb441 Add lbrynet4 dht seed node (#1734)
added `lbrynet4.lbry.io` to list of default DHT nodes
2019-01-04 11:07:02 -05:00
Lex Berezhny 97c0092923 v0.30.4rc1 2019-01-04 03:14:56 -05:00
Lex Berezhny e65e1fb5a2 added utxo_release command 2019-01-04 03:14:37 -05:00
Victor Shyba 20dd1e55f5 fix data_dir parameter parsing 2019-01-04 01:25:35 -05:00
Lex Berezhny 6aa709e34e v0.30.3 2018-12-21 18:45:34 -05:00
Lex Berezhny e73f121c08 v0.30.3rc1 2018-12-19 09:12:56 -05:00
Victor Shyba 3b9d6c4f8e Instantiate resolver before the request to hold the selected height 2018-12-19 09:04:30 -05:00
Alyssa Callahan 69b60c19c3 Merge pull request #1712 from lbryio/readme-update
Update README.md
2018-12-17 10:47:29 -05:00
Lex Berezhny 04278cb1ab release script automatically figures out prerelease status 2018-12-14 22:25:06 -05:00
Lex Berezhny e8dec5b440 v0.30.2 2018-12-14 21:50:48 -05:00
Lex Berezhny 75f82b107e updated release script with unittests and ability to go from rc to final release 2018-12-14 21:50:30 -05:00
Jack Robison 0e90c4b611 Merge pull request #1717 from lbryio/fix-directory-args
fix directory cli args and update help message
2018-12-14 17:09:15 -05:00
Jack Robison fb66b68bc8 fix directory cli args, update help message 2018-12-14 15:51:11 -05:00
Victor Shyba c601677386 run schema unit tests on travis 2018-12-14 09:21:31 -05:00
Lex Berezhny 1fbd1cd04e v0.30.2rc2 2018-12-12 19:52:11 -05:00
Jack Robison 4a71b91fbf Merge pull request #1671 from lbryio/remove-android-helpers
remove `lbrynet.androidhelpers` module
2018-12-12 18:35:31 -05:00
Jack Robison b8b0d8b1ac refactor default directory setup, remove lbrynet.androidhelpers
fixes https://github.com/lbryio/lbry/issues/1606
2018-12-12 17:46:10 -05:00
Jack Robison 8ef73d3985 update docs 2018-12-12 12:19:26 -05:00
Jack Robison 05e556a5e3 fix generate_json_api script 2018-12-12 12:18:32 -05:00
Lex Berezhny d1da42bea8 Update README.md
* updated description and added bullet points for main features of this SDK
* lex (@eukreign ) is the official product owner of lbrysdk
2018-12-12 11:47:28 -05:00
Lex Berezhny 90685e8357 deploy release for tags 2018-12-12 00:10:50 -05:00
Lex Berezhny a471d5d167 v0.30.2rc1 2018-12-11 23:44:02 -05:00
Lex Berezhny 16122299e4 revert version back 0.30.1 2018-12-11 23:39:08 -05:00
Lex Berezhny 9f18da4bae release script (#1697)
* travis deploying to github
* release script
2018-12-11 23:24:19 -05:00
Lex Berezhny 1abce52867 v0.30.2rc1 2018-12-11 23:09:31 -05:00
Lex Berezhny 072e2b65d3 revert version change 2018-12-11 22:31:45 -05:00
Lex Berezhny f6d11e5467 0.30.1rc1 2018-12-11 22:29:40 -05:00
Oleg Silkin 458bddcb65 Removes Faker from Dependencies (#1687)
* Removes faker import and its usage
* Replaces data generated with faker with static data
* Drops faker as a dependency & removes extras require section
* Removes installation instruction for the test dependencies in setup.py
2018-12-11 20:19:58 -05:00
Victor Shyba 5fad43ff69 fix test tear down and unify test_commands 2018-12-11 16:34:37 -05:00
Victor Shyba 6992d0d947 add new test to tox.ini 2018-12-11 16:34:37 -05:00
Victor Shyba d1b0180437 add tests for claim_list_by_channel 2018-12-11 16:34:37 -05:00
Victor Shyba ba59e126cd forward errors outside of uris on channel listing 2018-12-11 16:34:37 -05:00
Victor Shyba 4e3467bb7e fix windows release type set script 2018-12-11 16:30:35 -05:00
Lex Berezhny 4d4c705eb9 0.30.1 changelog updates 2018-12-09 23:30:09 -05:00
Lex Berezhny ce7b3184c4 0.30.1 2018-12-09 23:15:11 -05:00
Lex Berezhny b71faddaac upload zip files into the same directory 2018-12-08 19:01:13 -05:00
Lex Berezhny 1328cb4cce check file exists before reading config 2018-12-08 19:01:13 -05:00
Lex Berezhny 946488bce9 correctly save daemon_settings.yml on fresh install 2018-12-08 19:01:13 -05:00
Victor Shyba 7713881e71 additional tests for resolution border cases 2018-12-08 14:40:37 -05:00
Victor Shyba 26fcc48c5a fix for name collision on channel path 2018-12-08 14:40:37 -05:00
Thomas Zarebczan bf12fb68a2 use xcode8.3 for mac build (#1692)
use xcode8.3 for mac build
2018-12-08 14:38:35 -05:00
Jack Robison 3a4c9ff4f9 bump to 0.30.1rc11 2018-12-07 13:07:06 -05:00
Jack Robison 6180a2917d add epub and wmv content types 2018-12-07 13:06:36 -05:00
Lex Berezhny 014cbb4623 v0.30.1rc10 2018-12-07 12:04:28 -05:00
Jack Robison ee98251d79 Merge pull request #1700 from lbryio/mimetypes
improve contentType detection
2018-12-07 12:02:41 -05:00
Jack Robison e3d08ec773 improve contentType detection
fixes https://github.com/lbryio/lbry/issues/1690
2018-12-07 11:31:05 -05:00
Lex Berezhny f03e68e63e 0.30.1rc9 2018-12-06 00:51:07 -05:00
Lex Berezhny 72090b45dd confirmations is returned as int 2018-12-06 00:49:07 -05:00
Lex Berezhny 93f9622798 fix json_response_encoder to also return correct confirmations 2018-12-06 00:49:07 -05:00
Lex Berezhny 16ff4b4e2c confirmations fix 2018-12-06 00:49:07 -05:00
Lex Berezhny c90cca4b20 fix #1683 2018-12-05 14:38:35 -05:00
Lex Berezhny 98d83ea74f fix for integration test 2018-12-05 12:14:43 -05:00
Lex Berezhny a234f92d2c 0.30.1rc8 2018-12-05 11:53:37 -05:00
Lex Berezhny 8ba3575901 dropped aiorpcx 2018-12-05 09:17:36 -05:00
Jack Robison a857c83c59 Merge pull request #1685 from lbryio/fix-1674
fix int too large to convert to integer when updating claim
2018-12-04 18:16:24 -05:00
Jack Robison a6128ac8a2 fix https://github.com/lbryio/lbry/compare/fix-1674 2018-12-04 17:35:40 -05:00
hackrush 2693301ad0 Fix python and pip3 not available in xcode8 2018-12-04 15:12:07 -05:00
Victor Shyba f7c7cb9535 fix transaction_show 2018-12-04 12:12:08 -05:00
Jack Robison b36a5f9b6b Merge pull request #1678 from lbryio/warn_migration_bad_claim_id
check and log claim id validity during migration
2018-11-30 15:58:26 -05:00
Jack Robison 4826eb9969 migrate double hex encoded certificates 2018-11-30 14:00:35 -05:00
Victor Shyba c34b150794 check and log claim id validity during migration 2018-11-30 14:00:35 -05:00
Lex Berezhny c4c7670a7c do not run brew update 2018-11-30 12:20:07 -05:00
Lex Berezhny 0918bac86e mac build stuff 2018-11-30 11:50:32 -05:00
Victor Shyba a60a921706 resolve claims on claim_list (#1483) 2018-11-30 11:46:56 -05:00
Lex Berezhny b3ee29b66d maybe python is already installed on mac 2018-11-30 11:30:03 -05:00
Lex Berezhny 3a1a9b988c on mac brew, python 3 is installed with plain python while python2 now requires python@2 2018-11-30 10:55:15 -05:00
Lex Berezhny 12f2bebc46 build for mac using xcode8 2018-11-30 10:34:45 -05:00
Jack Robison 31a3812c8a bump 2018-11-29 17:23:32 -05:00
Lex Berezhny 70ec08e30b icons! 2018-11-29 17:15:18 -05:00
Jack Robison 23e8f0d849 Merge pull request #1676 from lbryio/fix-1573
unlock wallet after encrypting it
2018-11-29 16:13:21 -05:00
Jack Robison 1cdfcd5b4e unlock wallet after encrypting it 2018-11-29 15:16:53 -05:00
Lex Berezhny dc2786fbc0 assert instead of if 2018-11-28 18:03:33 -05:00
Lex Berezhny 7e39b0c4fe run ClaimManagement.test_create_update_and_abandon_claim 2018-11-28 18:03:33 -05:00
Lex Berezhny 4af830c405 tests for transaction_list going through claim creation -> claim update -> claim abandon 2018-11-28 18:03:33 -05:00
Lex Berezhny fe844581fd correctly split value/fee in transaction_list 2018-11-28 18:03:33 -05:00
Lex Berezhny 443ce6352b typo 2018-11-28 18:03:33 -05:00
Lex Berezhny 158c354a27 split out fee from value when possible 2018-11-28 18:03:33 -05:00
Lex Berezhny 92d1230a90 remove fees from value, abandon is positive 2018-11-28 18:03:33 -05:00
Jack Robison d64916a061 bump 2018-11-27 16:28:55 -05:00
Alyssa Callahan 20a0568253 Merge pull request #1567 from preethamvishy/master
Update pull request template
2018-11-27 15:45:54 -05:00
Alyssa Callahan 5a1f42ee54 Merge branch 'master' into master 2018-11-27 15:39:48 -05:00
Alyssa Callahan ab95f14e21 Update PULL_REQUEST_TEMPLATE.md 2018-11-27 15:38:06 -05:00
Jack Robison bae2676b69 Merge pull request #1669 from lbryio/fix_invalid_sigs_from_name
allow invalid sigs claims to show outside a channel
2018-11-27 14:41:28 -05:00
Victor Shyba ea6869a63e allow invalid sigs claims to show outside a channel 2018-11-27 15:36:53 -03:00
Jack Robison 953327c150 bump 2018-11-27 09:52:33 -05:00
Lex Berezhny e01ad6bc85 disable travis cache on mac build 2018-11-26 20:55:03 -05:00
Victor Shyba 47a33009ff remove python 3.6 steps from travis 2018-11-26 20:55:03 -05:00
Victor Shyba bc844013a1 (win build) add pywin32 and netifaces wheels 2018-11-26 20:55:03 -05:00
Victor Shyba 2fd2319cdb use lbry dockerhub image 2018-11-26 20:55:03 -05:00
Victor Shyba a50657c472 add dockerfile and readme on how win docker builder is generated 2018-11-26 20:55:03 -05:00
Victor Shyba ef5f6e06ba use vshyba/temporary_pyinstaller_32bits_py37:latest 2018-11-26 20:55:03 -05:00
Victor Shyba 977cf9f506 log python version on windows build script 2018-11-26 20:55:03 -05:00
Lex Berezhny ffae10d11f drop pytest 2018-11-26 18:02:12 -05:00
Thomas Zarebczan 221107e76e test mac build, do not merge (#1659)
* Update .travis.yml to use xcode 6.4 for mac build
2018-11-26 14:13:45 -05:00
Jack Robison e74c78b607 Merge pull request #1432 from lbryio/get-block-on-bytes-written
[API] block get on bytes being written to the output file
2018-11-26 12:16:42 -05:00
Jack Robison 434c7b57b1 tests 2018-11-26 11:36:28 -05:00
Jack Robison 9dc4608a62 combine StreamProgressManager and FullStreamProgressManager
-add wrote_first_data Deferred to FullStreamProgressManager
2018-11-26 11:36:28 -05:00
Jack Robison fc69366092 set the download manager before creating the first deferred 2018-11-26 11:36:28 -05:00
Victor Shyba 9f0b1f3e25 fix pylint failures unveiled after astroid pin 2018-11-26 11:32:41 -05:00
Victor Shyba 3f03a845ec only return invalid signatures if its a direct resolution 2018-11-26 11:32:41 -05:00
Victor Shyba bc28976e12 adds test for claims on abandoned channels 2018-11-26 11:32:41 -05:00
Victor Shyba 8f826098f9 validate claim id before resolving 2018-11-26 11:32:41 -05:00
Victor Shyba 0a765e6207 handle errors on resolve 2018-11-26 11:32:41 -05:00
Lex Berezhny f83aa61923 force astroid==2.0.4 in .travis.yml 2018-11-26 10:47:48 -05:00
Oleg Silkin f9ff3e5d14 Removes third-party mock dependency (#1661)
* Removes mock from setup.py dependencies
* Refactors name resetTime to reset_time
* Replaces import mock with built-in unittest.mock
* Refactors imports to separate built-in, external, and project modules
2018-11-25 11:12:30 -05:00
Oleg Silkin b3fde9d78d Removes six From Project (#1660)
* Replaces `six` urllib with python 3's urllib
* Replaces `six`'s int2byte method with native `bytes` class
* Removes edge case testing for python2 vs python3
* Removes useless object inheritance
* Uses native io.BytesIO instead of six.BytesIO
* Removes six from dependencies
2018-11-22 23:24:21 -05:00
Lex Berezhny 0c999083cc added jsonrpc_account_send and dropped torba[server] 2018-11-21 01:14:39 -05:00
Victor Shyba 561215db8a fix download script 2018-11-21 00:25:43 -05:00
Jack Robison 8dbb7f2c78 bump 2018-11-20 21:37:49 -05:00
Lex Berezhny 6f7f3bb316 fixes #1653 2018-11-20 16:12:51 -05:00
Jack Robison 8d507c4b73 bump 2018-11-19 21:12:39 -05:00
Lex Berezhny 2beb0eda94 SupportingSupports test does not exist 2018-11-19 20:34:38 -05:00
Lex Berezhny b7e07b72ca update tox.ini to run correct tests 2018-11-19 20:34:38 -05:00
Lex Berezhny fde8718c8c make sure only actual claims/updates are found when updating a claim or sending it to someone 2018-11-19 20:34:38 -05:00
Lex Berezhny b2d1bddcfe update to latest torba 2018-11-19 20:34:38 -05:00
Lex Berezhny dd1d1be994 working claim_send_to_address 2018-11-19 20:34:38 -05:00
Jack Robison a93957178a bump 2018-11-19 17:31:31 -05:00
Jack Robison 5026d0b2f3 Merge branch 'fix-get-bad-claim-error' 2018-11-19 17:31:01 -05:00
Jack Robison cc8b47d882 improve error when get fails to decode the resolved claim 2018-11-19 17:30:02 -05:00
Jack Robison db0b5961b3 Merge pull request #1642 from jessopb/readmeStuff
adds daemon config to and updates links in readme
2018-11-19 14:37:15 -05:00
Jessop Breth cb6f7ada42 adds sample config and links to settings documentation 2018-11-19 10:10:02 -05:00
Jack Robison 59464099b4 add example daemon_settings.yml 2018-11-19 09:42:13 -05:00
preserveddarnell aed00b53fd Update various typos & edits 2018-11-18 20:52:18 -05:00
YULIUS KURNIAWAN KRISTIANTO cdb0f97cb4 added more social links 2018-11-18 10:29:47 -05:00
Jack Robison 33bcc8256c Merge branch 'fix-1624' 2018-11-16 11:45:05 -05:00
Jack Robison 0a8d44f12f Merge branch 'populate-buckets' 2018-11-16 11:44:25 -05:00
Jack Robison 1b74594f5b don't create ssl/auth keys if disabled
fixes https://github.com/lbryio/lbry/issues/1624
2018-11-16 11:12:33 -05:00
Jack Robison b0718d295e fix empty buckets in routing_table_get 2018-11-16 10:59:12 -05:00
Jack Robison 9bb91b126d use midpoint id instead of random for refresh list 2018-11-16 10:59:12 -05:00
Jack Robison 6dbb00d8d5 populate buckets when joining the dht 2018-11-16 10:59:12 -05:00
Jack Robison 3a27a4509a Merge pull request #1639 from lbryio/fix-1638
fix bytes/str in iterativeFindValue
2018-11-14 16:22:30 -05:00
Jack Robison 1d438f5d23 fixes https://github.com/lbryio/lbry/issues/1638 2018-11-14 15:35:59 -05:00
YULIUS KURNIAWAN KRISTIANTO 3840f19869 fix mistake code 2018-11-14 14:20:02 -05:00
Lex Berezhny ec0483b975 header progress percent starts at 0 instead of None 2018-11-12 21:16:05 -05:00
Jack Robison 2fba1f417f less verbose logging 2018-11-12 14:47:11 -05:00
Jack Robison 5b419b55c5 fix peer_list 2018-11-12 14:45:41 -05:00
Victor Shyba b31379e51c log and handle misformed certificate claims from #1609 2018-11-10 20:49:36 -05:00
Victor Shyba 8394dcb663 log and handle misformed claims from #1611 2018-11-10 20:49:36 -05:00
Lex Berezhny 457a56ce04 Merge pull request #1604 from lbryio/fix-external-ip
Fix setting the external ip
2018-11-10 20:48:40 -05:00
Jack Robison 7c8016d807 remove accidentally added lbrynet.schema.signature 2018-11-09 16:52:46 -05:00
Jack Robison d704336f20 absolute imports 2018-11-09 16:52:46 -05:00
Jack Robison 1b4230b06e move is_valid_blobhash to lbrynet.blob.blob_file 2018-11-09 16:52:46 -05:00
Jack Robison 88095af1cd move system_info.py into lbrynet.extras 2018-11-09 16:52:46 -05:00
Jack Robison e9efd9f68a improve get_external_ip 2018-11-09 16:52:46 -05:00
Jack Robison 7b32e2ff7c don't block the dht component on populating the routing table 2018-11-09 16:52:46 -05:00
Jack Robison 572777305e reduce analytics event frequency 2018-11-09 16:52:46 -05:00
Jack Robison 7e27a6e37d fix 2018-11-09 16:52:46 -05:00
Jack Robison b98621d7ea avoid name errors 2018-11-09 16:52:46 -05:00
Jack Robison bc5824b8bc better setup of custom logger class 2018-11-09 16:52:46 -05:00
Jack Robison a6b643bb34 move files 2018-11-09 16:52:42 -05:00
Jack Robison 488b976734 move HashAnnouncer.py to extras 2018-11-09 14:16:33 -05:00
Jack Robison 253912b52e fix skipping dht, peer_protocol_server, and hash_announcer components
-move PeerFinder and PeerManager initialization to the ComponentManager

-remove dht component requirement from all but the hash_announcer component. This allows running the file manager component without either of the server components.
2018-11-09 14:16:33 -05:00
Jack Robison 0bf8416d9f fix components_to_skip setting 2018-11-09 14:16:33 -05:00
Jack Robison 2e921437d8 fix external ip setup
fixes https://github.com/lbryio/lbry/issues/1561
2018-11-09 14:16:33 -05:00
Jack Robison b6f9b60683 remove commit in version for source installs 2018-11-09 14:16:33 -05:00
Lex Berezhny 57430e9f5f fixed #1616 2018-11-09 12:03:55 -05:00
Victor Shyba 1e13fbee8d await missing on resolve.py 2018-11-09 11:11:57 -05:00
Lex Berezhny 3d8b9e2b66 block=False for abandon during erics epic adventure tests 2018-11-07 18:28:02 -05:00
Lex Berezhny 64898d43f1 tearDown() should run before daemon shutdown 2018-11-07 18:28:02 -05:00
Lex Berezhny b2117fddab on_transaction -> ledger.wait 2018-11-07 18:28:02 -05:00
Lex Berezhny 5658623f29 claims include supports 2018-11-07 18:28:02 -05:00
Lex Berezhny 934682bd0f --blocking support in abandon command 2018-11-07 18:28:02 -05:00
Victor Shyba daecc582f3 simplify bind statement 2018-11-07 11:29:56 -05:00
Victor Shyba 3cb4012dbf unify loop 2018-11-07 11:29:56 -05:00
Victor Shyba f7bd44a4f5 fix query for windows (999 vars limit) 2018-11-07 11:29:56 -05:00
hackrush 52317b05bb Better help message instructions 2018-11-07 11:02:20 -05:00
Lex Berezhny 83c0baa3cb test abandoning claim at a loss, required a fix in torba 2018-11-07 10:57:11 -05:00
Lex Berezhny 340c5addab insufficientfundserror test 2018-11-07 10:57:11 -05:00
Lex Berezhny e3fd94dbf0 pylint fix 2018-11-07 10:57:11 -05:00
Lex Berezhny 923fcfb9e9 run publish integration tests separately 2018-11-07 10:57:11 -05:00
Lex Berezhny 0ae5f1a4c6 added fix for #1533 2018-11-07 10:57:11 -05:00
Lex Berezhny 90e06abba2 fixes for #1569 2018-11-07 10:57:11 -05:00
Victor Shyba 6ecfc94ff1 add balance to startup report 2018-11-07 10:50:13 -05:00
Alyssa Callahan 60c4a727d3 Merge pull request #1574 from czakarian/patch-1
Fix typos / grammar changes
2018-11-07 10:19:59 -05:00
Alyssa Callahan 9fbb9f2e98 Merge branch 'master' into patch-1 2018-11-07 10:18:58 -05:00
Lex Berezhny f1feec66c0 fixed path to cli.py in setup.py 2018-11-06 13:55:22 -05:00
Lex Berezhny 73b9b983b1 pinned protobuf to specific version, fixes #1602 2018-11-06 00:05:40 -05:00
Victor Shyba 588779e3a4 makefile for using types repo 2018-11-05 20:48:21 -05:00
Jack Robison b7ae446059 Merge pull request #1495 from lbryio/retry_dht
[DHT] retry on protocol level
2018-11-05 15:16:53 -05:00
Victor Shyba db898307b5 obey the rpcAttempts constant on retries 2018-11-05 15:16:39 -03:00
Victor Shyba 7e3512cbf2 retry on protocol level 2018-11-05 15:16:39 -03:00
Lex Berezhny cdd65c6783 updated pyinstaller build instructions 2018-11-04 20:25:24 -05:00
Lex Berezhny 320ad717cd pylint cleanup 2018-11-04 20:25:24 -05:00
Lex Berezhny 264e6744e1 updated references after moving conf to lbrynet.extras.daemon.conf 2018-11-04 20:25:24 -05:00
Lex Berezhny 64e2b79c07 moved conf into daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 9255bb5abe updated references after moving analytics to lbrynet.extras.daemon.analytics 2018-11-04 20:25:24 -05:00
Lex Berezhny d7525dacc3 moved analytics to lbrynet/extra/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 5cb07100f8 deleted unused interfaces file 2018-11-04 20:25:24 -05:00
Lex Berezhny 171c080d33 updated references lbrynet.file_manager -> lbrynet.blob 2018-11-04 20:25:24 -05:00
Lex Berezhny cfc3ec2a05 merged lbrynet/file_manager into lbrynet/blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 1392b8f83b removed lbrynet/lbry_file/__init__.py 2018-11-04 20:25:24 -05:00
Lex Berezhny 8b265801df updated references lbrynet.lbry_file.client -> lbrynet.file_manager.client 2018-11-04 20:25:24 -05:00
Lex Berezhny 6c4fb8dab1 moved lbrynet/lbry_file/client into lbrynet/file_manager/client 2018-11-04 20:25:24 -05:00
Lex Berezhny c1a9cd93ee updated references lbrynet.crypstream -> lbrynet.blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 0d4e83e167 merged lbrynet/cryptstream into lbrynet/blob 2018-11-04 20:25:24 -05:00
Lex Berezhny 1292eb345d fix dbmigrator pylint issue 2018-11-04 20:25:24 -05:00
Lex Berezhny f4c6dc90c9 updated references lbrynet.core -> lbrynet.p2p 2018-11-04 20:25:24 -05:00
Lex Berezhny 8c8f5d8f58 renamed lbrynet/core -> lbrynet/p2p 2018-11-04 20:25:24 -05:00
Lex Berezhny b6d986cb40 updated references after moving lbrynet/database into lbrynet/extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny 6cc134c7c8 merged lbrynet/database with lbrynet/extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny eee964cfa0 updated references to daemon after moving it to extras 2018-11-04 20:25:24 -05:00
Lex Berezhny a03c94bfad moved daemon to extras/daemon 2018-11-04 20:25:24 -05:00
Lex Berezhny f1f0917063 updated all references to point to new reflector location 2018-11-04 20:25:24 -05:00
Lex Berezhny fe2476b5aa moved reflector into extras 2018-11-04 20:25:24 -05:00
Lex Berezhny 9c5fa8bf76 winhelpers/winpaths.py -> extras/winpaths.py 2018-11-04 20:25:24 -05:00
Lex Berezhny cd43a71172 update winpaths.py references 2018-11-04 20:25:24 -05:00
Lex Berezhny e2c6de76f4 renamed knownpaths.py -> winpaths.py 2018-11-04 20:25:24 -05:00
Lex Berezhny 05aeadc9fe extraneous LICENSE file since the whole project is already MIT, moved copyright directly into file header 2018-11-04 20:25:24 -05:00
Lex Berezhny 7153417372 #egg=torba[server] 2018-11-04 10:50:15 -05:00
Lex Berezhny fceb2ae0a2 hmm 2018-11-04 10:50:15 -05:00
Lex Berezhny 0cef2fdd82 install extras from github 2018-11-04 10:50:15 -05:00
Lex Berezhny 9c91b135bd pylint requires [wallet-server] 2018-11-04 10:50:15 -05:00
Lex Berezhny e3b127192c install .[test] includes the [wallet-server] 2018-11-04 10:50:15 -05:00
Lex Berezhny ee1738da6e lbrynet.extras.wallet.server pylint fixes 2018-11-04 10:50:15 -05:00
Lex Berezhny 8ca2b75c80 lbryumx -> lbrynet.extras.wallet.server 2018-11-04 10:50:15 -05:00
Lex Berezhny dc81b6def1 install lbryumx after lbrynet so that we dont get the old protobuf 2018-11-04 10:50:15 -05:00
Lex Berezhny 22f5aaee70 regenerated protobufs 2018-11-04 10:50:15 -05:00
Lex Berezhny 4bd6569c9c lbrynet.schema.schema 2018-11-04 10:50:15 -05:00
Lex Berezhny dbf39baeb1 fix lbrynet.schema import 2018-11-04 10:50:15 -05:00
Lex Berezhny 3adf3c26f2 add set_build.py back in 2018-11-04 10:50:15 -05:00
Lex Berezhny fa0d13e630 protobuf==3.2.0 2018-11-04 10:50:15 -05:00
Lex Berezhny 227a82d6d3 update protobuf version 2018-11-04 10:50:15 -05:00
Lex Berezhny 65cfc1f63c removed lbryschema references from scripts and copied lbryschema requirements into local setup.py 2018-11-04 10:50:15 -05:00
Victor Shyba e46f0e2444 fix namespace lbryschema->lbrynet.schema 2018-11-04 10:50:15 -05:00
Victor Shyba 00bc7995c4 move lbryschema/tests/ into tests/unit/schema/ 2018-11-04 10:50:15 -05:00
Victor Shyba 2dcebe48b7 move lbryschema/lbryschema into lbrynet/schema 2018-11-04 10:50:15 -05:00
Lex Berezhny d12b6cddb3 fix integration tests 2018-11-04 02:28:36 -05:00
Lex Berezhny 699c057f2d upgrading to new torba 2018-11-04 02:28:36 -05:00
Lex Berezhny 3ce9f31f4b recursive channel lookup 2018-11-04 02:28:36 -05:00
Lex Berezhny 485d5d55e8 pylint fix 2018-11-04 02:28:36 -05:00
Lex Berezhny 6ee9f2513b add extra debug logging when a channel is not a claim errors 2018-11-04 02:28:36 -05:00
Daniel Brereton 81a6e14a5c added alt text 2018-11-02 19:38:30 -04:00
czakarian 496ffe0096 Fix typos / grammar changes 2018-10-31 16:35:39 -07:00
Preetham Viswanathan 216c760179 Merge pull request #1 from preethamvishy/preethamvishy/pr-template
Update pull request template
2018-10-30 17:16:56 -04:00
Preetham Viswanathan 01eca3004f Update PULL_REQUEST_TEMPLATE.md 2018-10-30 16:57:41 -04:00
GwanYeong Kim 2f47a3758e Remove unused Python modules in tests/unit/test_cli.py 2018-10-30 10:32:34 -04:00
pushkyn 73bcc1796f Add GitHub pull request template 2018-10-30 10:27:47 -04:00
Jack Robison e03e41ad31 bump 2018-10-26 16:51:23 -04:00
Jack Robison 6cb30f8e05 Merge branch 'fix-update-stream'
[API] add `aioupnp_version` to `upnp` field in the response to `status`
[API] fix uncaught claim signature errors
2018-10-26 16:49:51 -04:00
Jack Robison d9b1be6214 add aioupnp version to upnp status
-set non-blank not found message if upnp fails
2018-10-26 16:48:52 -04:00
Jack Robison 5f2f06c269 fix loggly 2018-10-26 13:01:50 -04:00
Jack Robison e1dcdcf4db catch claim signature error 2018-10-26 13:01:50 -04:00
Jack Robison 416472c5d8 log stream update failure 2018-10-26 13:01:46 -04:00
Alyssa Callahan 196f3ba92d added Internal use section 2018-10-24 15:47:07 -04:00
Lex Berezhny 6feae93c58 Update lbrynet/wallet/claim_proofs.py 2018-10-23 12:02:10 -04:00
Lex Berezhny 203d30d4f7 Update lbrynet/wallet/claim_proofs.py 2018-10-23 12:02:10 -04:00
Hugo d0a7ca841b Upgrade unit tests to use more useful asserts 2018-10-23 12:02:10 -04:00
Hugo 2e6c848c39 Upgrade Python syntax with pyupgrade --py36-plus 2018-10-23 12:02:10 -04:00
Jack Robison 18a0204105 set_build regex 2018-10-22 22:27:57 -04:00
Jack Robison 01baa193d5 fix set_build 2018-10-22 20:58:52 -04:00
Jack Robison ba57af5502 fix set_build 2018-10-22 17:55:32 -04:00
Jack Robison 8139f93ff1 set build type 2018-10-22 16:31:05 -04:00
Jack Robison 4596d5c26f bump 2018-10-22 09:12:49 -04:00
Nikita Titov 810a63782d Fixed long description of package 2018-10-20 10:15:29 -04:00
Nikita Titov ea83827d57 Update macOS naming 2018-10-20 10:02:24 -04:00
Jack Robison e19f8f3d95 bump 2018-10-19 12:28:08 -04:00
Jack Robison 7af501e0bc Merge branch 'temp-blob-pass-sqlite-object' 2018-10-19 12:27:27 -04:00
hackrush a692b6ff7d Pass SQLite storage object to DiskBlobManager for single(temp) blob download 2018-10-19 10:39:39 -04:00
Jack Robison 3364ac0976 bump 2018-10-18 18:22:46 -04:00
Jack Robison cf1157f02a Merge branch 'upnp-status' 2018-10-18 18:21:53 -04:00
Jack Robison 0ed56bbe1a tests 2018-10-18 18:12:50 -04:00
Jack Robison 213bdac55f check upnp redirects and external ip in a looping call
-add upnp setup success/fail analytics event
2018-10-18 17:41:49 -04:00
Jack Robison 6d8ab97ca9 fix analytics 2018-10-18 17:41:17 -04:00
Jack Robison fb34476cd7 add upnp status dictionary to status response
-log upnp debugging information as an error
2018-10-18 15:10:00 -04:00
Jack Robison 07d353a947 bump 2018-10-18 13:19:25 -04:00
Jack Robison 3c5fb6490d catch errors 2018-10-18 13:00:27 -04:00
Jack Robison a5d21081b9 Merge pull request #1549 from hugovk/link-to-key
Add link to GPG key
2018-10-18 12:41:42 -04:00
Jack Robison ed98a3cc21 Merge branch 'master' into link-to-key 2018-10-18 12:39:39 -04:00
Jack Robison 45c2cd2986 Merge pull request #1550 from hugovk/fix-spelling
Fix spelling
2018-10-18 12:37:56 -04:00
Hugo 75f21950da Add link to GPG key
And fix some typos.
2018-10-18 14:44:48 +03:00
Hugo 3f704be85c Fix spelling 2018-10-18 14:40:37 +03:00
Jack Robison 67a15ea788 bump 2018-10-18 00:34:58 -04:00
Jack Robison 55bcd2dfee Merge branch 'fix-asyncio-build' 2018-10-18 00:34:04 -04:00
Jack Robison d9757fdae8 remove asyncio from requirements 2018-10-18 00:32:44 -04:00
Jack Robison a4ae8d7c53 bump 2018-10-17 21:36:54 -04:00
Jack Robison 78078b4c9f Merge remote-tracking branch 'origin/fix_dht_logging' 2018-10-17 21:26:46 -04:00
Victor Shyba 1331e2bbe6 fix DHT logging 2018-10-17 22:25:41 -03:00
Lex Berezhny 4c4d652f1b fixes for #1526 and #1483 2018-10-17 21:10:23 -04:00
Lex Berezhny 73bd76b631 format publish output bid value in lbc instead of dewies 2018-10-17 19:50:31 -04:00
Lex Berezhny 59a0f9f602 removed the max response size from Known Issues as it was fixed 2018-10-17 19:38:14 -04:00
Lex Berezhny 1b1847bc31 added Known Issues section 2018-10-17 19:38:14 -04:00
Lex Berezhny 21a569edcc updated CHANGELOG for v0.30 2018-10-17 19:38:14 -04:00
Lex Berezhny 2ddf1a08f6 publish command supports custom account list to lookup channels 2018-10-17 19:37:54 -04:00
Victor Shyba 8754c6fabe fix resolve_name for asyncio 2018-10-17 17:17:25 -04:00
Victor Shyba f049109a38 fix claim_show 2018-10-17 17:17:25 -04:00
Victor Shyba 8db7c15aa1 fix address query handler 2018-10-17 15:33:22 -04:00
Victor Shyba 90e0694d2e fix download from peer script 2018-10-17 15:33:22 -04:00
Victor Shyba 4748cbf654 fix tests for cost estimate 2018-10-17 14:16:00 -04:00
Victor Shyba 74eb991983 fix paid downloads 2018-10-17 14:16:00 -04:00
Victor Shyba e11bae2dda stream cost estimate fix for uri without size 2018-10-17 14:16:00 -04:00
Victor Shyba 8462f56b13 fix blob_list for uris 2018-10-17 14:16:00 -04:00
Victor Shyba 0bec3c7294 fix claim_list_by_channel 2018-10-17 14:16:00 -04:00
Victor Shyba 2bb4052193 fix claim_list 2018-10-17 14:16:00 -04:00
Lex Berezhny 101a3f2b22 respect --show_seed argument to account_list command 2018-10-17 13:45:01 -04:00
Lex Berezhny 04a2bde8dd removed extraneous asyncio import 2018-10-17 13:29:28 -04:00
Jack Robison a1368b05da txupnp --> aioupnp
-update INSTALL.md
2018-10-17 13:29:28 -04:00
Victor Shyba 99ea7a26b0 fix get for asyncio torba 2018-10-17 10:27:57 -04:00
Lex Berezhny 8b3a457925 improve integration test reliability 2018-10-16 21:28:47 -04:00
Lex Berezhny e02e2a249a fix wallet component 2018-10-16 17:09:21 -04:00
Lex Berezhny 3019a1c222 revert tox config 2018-10-16 17:09:21 -04:00
Lex Berezhny 1f3a19c100 pylint 2018-10-16 17:09:21 -04:00
Lex Berezhny 5eac4247e1 working integration tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 1ca82b9d52 major facepalm 2018-10-16 17:09:21 -04:00
Lex Berezhny 21702ec43c asyncio progress for victor 2018-10-16 17:09:21 -04:00
Lex Berezhny d2083bc17e for victor 2018-10-16 17:09:21 -04:00
Lex Berezhny 52edf880e0 fix typo in .travis.yml file 2018-10-16 17:09:21 -04:00
Lex Berezhny 1f45af991f install dependencies needed for asyncio unit tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 8afdd0b2e9 pylint fix 2018-10-16 17:09:21 -04:00
Lex Berezhny 99efc70ba3 run wallet unit tests separate from the other twisted trial based tests 2018-10-16 17:09:21 -04:00
Lex Berezhny 64e306801d updated wallet to use asyncio 2018-10-16 17:09:21 -04:00
Victor Shyba 330db61b36 report balance and summary on startup 2018-10-12 20:53:31 -04:00
Lex Berezhny 357763ff61 respect blockchain_name in daemon settings when creating initial wallet 2018-10-12 12:28:39 -04:00
Lex Berezhny 8a6e6b4d77 add category attribute in JSON RPC output for backwards compatibility 2018-10-12 11:08:01 -04:00
Lex Berezhny 64913c4288 JSON RPC output for TXOs now includes: height, confirmations and valid_signature 2018-10-12 10:41:49 -04:00
Victor Shyba f01ad0e4bd use smart_decode on transaction parsing to support legacy claims 2018-10-11 17:38:23 -04:00
Victor Shyba d9fcc458dc refactor part of peer logic out of try/catch 2018-10-10 15:27:03 -04:00
Victor Shyba 24a11d3f21 make peer search asynchronous 2018-10-10 15:27:03 -04:00
Victor Shyba 234eaf8890 fix download_blob_from_peer script 2018-10-10 15:27:03 -04:00
Tomasz Kopeć 9e2e53147e --conf option improvements (#1455)
* Conf file improvements
* Add test for loading config file
2018-10-10 13:44:51 -04:00
Victor Shyba fc41af5889 set wallet max_gap after sync 2018-10-10 13:38:14 -04:00
Victor Shyba 72ce0cd0b0 explicitly define a default for signatures as expected on 0.21 2018-10-10 11:36:20 -04:00
Jack Robison e4497c1327 Merge remote-tracking branch 'origin/fix-1414' 2018-10-10 11:16:45 -04:00
hackrush a1bf832ea6 fix blob_availability 2018-10-10 16:21:54 +05:30
Lex Berezhny d4e76500fb removed unused import 2018-10-10 00:02:50 -04:00
Lex Berezhny f898bb3d0e database index fix 2018-10-09 23:58:32 -04:00
Lex Berezhny ad13282494 txo (claim_id) column index 2018-10-09 23:40:02 -04:00
Lex Berezhny 73519f8c6a claim_name updated to use new account.get_claims() method 2018-10-09 21:46:54 -04:00
Lex Berezhny b2c95f35c7 fixed test_transaction reliability 2018-10-09 21:32:32 -04:00
Lex Berezhny 0bfccb86ff refactored pagination per @seanyesmunt feedback 2018-10-09 20:46:41 -04:00
Lex Berezhny 5118240897 fix log message in header downloader to log local_height as integer instead of float 2018-10-09 17:39:28 -04:00
Lex Berezhny 427fc179ac fix for #1523 2018-10-09 16:48:12 -04:00
Lex Berezhny 55bcc520e4 pagination method fixes and not showing fee for recieved TX in get_history 2018-10-09 14:46:44 -04:00
Lex Berezhny bc8251696e fix cli version test 2018-10-09 11:37:49 -04:00
Lex Berezhny 677ba25ccd use .format() instead of string concatenation 2018-10-09 11:15:11 -04:00
Alex Grintsvayg 95436b233c fix lbrynet --version 2018-10-09 11:15:11 -04:00
Lex Berezhny 82871b8f6b fix command line argument for account_balance and other paginated commands 2018-10-09 10:30:30 -04:00
Lex Berezhny 9525b87224 inlineCallbacks fix 2018-10-08 22:44:00 -04:00
Lex Berezhny f06098978c ledger.get_transaction dropped in favor of ledger.db.get_transaction 2018-10-08 19:51:44 -04:00
Jack Robison 141491f2da bump version 2018-10-08 16:11:36 -04:00
Victor Shyba 38006f7d29 migrate addresses before starting the migrated wallet 2018-10-08 15:29:57 -04:00
Lex Berezhny 3664c25d98 changed lbry account.get_balance() to have same confirmations default as baseaccount.get_balance() 2018-10-08 15:02:52 -04:00
Lex Berezhny 1747c9d866 removed --include_reserved from account_list command 2018-10-08 15:02:52 -04:00
Lex Berezhny 1066fbaa7c pylint fix 2018-10-08 15:02:52 -04:00
Lex Berezhny 7f7961f4cc updated integration tests after reducing required confirmations to 0 2018-10-08 15:02:52 -04:00
Lex Berezhny 9d9916548b added channel/signature annotation to all TXOs, support for pagination 2018-10-08 15:02:52 -04:00
Lex Berezhny ada5b55f22 remove unused import 2018-10-08 15:02:52 -04:00
Lex Berezhny a2f28e7788 + permanent_url and TXO annotations 2018-10-08 15:02:52 -04:00
Jack Robison c6092a4c79 bump version 2018-10-08 09:16:54 -04:00
Jack Robison 315a557019 Merge remote-tracking branch 'origin/1439' 2018-10-08 09:15:46 -04:00
Jack Robison 20221e1186 Merge remote-tracking branch 'origin/reduce_routing_table_ping_flood' 2018-10-08 09:15:30 -04:00
Victor Shyba 50003d2600 restore delay on ping queue 2018-10-03 19:21:48 -03:00
Victor Shyba ea6b2b98fb refactor ping queue 2018-10-03 19:21:48 -03:00
Victor Shyba 4890fdfb50 remove semaphore from ping queue 2018-10-03 19:21:48 -03:00
Lex Berezhny 2f95c3a9d1 wallet integration test fixes 2018-10-03 18:06:22 -04:00
Lex Berezhny 6039159ecc use d2l/l2d in integration tests 2018-10-03 18:06:22 -04:00
Lex Berezhny 8e4d8345bd removed unused import 2018-10-03 18:06:22 -04:00
Lex Berezhny 22df26eeb6 dewies_to_lbc and lbc_to_dewies 2018-10-03 18:06:22 -04:00
Victor Shyba 463cbde4b1 cast dict_keys to list before using as shortlist 2018-10-01 15:23:50 -03:00
Victor Shyba 1adce8b227 reduce routing table ping flood 2018-10-01 15:17:10 -03:00
351 changed files with 34488 additions and 28695 deletions

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
.git
.tox
__pycache__
dist
lbrynet.egg-info
docs
tests
+22 -1
View File
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
We are also available on Discord at https://chat.lbry.com
-->
@@ -51,3 +51,24 @@ Tell us what happens instead
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
+38
View File
@@ -0,0 +1,38 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+4
View File
@@ -1,9 +1,13 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+5 -7
View File
@@ -9,7 +9,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
ignore=CVS,schema
# Add files or directories matching the regex patterns to the
# blacklist. The regex matches against base names, not paths.
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
miniupnpc,
unqlite
# extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
@@ -62,7 +60,7 @@ confidence=
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# disable everything first and then re-enable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
@@ -354,13 +352,13 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=twisted.internet.reactor,leveldb,distutils
ignored-modules=leveldb,distutils
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
ignored-classes=twisted.internet.reactor,RequestMessage
# ignored-classes=
# List of members which are set dynamically and missed by pylint inference
+75 -38
View File
@@ -3,88 +3,105 @@ dist: xenial
language: python
python: "3.7"
env:
global:
# must also be updated in wine_build.sh
- TORBA=master
jobs:
include:
- stage: code quality
name: "pylint lbrynet"
install:
- pip install pylint
- pip install git+https://github.com/lbryio/torba.git
- pip install git+https://github.com/lbryio/lbryschema.git
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script: pylint lbrynet
- &tests
stage: test
name: "Unit Tests w/ Python 3.7"
- stage: test
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git
- pip install git+https://github.com/lbryio/lbryschema.git
- pip install -e .[test]
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- <<: *tests
name: "Unit Tests w/ Python 3.6"
python: "3.6"
- <<: *tests
name: "DHT Tests w/ Python 3.7"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- <<: *tests
name: "DHT Tests w/ Python 3.6"
python: "3.6"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/electrumx.git --branch lbryumx && popd
- pushd .. && git clone https://github.com/lbryio/orchstr8.git && popd
- pushd .. && git clone https://github.com/lbryio/lbryschema.git && popd
- pushd .. && git clone https://github.com/lbryio/lbryumx.git && cd lbryumx && git checkout afd34f323dd94c516108a65240f7d17aea8efe85 && cd .. && popd
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
services:
- docker
install:
- docker pull cdrx/pyinstaller-windows:python3-32bit
- docker pull lbry/pyinstaller34_32bits:py371
script:
- docker run -v "$(pwd):/src/lbry" cdrx/pyinstaller-windows:python3-32bit lbry/scripts/wine_build.sh
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/win/
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
python: "3.6"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- pip3 install git+https://github.com/lbryio/lbryschema.git
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/cli.py
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
env: OS=linux
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
@@ -92,14 +109,34 @@ jobs:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/${OS}/lbrynet
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode9.4
osx_image: xcode8.3
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- brew upgrade python || true
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
cache:
directories:
+64 -13
View File
@@ -7,17 +7,64 @@ regard to the json-rpc api. As we're currently pre-1.0 release, we
can and probably will change functionality and break backwards compatibility
at anytime.
## [Unreleased]
Python 3 upgrade of the entire code base and switching to a brand new wallet
implementation are the major changes in this release.
## [0.30.1] - 2018-12-09
### API
* add `aioupnp_version` to `upnp` field in the response to `status`
* fixed uncaught claim signature errors
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
### Schema
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
### Wallet
* fixed resolving names with invalid channel signatures
* fixed wallet sync issues
* fixed wallet locking/encryption issues
### DHT
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
### Database
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
### Misc
* switch to Python 3.7
* brought back icon for windows binary
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
## [0.30.0] - 2018-10-22
This release is the most ambitious upgrade to lbrynet yet; including a brand
new wallet implementation, switch to Python 3 of the entire code base and
many changes to the API such as pagination and a more consistent output from
most commands.
### Security
* upgraded `cryptography` package.
* switch to `coincurve` for private / public key generation (addresses)
* support for connecting to daemon via SSL
* improved workflow for encrypt/decrypt of wallet
### API
* unified all command line executables into a single `lbrynet` executable.
* added `use_https` configuration setting (off by default), if true the api server will generate a self signed ssl certificate when started and only handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* added `use_https` configuration setting (off by default), if true the api
server will generate a self signed ssl certificate when started and only
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* deprecated `daemon_stop` command, use `stop` instead.
* deprecated `wallet_balance` command, use `account_balance` instead.
* deprecated `wallet_unlock` command, use `account_unlock` instead.
@@ -27,7 +74,7 @@ implementation are the major changes in this release.
* deprecated `wallet_list` command, use `address_list` instead.
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
* deprecated `wallet_public_key` command.
* deprecated `wallet_new_address` command, use `address_generate` instead.
* deprecated `wallet_new_address` command.
* deprecated `wallet_unused_address` command, use `address_unused` instead.
* deprecated `claim_renew` command.
* added `account_list` command to list accounts including their balance.
@@ -44,12 +91,14 @@ implementation are the major changes in this release.
* added `account_max_address_gap` command to find large gaps of unused addresses.
* added `address_list` command to list addresses.
* added `address_is_mine` command to check if an address is one of your addresses.
* added `address_public_key` command to get public key of an address.
* added `address_generate` command to generate a new address.
* added `address_unused` command to get existing or generate a new unused address.
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
`transaction_list` and `utxo_list`.
* added `upnp` field to `status` response
* removed `send_amount_to_address` command previously marked as deprecated
* removed `channel_list_mine` command previously marked as deprecated
* removed `get_availability` command previously marked as deprecated
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
### Wallet
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
@@ -68,11 +117,13 @@ implementation are the major changes in this release.
* Extensive internal changes as a result of porting to Python 3.
### Reflector
*
* No major changes to reflector.
### Database
*
* No changes to existing storage module.
### Known Issues
* Channel certificate import/export is currently not working.
## [0.21.2] - 2018-08-23
### Fixed
@@ -435,7 +486,7 @@ implementation are the major changes in this release.
* Fixed https://github.com/lbryio/lbry/issues/923
* Fixed concurrent reflects opening too many files
* Fixed cases when reflecting would fail on error conditions
* Fixed deadlocks from occuring during blob writes
* Fixed deadlocks from occurring during blob writes
* Fixed and updated`lbrynet.tests.dht`
* Fixed redundant dht id
* Fixed dht `ping` method
@@ -822,7 +873,7 @@ implementation are the major changes in this release.
* Removed check_pending logic from Daemon
* Switched to txrequests so requests can use twisted event loop
* Renamed API command file_seed to file_set_status
* Dont add expected payment to wallet when payment rate is 0
* Don't add expected payment to wallet when payment rate is 0
### Fixed
* Fix restart procedure in DaemonControl
* Create download directory if it doesn't exist
@@ -853,7 +904,7 @@ implementation are the major changes in this release.
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
* Remove unused API commands from daemon
* Fix file filter `outpoint`
* Made dictionary key names in API commmand outputs to be more consistent
* Made dictionary key names in API command outputs to be more consistent
### Added
* Add file filters: `claim_id`, `outpoint`, and `rowid`
* Make loggly logs less verbose
@@ -905,7 +956,7 @@ implementation are the major changes in this release.
* Include download url in version check
### Fixed
* add misssing traceback to logging
* add missing traceback to logging
## [0.8.3] - 2017-02-15
### Fixed
+1 -1
View File
@@ -1,3 +1,3 @@
## Contributing to LBRY
https://lbry.io/faq/contributing
https://lbry.tech/contribute
+31 -59
View File
@@ -1,67 +1,45 @@
# Installing LBRY
If only the json-rpc API server is needed the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
[![Setup for development](https://spee.ch/2018-10-04-17-13-54-017046806.png)](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
## Prerequisites
Before starting, you'll need to install some additional libraries depending on your operating system.
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
### Python
After installing python 3, you'll need to install some additional libraries depending on your operating system.
Running `lbrynet` from source requires [Python 3.7](https://www.python.org/downloads/release/python-370/)
### macOS
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
### OS X Prerequisites
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set
1. PYTHONUNBUFFERED=1
2. EVENT_NOKQUEUE=1
Remaining dependencies can then be installed by running:
```
brew install python3 protobuf
python3 -m pip install virtualenv
brew install python protobuf
```
### Linux Prerequisites
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
On Ubuntu (we recommend 16.04), install the following:
### Linux
On Ubuntu (we recommend 18.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
If you're running another Linux flavor, install the equivalent of the above packages for your system.
### Windows Prerequisites
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
Add C:\MinGW\bin to the windows PATH.
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
```
[build]
compiler = mingw32
```
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
Currently, available binaries include:
- Twisted
- Zope.interface
- pywin32
Install the above binaries with `pip install *.whl`
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
If you're running another Linux distro, install the equivalent of the above packages for your system.
## Installation
@@ -70,39 +48,33 @@ To install:
```
git clone https://github.com/lbryio/lbry.git
cd lbry
Creating venv:
python -m venv lbry-venv
virtualenv lbry-venv --python=python3.7
Activating lbry-venv on Linux/Mac:
source lbry-venv/bin/activate
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
pip install --editable .[test] # [test] installs extras needed for running the tests
pip install -e .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Run the tests
To run the unit tests from the repo directory:
```
trial --reactor=asyncio tests.unit
```
## Usage
To start the API server:
`lbrynet start`
To verify your install, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Making a Change
If you are not already inside the virtual environment, run:
```
source lbry-venv/bin/activate
```
Then run:
```
pip install --editable .
```
This will install `lbrynet` in such a way that the changes you make to the code will be used as soon as you restart the API server.
Happy hacking!
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2018 LBRY Inc
Copyright (c) 2015-2019 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
+23 -10
View File
@@ -1,12 +1,19 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
This project provides an SDK for building applications using the LBRY protocol. The SDK is written in Python 3.7+ using Twisted. For other languages or platforms, a JSONRPC API is provided.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
@@ -14,17 +21,19 @@ Run `lbrynet start` to launch the API server.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Running from source
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
@@ -32,12 +41,16 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. Our PGP key is here if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io)
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
The documentation for the api can be found [here](https://lbry.tech/api/sdk).
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
+3 -3
View File
@@ -100,7 +100,7 @@
<div class="md-flex__cell md-flex__cell--shrink">
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
</a>
</div>
@@ -196,7 +196,7 @@
<label class="md-nav__title md-nav__title--site" for="drawer">
<span class="md-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
</span>
LBRY
@@ -304,4 +304,4 @@
</body>
</html>
</html>
+2768 -1446
View File
File diff suppressed because it is too large. Load diff
+3 -5
View File
@@ -1409,7 +1409,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1903,7 +1903,7 @@ Options:
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address wil automatically be created
new address will automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1954,7 +1954,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1979,7 +1978,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2586,4 +2584,4 @@ Returns:
</body>
</html>
</html>
+3 -5
View File
@@ -1339,7 +1339,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1743,7 +1743,7 @@ Args:
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address wil automatically be created
new address will automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1786,7 +1786,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1811,7 +1810,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2312,4 +2310,4 @@ Returns:
</body>
</html>
</html>
+4 -4
View File
@@ -2,7 +2,7 @@
"docs": [
{
"location": "/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"title": "API"
},
{
@@ -167,7 +167,7 @@
},
{
"location": "/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
@@ -292,7 +292,7 @@
},
{
"location": "/cli/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"title": "CLI"
},
{
@@ -457,7 +457,7 @@
},
{
"location": "/cli/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
+25
View File
@@ -0,0 +1,25 @@
# This is an example daemon_settings.yml file.
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.com:50001
- lbryumx2.lbry.com:50001
- lbryumx4.lbry.com:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
save_blobs: true
save_files: false
dht_node_port: 4444
peer_port: 3333
use_upnp: true
#components_to_skip:
# - hash_announcer
# - blob_server
# - dht
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 6.1 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 97 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 361 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 5.3 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

+2 -4
View File
@@ -1,6 +1,4 @@
import logging
__version__ = "0.30.0rc3"
__name__ = "lbrynet"
__version__ = "0.38.0rc6"
version = tuple(__version__.split('.'))
logging.getLogger(__name__).addHandler(logging.NullHandler())
-293
View File
@@ -1,293 +0,0 @@
import collections
import logging
import treq
from twisted.internet import defer, task
from lbrynet import conf
from lbrynet.core import looping_call_manager, utils, system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
SERVER_STARTUP_ERROR = 'Server Startup Error'
DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
log = logging.getLogger(__name__)
class Manager:
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
self.analytics_api = analytics_api
self._tracked_data = collections.defaultdict(list)
self.looping_call_manager = self._setup_looping_calls()
self.context = context or self._make_context(
system_info.get_platform(), conf.settings['wallet'])
self.installation_id = installation_id or conf.settings.installation_id
self.session_id = session_id or conf.settings.get_session_id()
self.is_started = False
@classmethod
def new_instance(cls, enabled=None):
api = Api.new_instance(enabled)
return cls(api)
# Things We Track
def send_new_download_start(self, download_id, name, claim_dict):
self._send_new_download_stats("start", download_id, name, claim_dict)
def send_new_download_success(self, download_id, name, claim_dict):
self._send_new_download_stats("success", download_id, name, claim_dict)
def send_new_download_fail(self, download_id, name, claim_dict, e):
self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
self.analytics_api.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
def send_server_startup(self):
self.analytics_api.track(self._event(SERVER_STARTUP))
def send_server_startup_success(self):
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
def send_server_startup_error(self, message):
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
def send_download_started(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
def send_claim_action(self, action):
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
def send_new_channel(self):
self.analytics_api.track(self._event(NEW_CHANNEL))
def send_credits_sent(self):
self.analytics_api.track(self._event(CREDITS_SENT))
def _send_heartbeat(self):
self.analytics_api.track(self._event(HEARTBEAT))
def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
# Setup / Shutdown
def start(self):
if not self.is_started:
for name, _, interval in self._get_looping_calls():
self.looping_call_manager.start(name, interval)
self.is_started = True
def shutdown(self):
self.looping_call_manager.shutdown()
def register_repeating_metric(self, event_name, value_generator, frequency=300):
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
self.looping_call_manager.register_looping_call(event_name, lcall)
lcall.start(frequency)
def _get_looping_calls(self):
return [
('send_heartbeat', self._send_heartbeat, 60),
('update_tracked_metrics', self._update_tracked_metrics, 300),
]
def _setup_looping_calls(self):
call_manager = looping_call_manager.LoopingCallManager()
for name, fn, _ in self._get_looping_calls():
call_manager.register_looping_call(name, task.LoopingCall(fn))
return call_manager
def _send_repeating_metric(self, event_name, value_generator):
result = value_generator()
self._if_deferred(result, self._send_repeating_metric_value, event_name)
def _send_repeating_metric_value(self, result, event_name):
should_send, value = result
if should_send:
self.analytics_api.track(self._metric_event(event_name, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
@staticmethod
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
if isinstance(maybe_deferred, defer.Deferred):
maybe_deferred.addCallback(callback, *args, **kwargs)
else:
callback(maybe_deferred, *args, **kwargs)
class Api:
def __init__(self, cookies, url, write_key, enabled):
self.cookies = cookies
self.url = url
self._write_key = write_key
self._enabled = enabled
def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
def update_cookies(response):
self.cookies.update(response.cookies())
return response
assert endpoint[0] == '/'
headers = {b"Connection": b"close"}
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
headers=headers, cookies=self.cookies)
d.addCallback(update_cookies)
return d
def track(self, event):
"""Send a single tracking event"""
if not self._enabled:
return defer.succeed('Analytics disabled')
def _log_error(failure, event):
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
log.debug('Sending track event: %s', event)
d = self._post('/track', event)
d.addErrback(_log_error, event)
return d
@classmethod
def new_instance(cls, enabled=None):
"""Initialize an instance using values from the configuration"""
if enabled is None:
enabled = conf.settings['share_usage_data']
return cls(
{},
conf.settings['ANALYTICS_ENDPOINT'],
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
enabled,
)
-1
View File
@@ -1 +0,0 @@
from . import paths
-26
View File
@@ -1,26 +0,0 @@
# App root files dir
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
def android_files_dir():
return None
# Base internal storage path
def android_internal_storage_dir():
return None
# Base external (SD card, if present) storage path
def android_external_storage_dir():
return None
# Internal device storage (private app folder)
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_internal_storage_dir():
return None
# External (app folder on SD card, if present) storage
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_external_storage_dir():
return None
+6 -4
View File
@@ -1,4 +1,6 @@
from .blob_file import BlobFile
from .creator import BlobFileCreator
from .writer import HashBlobWriter
from .reader import HashBlobReader
from lbrynet.cryptoutils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
+300 -174
View File
@@ -1,209 +1,335 @@
import logging
import os
from twisted.internet import defer, threads
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.core.utils import is_valid_blobhash
import re
import asyncio
import binascii
import logging
import typing
import contextlib
from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
log = logging.getLogger(__name__)
MAX_BLOB_SIZE = 2 * 2 ** 20
_hexmatch = re.compile("^[a-f,0-9]+$")
class BlobFile:
def is_valid_blobhash(blobhash: str) -> bool:
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
A chunk of data available on the network which is specified by a hashsum
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
This class is used to create blobs on the local filesystem
when we already know the blob hash before hand (i.e., when downloading blobs)
Also can be used for reading from blobs on the local filesystem
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
padder = PKCS7(AES.block_size).padder()
encryptor = cipher.encryptor()
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
digest = get_lbry_hash_obj()
digest.update(encrypted)
return encrypted, digest.hexdigest()
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
if len(data) != length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
class AbstractBlob:
"""
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
def __str__(self):
return self.blob_hash[:16]
This class is non-io specific
"""
__slots__ = [
'loop',
'blob_hash',
'length',
'blob_completed_callback',
'blob_directory',
'writers',
'verified',
'writing',
'readers'
]
def __repr__(self):
return '<{}({})>'.format(self.__class__.__name__, str(self))
def __init__(self, blob_dir, blob_hash, length=None):
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self.loop = loop
self.blob_hash = blob_hash
self.length = length
self.writers = {} # {Peer: writer, finished_deferred}
self._verified = False
self.readers = 0
self.blob_dir = blob_dir
self.file_path = os.path.join(blob_dir, self.blob_hash)
self.blob_write_lock = defer.DeferredLock()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
self.set_length(os.path.getsize(self.file_path))
# This assumes that the hash of the blob has already been
# checked as part of the blob creation process. It might
# be worth having a function that checks the actual hash;
# its probably too expensive to have that check be part of
# this call.
self._verified = True
self.blob_completed_callback = blob_completed_callback
self.blob_directory = blob_directory
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
self.readers: typing.List[typing.BinaryIO] = []
def open_for_writing(self, peer):
"""
open a blob file to be written by peer, supports concurrent
writers, as long as they are from differnt peers.
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
returns tuple of (writer, finished_deferred)
def __del__(self):
if self.writers or self.readers:
log.warning("%s not closed before being garbage collected", self.blob_hash)
self.close()
writer - a file like object with a write() function, close() when finished
finished_deferred - deferred that is fired when write is finished and returns
a instance of itself as HashBlob
"""
if peer not in self.writers:
log.debug("Opening %s to be written by %s", str(self), str(peer))
finished_deferred = defer.Deferred()
writer = HashBlobWriter(self.get_length, self.writer_finished)
self.writers[peer] = (writer, finished_deferred)
return writer, finished_deferred
log.warning("Tried to download the same file twice simultaneously from the same peer")
return None, None
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
raise NotImplementedError()
def open_for_reading(self):
"""
open blob for reading
@contextlib.contextmanager
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
with self._reader_context() as reader:
try:
self.readers.append(reader)
yield reader
finally:
if reader in self.readers:
self.readers.remove(reader)
returns a file like object that can be read() from, and closed() when
finished
"""
if self._verified is True:
f = open(self.file_path, 'rb')
reader = HashBlobReader(f, self.reader_finished)
self.readers += 1
return reader
return None
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
def delete(self):
"""
delete blob file from file system, prevent deletion
if a blob is being read from or written to
returns a deferred that firesback when delete is completed
"""
if not self.writers and not self.readers:
self._verified = False
self.saved_verified_blob = False
def delete_from_file_system():
if os.path.isfile(self.file_path):
os.remove(self.file_path)
d = threads.deferToThread(delete_from_file_system)
def log_error(err):
log.warning("An error occurred deleting %s: %s",
str(self.file_path), err.getErrorMessage())
return err
d.addErrback(log_error)
return d
else:
return defer.fail(Failure(
ValueError("File is currently being read or written and cannot be deleted")))
@property
def verified(self):
"""
Protect verified from being modified by other classes.
verified is True if a write to a blob has completed succesfully,
or a blob has been read to have the same length as specified
in init
"""
return self._verified
def set_length(self, length):
def set_length(self, length) -> None:
if self.length is not None and length == self.length:
return True
return
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
self.length = length
return True
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
self.length, length)
return False
return
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
def get_length(self):
def get_length(self) -> typing.Optional[int]:
return self.length
def get_is_verified(self):
return self.verified
def get_is_verified(self) -> bool:
return self.verified.is_set()
def is_downloading(self):
if self.writers:
return True
return False
def is_readable(self) -> bool:
return self.verified.is_set()
def reader_finished(self, reader):
self.readers -= 1
return defer.succeed(True)
def is_writeable(self) -> bool:
return not self.writing.is_set()
def writer_finished(self, writer, err=None):
def fire_finished_deferred():
self._verified = True
for p, (w, finished_deferred) in list(self.writers.items()):
if w == writer:
del self.writers[p]
finished_deferred.callback(self)
return True
log.warning(
"Somehow, the writer that was accepted as being valid was already removed: %s",
writer)
return False
def write_blob(self, blob_bytes: bytes):
if not self.is_writeable():
raise OSError("cannot open blob for writing")
try:
self.writing.set()
self._write_blob(blob_bytes)
finally:
self.writing.clear()
def errback_finished_deferred(err):
for p, (w, finished_deferred) in list(self.writers.items()):
if w == writer:
del self.writers[p]
finished_deferred.errback(err)
def close(self):
while self.writers:
peer, writer = self.writers.popitem()
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
writer.finished.cancel()
while self.readers:
reader = self.readers.pop()
if reader:
reader.close()
def cancel_other_downloads():
for p, (w, finished_deferred) in self.writers.items():
w.close()
def delete(self):
self.close()
self.verified.clear()
self.length = None
if err is None:
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
if self._verified is False:
d = self.save_verified_blob(writer)
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
d.addCallback(lambda _: cancel_other_downloads())
else:
d = defer.succeed(None)
fire_finished_deferred()
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
"""
if not self.is_readable():
raise OSError('blob files cannot be read')
with self.reader_context() as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with self.reader_context() as reader:
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
writer = blob.get_blob_writer()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers[(peer_address, peer_port)] = writer
def remove_writer(_):
if (peer_address, peer_port) in self.writers:
del self.writers[(peer_address, peer_port)]
fut.add_done_callback(remove_writer)
def writer_finished_callback(finished: asyncio.Future):
try:
err = finished.exception()
if err:
raise err
verified_bytes = finished.result()
while self.writers:
_, other = self.writers.popitem()
if other is not writer:
other.close_handle()
self.save_verified_blob(verified_bytes)
except (InvalidBlobHashError, InvalidDataError) as error:
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
pass
fut.add_done_callback(writer_finished_callback)
return writer
class BlobBuffer(AbstractBlob):
"""
An in-memory only blob
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self._verified_bytes: typing.Optional[BytesIO] = None
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError("cannot open blob for reading")
try:
yield self._verified_bytes
finally:
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
return super().delete()
def __del__(self):
super().__del__()
if self._verified_bytes:
self.delete()
class BlobFile(AbstractBlob):
"""
A blob existing on the local file system
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
if not blob_directory or not os.path.isdir(blob_directory):
raise OSError(f"invalid blob directory '{blob_directory}'")
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
if self.file_exists:
file_size = int(os.stat(self.file_path).st_size)
if length and length != file_size:
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
self.delete()
else:
if writer.len_so_far != self.length:
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
else:
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
errback_finished_deferred(Failure(InvalidDataError(err_string)))
d = defer.succeed(None)
else:
errback_finished_deferred(err)
d = defer.succeed(None)
d.addBoth(lambda _: writer.close_handle())
return d
self.length = file_size
self.verified.set()
def save_verified_blob(self, writer):
# we cannot have multiple _save_verified_blob interrupting
# each other, can happen since startProducing is a deferred
return self.blob_write_lock.run(self._save_verified_blob, writer)
@property
def file_exists(self):
return os.path.isfile(self.file_path)
@defer.inlineCallbacks
def _save_verified_blob(self, writer):
if self.saved_verified_blob is False:
writer.write_handle.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(writer.write_handle)
yield producer.startProducing(open(out_path, 'wb'))
self.saved_verified_blob = True
defer.returnValue(True)
else:
raise DownloadCanceledError()
def is_writeable(self) -> bool:
return super().is_writeable() and not os.path.isfile(self.file_path)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
return super().get_blob_writer(peer_address, peer_port)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
handle = open(self.file_path, 'rb')
try:
yield handle
finally:
handle.close()
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def delete(self):
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> BlobInfo:
if not blob_dir or not os.path.isdir(blob_dir):
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
return await super().create_from_unencrypted(
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
)
+26
View File
@@ -0,0 +1,26 @@
import typing
class BlobInfo:
__slots__ = [
'blob_hash',
'blob_num',
'length',
'iv',
]
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
self.iv = iv
def as_dict(self) -> typing.Dict:
d = {
'length': self.length,
'blob_num': self.blob_num,
'iv': self.iv,
}
if self.blob_hash: # non-terminator blobs have a blob hash
d['blob_hash'] = self.blob_hash
return d
+135
View File
@@ -0,0 +1,135 @@
import os
import typing
import asyncio
import logging
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
blob_dir - directory where blobs are stored
storage - SQLiteStorage object
"""
self.loop = loop
self.blob_dir = blob_dir
self.storage = storage
self._node_data_store = node_data_store
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
buffer = self.blobs.pop(blob_hash)
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
if buffer.is_readable():
with buffer.reader_context() as reader:
self.blobs[blob_hash].write_blob(reader.read())
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
return self.blobs[blob_hash]
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
def get_files_in_blob_dir() -> typing.Set[str]:
if not self.blob_dir:
return set()
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
self.connection_manager.start()
return True
def stop(self):
self.connection_manager.stop()
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
self.completed_blob_hashes.clear()
def get_stream_descriptor(self, sd_hash):
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
if blob.blob_hash is None:
raise Exception("Blob hash is None")
if not blob.length:
raise Exception("Blob has a length of 0")
if isinstance(blob, BlobFile):
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
else:
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
if blob_hash not in self.blobs:
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
for blob_hash in blob_hashes:
self.delete_blob(blob_hash)
if delete_from_db:
await self.storage.delete_blobs_from_db(blob_hashes)
-51
View File
@@ -1,51 +0,0 @@
import os
import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class BlobFileCreator:
"""
This class is used to create blobs on the local filesystem
when we do not know the blob hash beforehand (i.e, when creating
a new stream)
"""
def __init__(self, blob_dir):
self.blob_dir = blob_dir
self.buffer = BytesIO()
self._is_open = True
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.blob_hash = None
self.length = None
@defer.inlineCallbacks
def close(self):
self.length = self.len_so_far
self.blob_hash = self._hashsum.hexdigest()
if self.blob_hash and self._is_open and self.length > 0:
# do not save 0 length files (empty tail blob in streams)
# or if its been closed already
self.buffer.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(self.buffer)
yield producer.startProducing(open(out_path, 'wb'))
self._is_open = False
if self.length > 0:
defer.returnValue(self.blob_hash)
else:
# 0 length files (empty tail blob in streams )
# must return None as their blob_hash for
# it to be saved properly by EncryptedFileMetadataManagers
defer.returnValue(None)
def write(self, data):
if not self._is_open:
raise IOError
self._hashsum.update(data)
self.len_so_far += len(data)
self.buffer.write(data)
-30
View File
@@ -1,30 +0,0 @@
import logging
log = logging.getLogger(__name__)
class HashBlobReader:
"""
This is a file like reader class that supports
read(size) and close()
"""
def __init__(self, read_handle, finished_cb):
self.finished_cb = finished_cb
self.finished_cb_d = None
self.read_handle = read_handle
def __del__(self):
if self.finished_cb_d is None:
log.warning("Garbage collection was called, but reader for %s was not closed yet",
self.read_handle.name)
self.close()
def read(self, size=-1):
return self.read_handle.read(size)
def close(self):
# if we've already closed and called finished_cb, do nothing
if self.finished_cb_d is not None:
return
self.read_handle.close()
self.finished_cb_d = self.finished_cb(self)
+46 -36
View File
@@ -1,58 +1,68 @@
import typing
import logging
import asyncio
from io import BytesIO
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class HashBlobWriter:
def __init__(self, length_getter, finished_cb):
self.write_handle = BytesIO()
self.length_getter = length_getter
self.finished_cb = finished_cb
self.finished_cb_d = None
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
finished: asyncio.Future):
self.expected_blob_hash = expected_blob_hash
self.get_length = get_length
self.buffer = BytesIO()
self.finished = finished
self.finished.add_done_callback(lambda *_: self.close_handle())
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
def __del__(self):
if self.finished_cb_d is None:
if self.buffer is not None:
log.warning("Garbage collection was called, but writer was not closed yet")
self.close()
self.close_handle()
@property
def blob_hash(self):
def calculate_blob_hash(self) -> str:
return self._hashsum.hexdigest()
def write(self, data):
if self.write_handle is None:
def closed(self):
return self.buffer is None or self.buffer.closed
def write(self, data: bytes):
expected_length = self.get_length()
if not expected_length:
raise IOError("unknown blob length")
if self.buffer is None:
log.warning("writer has already been closed")
if not self.finished.done():
self.finished.cancel()
return
raise IOError('I/O operation on closed file')
self._hashsum.update(data)
self.len_so_far += len(data)
if self.len_so_far > self.length_getter():
self.finished_cb_d = self.finished_cb(
self,
Failure(InvalidDataError("Length so far is greater than the expected length."
" %s to %s" % (self.len_so_far,
self.length_getter()))))
else:
self.write_handle.write(data)
if self.len_so_far == self.length_getter():
self.finished_cb_d = self.finished_cb(self)
if self.len_so_far > expected_length:
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
self.close_handle()
return
self.buffer.write(data)
if self.len_so_far == expected_length:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(self.buffer.getvalue())
self.close_handle()
def close_handle(self):
if self.write_handle is not None:
self.write_handle.close()
self.write_handle = None
def close(self, reason=None):
# if we've already called finished_cb because we either finished writing
# or closed already, do nothing
if self.finished_cb_d is not None:
return
if reason is None:
reason = Failure(DownloadCanceledError())
self.finished_cb_d = self.finished_cb(self, reason)
if not self.finished.done():
self.finished.cancel()
if self.buffer is not None:
self.buffer.close()
self.buffer = None
File renamed without changes.
+245
View File
@@ -0,0 +1,245 @@
import asyncio
import logging
import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.connection_manager import ConnectionManager
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
connection_manager: typing.Optional['ConnectionManager'] = None):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.transport: typing.Optional[asyncio.Transport] = None
self.peer_timeout = peer_timeout
self.connection_manager = connection_manager
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
self._blob_bytes_received = 0
self._response_fut: typing.Optional[asyncio.Future] = None
self.buf = b''
# this is here to handle the race when the downloader is closed right as response_fut gets a result
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
if self.connection_manager:
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
# assert self.peer_address is not None
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if not self._response_fut:
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
return self.close()
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
response = BlobResponse.deserialize(self.buf + data)
if not response.responses and not self._response_fut.done():
self.buf += data
return
else:
self.buf = b''
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def _write(self, data: bytes):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
else:
data = data
self._blob_bytes_received += len(data)
try:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
except (asyncio.TimeoutError) as err: # TODO: is this needed?
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
log.info(msg)
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
def close(self):
self.closed.set()
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
self._response_fut = None
self.writer = None
self.blob = None
if self.transport:
self.transport.close()
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
self.closed.clear()
blob_hash = blob.blob_hash
if blob.get_is_verified() or not blob.is_writeable():
return 0, self.transport
try:
self._blob_bytes_received = 0
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
# i'm not sure how to fix this race condition - jack
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
self.close()
return self._blob_bytes_received, None
except asyncio.CancelledError:
self.close()
raise
finally:
if self.writer and not self.writer.closed():
self.writer.close_handle()
self.writer = None
def connection_made(self, transport: asyncio.Transport):
addr = transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr[0], addr[1]
self.transport = transport
if self.connection_manager:
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(
loop, blob_download_timeout, connection_manager
)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.transport = connected_transport
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
try:
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or not blob.is_writeable():
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return 0, None
+130
View File
@@ -0,0 +1,130 @@
import asyncio
import typing
import logging
from lbrynet.utils import cache_concurrent
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
log = logging.getLogger(__name__)
class BlobDownloader:
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.peer_queue = peer_queue
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.failures: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
if blob.get_is_verified():
return
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
)
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = self.failures.get(peer, 0) + 1
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = 0
self.connections[peer] = transport
elapsed = self.loop.time() - start
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
def cleanup_active(self):
if not self.active_connections and not self.connections:
self.clearbanned()
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
def clearbanned(self):
now = self.loop.time()
self.ignored = dict((
(peer, when) for (peer, when) in self.ignored.items()
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
))
@cache_concurrent
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
connection_id: int = 0) -> 'AbstractBlob':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
self.is_running.set()
try:
while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set()
while not self.peer_queue.empty():
batch.update(self.peer_queue.get_nowait())
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
await self.new_peer_or_finished()
self.cleanup_active()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
blob.close()
def close(self):
self.scores.clear()
self.ignored.clear()
self.is_running.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
blob_hash: str) -> 'AbstractBlob':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try:
return await downloader.download_blob(blob_hash)
finally:
if accumulate_task and not accumulate_task.done():
accumulate_task.cancel()
downloader.close()
+283
View File
@@ -0,0 +1,283 @@
import typing
import json
import logging
log = logging.getLogger(__name__)
class BlobMessage:
key = ''
def to_dict(self) -> typing.Dict:
raise NotImplementedError()
class BlobPriceRequest(BlobMessage):
key = 'blob_data_payment_rate'
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobPriceResponse(BlobMessage):
key = 'blob_data_payment_rate'
rate_accepted = 'RATE_ACCEPTED'
rate_too_low = 'RATE_TOO_LOW'
rate_unset = 'RATE_UNSET'
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
raise ValueError(blob_data_payment_rate)
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobAvailabilityRequest(BlobMessage):
key = 'requested_blobs'
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
**kwargs) -> None:
assert len(requested_blobs)
self.requested_blobs = requested_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blobs,
'lbrycrd_address': self.lbrycrd_address
}
class BlobAvailabilityResponse(BlobMessage):
key = 'available_blobs'
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
**kwargs) -> None:
self.available_blobs = available_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
d = {
self.key: self.available_blobs
}
if self.lbrycrd_address:
d['lbrycrd_address'] = self.lbrycrd_address
return d
class BlobDownloadRequest(BlobMessage):
key = 'requested_blob'
def __init__(self, requested_blob: str, **kwargs) -> None:
self.requested_blob = requested_blob
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blob
}
class BlobDownloadResponse(BlobMessage):
key = 'incoming_blob'
def __init__(self, **response: typing.Dict) -> None:
incoming_blob = response[self.key]
self.error = None
self.incoming_blob = None
if 'error' in incoming_blob:
self.error = incoming_blob['error']
else:
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
self.length = None if not self.incoming_blob else self.incoming_blob['length']
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
def to_dict(self) -> typing.Dict:
return {
self.key if not self.error else 'error': self.incoming_blob or self.error,
}
class BlobPaymentAddressRequest(BlobMessage):
key = 'lbrycrd_address'
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.lbrycrd_address
}
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
pass
class BlobErrorResponse(BlobMessage):
key = 'error'
def __init__(self, error: str, **kwargs) -> None:
self.error = error
def to_dict(self) -> typing.Dict:
return {
self.key: self.error
}
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest]
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse]
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
# scenarios:
# <json>
# <blob bytes>
# <json><blob bytes>
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
return None, response_msg
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
except ValueError:
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
return response, response_msg[curr_pos:]
return None, response_msg
class BlobRequest:
def __init__(self, requests: typing.List[blob_request_types]) -> None:
self.requests = requests
def to_dict(self):
d = {}
for request in self.requests:
d.update(request.to_dict())
return d
def _get_request(self, request_type: blob_request_types):
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
if request:
return request[0]
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
response = self._get_request(BlobAvailabilityRequest)
if response:
return response
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
response = self._get_request(BlobPriceRequest)
if response:
return response
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
response = self._get_request(BlobDownloadRequest)
if response:
return response
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
response = self._get_request(BlobPaymentAddressRequest)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobRequest':
request = json.loads(data)
return cls([
request_type(**request)
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest)
if request_type.key in request
])
@classmethod
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
return cls(
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
)
class BlobResponse:
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
self.responses = responses
self.blob_data = blob_data
def to_dict(self):
d = {}
for response in self.responses:
d.update(response.to_dict())
return d
def _get_response(self, response_type: blob_response_types):
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
if response:
return response[0]
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
error = self._get_response(BlobErrorResponse)
if error:
log.error(error)
return error
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
response = self._get_response(BlobAvailabilityResponse)
if response:
return response
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
response = self._get_response(BlobPriceResponse)
if response:
return response
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
response = self._get_response(BlobDownloadResponse)
if response:
return response
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
response = self._get_response(BlobPaymentAddressResponse)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobResponse':
response, extra = _parse_blob_response(data)
requests = []
if response:
requests.extend([
response_type(**response)
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse)
if response_type.key in response
])
return cls(requests, extra)
+134
View File
@@ -0,0 +1,134 @@
import asyncio
import binascii
import logging
import typing
from json.decoder import JSONDecodeError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobManager
log = logging.getLogger(__name__)
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None
def connection_made(self, transport):
self.transport = transport
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
serialized = BlobResponse(to_send).serialize()
self.transport.write(serialized)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
responses = []
address_request = request.get_address_request()
if address_request:
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
availability_request = request.get_availability_request()
if availability_request:
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
availability_request.requested_blobs)
)))))
price_request = request.get_price_request()
if price_request:
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
download_request = request.get_blob_request()
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
if self.transport:
self.transport.close()
return
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
# self.transport.close()
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
try:
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
len(data), '' if not data else binascii.hexlify(data).decode())
if not request:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
self.transport.close()
return
self.loop.create_task(self.handle_request(request))
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
raise Exception("already running")
async def _start_server():
server = await self.loop.create_server(
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
log.info("Blob server listening on TCP %s:%i", interface, port)
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
def stop_server(self):
if self.server_task:
self.server_task.cancel()
self.server_task = None
log.info("Stopped blob server")
+2 -1
View File
@@ -1,2 +1,3 @@
# dont touch this. CI server changes this during build/deployment
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
-168
View File
@@ -1,168 +0,0 @@
import sys
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
from twisted.internet import reactor
import json
import asyncio
from aiohttp.client_exceptions import ClientConnectorError
from requests.exceptions import ConnectionError
from docopt import docopt
from textwrap import dedent
from lbrynet.daemon.Daemon import Daemon
from lbrynet.daemon.DaemonControl import start as daemon_main
from lbrynet.daemon.DaemonConsole import main as daemon_console
from lbrynet.daemon.auth.client import LBRYAPIClient
from lbrynet.core.system_info import get_platform
async def execute_command(method, params, conf_path=None):
# this check if the daemon is running or not
try:
api = await LBRYAPIClient.get_client(conf_path)
await api.status()
except (ClientConnectorError, ConnectionError):
await api.session.close()
print("Could not connect to daemon. Are you sure it's running?")
return 1
# this actually executes the method
try:
resp = await api.call(method, params)
await api.session.close()
print(json.dumps(resp["result"], indent=2))
except KeyError:
if resp["error"]["code"] == -32500:
print(json.dumps(resp["error"], indent=2))
else:
print(json.dumps(resp["error"]["message"], indent=2))
def print_help():
print(dedent("""
NAME
lbrynet - LBRY command line client.
USAGE
lbrynet [--conf <config file>] <command> [<args>]
EXAMPLES
lbrynet commands # list available commands
lbrynet status # get daemon status
lbrynet --conf ~/l1.conf status # like above but using ~/l1.conf as config file
lbrynet resolve_name what # resolve a name
lbrynet help resolve_name # get help for a command
"""))
def print_help_for_command(command):
fn = Daemon.callable_methods.get(command)
if fn:
print(dedent(fn.__doc__))
else:
print("Invalid command name")
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
if x.lower() == 'false':
return False
if x.isdigit():
return int(x)
return x
def remove_brackets(key):
if key.startswith("<") and key.endswith(">"):
return str(key[1:-1])
return key
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
k = None
if arg is None:
continue
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
kwargs[k] = normalize_value(arg, k)
return kwargs
def main(argv=None):
argv = argv or sys.argv[1:]
if not argv:
print_help()
return 1
conf_path = None
if len(argv) and argv[0] == "--conf":
if len(argv) < 2:
print("No config file specified for --conf option")
print_help()
return 1
conf_path = argv[1]
argv = argv[2:]
method, args = argv[0], argv[1:]
if method in ['help', '--help', '-h']:
if len(args) == 1:
print_help_for_command(args[0])
else:
print_help()
return 0
elif method in ['version', '--version', '-v']:
print(json.dumps(get_platform(get_ip=False), sort_keys=True, indent=2, separators=(',', ': ')))
return 0
elif method == 'start':
sys.exit(daemon_main(args, conf_path))
elif method == 'console':
sys.exit(daemon_console())
elif method not in Daemon.callable_methods:
if method not in Daemon.deprecated_methods:
print('{} is not a valid command.'.format(method))
return 1
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print("{} is permanently deprecated and does not have a replacement command.".format(method))
return 0
print("{} is deprecated, using {}.".format(method, new_method))
method = new_method
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(method, params, conf_path))
return 0
if __name__ == "__main__":
sys.exit(main())
+595 -607
View File
File diff suppressed because it is too large. Load diff
+90
View File
@@ -0,0 +1,90 @@
import asyncio
import typing
import collections
import logging
log = logging.getLogger(__name__)
CONNECTED_EVENT = "connected"
DISCONNECTED_EVENT = "disconnected"
TRANSFERRED_EVENT = "transferred"
class ConnectionManager:
def __init__(self, loop: asyncio.AbstractEventLoop):
self.loop = loop
self.incoming_connected: typing.Set[str] = set()
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._task: typing.Optional[asyncio.Task] = None
@property
def status(self):
return self._status
def sent_data(self, host_and_port: str, size: int):
self.outgoing[host_and_port] += size
def received_data(self, host_and_port: str, size: int):
self.incoming[host_and_port] += size
def connection_made(self, host_and_port: str):
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
def outgoing_connection_lost(self, host_and_port: str):
if host_and_port in self.outgoing_connected:
self.outgoing_connected.remove(host_and_port)
def incoming_connection_lost(self, host_and_port: str):
if host_and_port in self.incoming_connected:
self.incoming_connected.remove(host_and_port)
async def _update(self):
self._status = {
'incoming_bps': {},
'outgoing_bps': {},
'total_incoming_mbs': 0.0,
'total_outgoing_mbs': 0.0,
'time': self.loop.time()
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
) / (now - last)) / 1000000.0
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
) / (now - last)) / 1000000.0
self._status['time'] = now
def stop(self):
if self._task:
self._task.cancel()
self._task = None
self.outgoing.clear()
self.outgoing_connected.clear()
self.incoming.clear()
self.incoming_connected.clear()
self._status.clear()
def start(self):
self.stop()
self._task = self.loop.create_task(self._update())
-96
View File
@@ -1,96 +0,0 @@
import logging
import random
import time
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from decimal import Decimal
log = logging.getLogger(__name__)
class BlobAvailabilityTracker:
"""
Class to track peer counts for known blobs, and to discover new popular blobs
Attributes:
availability (dict): dictionary of peers for known blobs
"""
def __init__(self, blob_manager, peer_finder, dht_node):
self.availability = {}
self._last_mean_availability = Decimal(0.0)
self._blob_manager = blob_manager
self._peer_finder = peer_finder
self._dht_node = dht_node
self._check_mine = LoopingCall(self._update_mine)
def start(self):
log.info("Starting blob availability tracker.")
self._check_mine.start(600)
def stop(self):
log.info("Stopping blob availability tracker.")
if self._check_mine.running:
self._check_mine.stop()
def get_blob_availability(self, blob, timeout=None):
def _get_peer_count(peers):
have_blob = sum(1 for peer in peers if peer.is_available())
return {blob: have_blob}
d = self._peer_finder.find_peers_for_blob(blob, timeout)
d.addCallback(_get_peer_count)
return d
def get_availability_for_blobs(self, blobs, timeout=None):
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
d = defer.DeferredList(dl)
d.addCallback(lambda results: [val for success, val in results if success])
return d
@property
def last_mean_availability(self):
return max(Decimal(0.01), self._last_mean_availability)
def _update_peers_for_blob(self, blob):
def _save_peer_info(blob_hash, peers):
v = {blob_hash: peers}
self.availability.update(v)
return v
d = self._peer_finder.find_peers_for_blob(blob)
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
d.addCallback(lambda peers: _save_peer_info(blob, peers))
return d
def _update_mine(self):
def _get_peers(blobs):
dl = []
for hash in blobs:
dl.append(self._update_peers_for_blob(hash))
return defer.DeferredList(dl)
def sample(blobs):
return random.sample(blobs, min(len(blobs), 10))
start = time.time()
log.debug('==> Updating the peers for my blobs')
d = self._blob_manager.get_all_verified_blobs()
# as far as I can tell, this only is used to set _last_mean_availability
# which... seems like a very expensive operation for such little payoff.
# so taking a sample should get about the same effect as querying the entire
# list of blobs
d.addCallback(sample)
d.addCallback(_get_peers)
d.addCallback(lambda _: self._set_mean_peers())
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
time.time() - start))
# although unused, need to return or else the looping call
# could overrun on a previous call
return d
def _set_mean_peers(self):
num_peers = [len(self.availability[blob]) for blob in self.availability]
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
self._last_mean_availability = mean
-18
View File
@@ -1,18 +0,0 @@
class BlobInfo:
"""
This structure is used to represent the metadata of a blob.
@ivar blob_hash: The sha384 hashsum of the blob's data.
@type blob_hash: string, hex-encoded
@ivar blob_num: For streams, the position of the blob in the stream.
@type blob_num: integer
@ivar length: The length of the blob in bytes.
@type length: integer
"""
def __init__(self, blob_hash, blob_num, length):
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
-139
View File
@@ -1,139 +0,0 @@
import logging
import os
from binascii import unhexlify
from sqlite3 import IntegrityError
from twisted.internet import threads, defer
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.creator import BlobFileCreator
log = logging.getLogger(__name__)
class DiskBlobManager:
def __init__(self, blob_dir, storage, node_datastore=None):
"""
This class stores blobs on the hard disk
blob_dir - directory where blobs are stored
storage - SQLiteStorage object
"""
self.storage = storage
self.blob_dir = blob_dir
self._node_datastore = node_datastore
self.blob_creator_type = BlobFileCreator
# TODO: consider using an LRU for blobs as there could potentially
# be thousands of blobs loaded up, many stale
self.blobs = {}
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
@defer.inlineCallbacks
def setup(self):
if self._node_datastore is not None:
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
self._node_datastore.completed_blobs.update(raw_blob_hashes)
defer.returnValue(True)
def stop(self):
return defer.succeed(True)
def get_blob(self, blob_hash, length=None):
"""Return a blob identified by blob_hash, which may be a new blob or a
blob that is already on the hard disk
"""
if length is not None and not isinstance(length, int):
raise Exception("invalid length type: %s (%s)" % (length, str(type(length))))
if blob_hash in self.blobs:
return defer.succeed(self.blobs[blob_hash])
return self._make_new_blob(blob_hash, length)
def get_blob_creator(self):
return self.blob_creator_type(self.blob_dir)
def _make_new_blob(self, blob_hash, length=None):
log.debug('Making a new blob for %s', blob_hash)
blob = BlobFile(self.blob_dir, blob_hash, length)
self.blobs[blob_hash] = blob
return defer.succeed(blob)
@defer.inlineCallbacks
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
yield self.storage.add_completed_blob(
blob.blob_hash, blob.length, next_announce_time, should_announce
)
if self._node_datastore is not None:
self._node_datastore.completed_blobs.add(unhexlify(blob.blob_hash))
def completed_blobs(self, blobhashes_to_check):
return self._completed_blobs(blobhashes_to_check)
def count_should_announce_blobs(self):
return self.storage.count_should_announce_blobs()
def set_should_announce(self, blob_hash, should_announce):
now = self.storage.clock.seconds()
return self.storage.set_should_announce(blob_hash, now, should_announce)
def get_should_announce(self, blob_hash):
return self.storage.should_announce(blob_hash)
def creator_finished(self, blob_creator, should_announce):
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
if blob_creator.blob_hash is None:
raise Exception("Blob hash is None")
if blob_creator.blob_hash in self.blobs:
raise Exception("Creator finished for blob that is already marked as completed")
if blob_creator.length is None:
raise Exception("Blob has a length of 0")
new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
self.blobs[blob_creator.blob_hash] = new_blob
return self.blob_completed(new_blob, should_announce)
def get_all_verified_blobs(self):
d = self._get_all_verified_blob_hashes()
d.addCallback(self.completed_blobs)
return d
@defer.inlineCallbacks
def delete_blobs(self, blob_hashes):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
if not blob_hash:
continue
if self._node_datastore is not None:
try:
self._node_datastore.completed_blobs.remove(unhexlify(blob_hash))
except KeyError:
pass
try:
blob = yield self.get_blob(blob_hash)
yield blob.delete()
bh_to_delete_from_db.append(blob_hash)
del self.blobs[blob_hash]
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
try:
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if str(err) != "FOREIGN KEY constraint failed":
raise err
@defer.inlineCallbacks
def _completed_blobs(self, blobhashes_to_check):
"""Returns of the blobhashes_to_check, which are valid"""
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
defer.returnValue(blob_hashes)
def _get_all_verified_blob_hashes(self):
d = self.storage.get_all_blob_hashes()
def get_verified_blobs(blobs):
verified_blobs = []
for blob_hash in blobs:
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
verified_blobs.append(blob_hash)
return verified_blobs
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
return d
-21
View File
@@ -1,21 +0,0 @@
class DownloadOptionChoice:
"""A possible choice that can be picked for some option.
An option can have one or more choices that can be picked from.
"""
def __init__(self, value, short_description, long_description, bool_options_description=None):
self.value = value
self.short_description = short_description
self.long_description = long_description
self.bool_options_description = bool_options_description
class DownloadOption:
"""An option for a user to select a value from several different choices."""
def __init__(self, option_types, long_description, short_description, default_value,
default_value_description):
self.option_types = option_types
self.long_description = long_description
self.short_description = short_description
self.default_value = default_value
self.default_value_description = default_value_description
-189
View File
@@ -1,189 +0,0 @@
from random import choice
import logging
from twisted.internet import defer, task
from twisted.internet.error import ConnectingCancelledError
import treq
from lbrynet.core.utils import DeferredDict
from lbrynet.core.Error import DownloadCanceledError
log = logging.getLogger(__name__)
class HTTPBlobDownloader:
'''
A downloader that is able to get blobs from HTTP mirrors.
Note that when a blob gets downloaded from a mirror or from a peer, BlobManager will mark it as completed
and cause any other type of downloader to progress to the next missing blob. Also, BlobFile is naturally able
to cancel other writers when a writer finishes first. That's why there is no call to cancel/resume/stop between
different types of downloaders.
'''
def __init__(self, blob_manager, blob_hashes=None, servers=None, client=None, sd_hashes=None, retry=True,
clock=None):
if not clock:
from twisted.internet import reactor
self.clock = reactor
else:
self.clock = clock
self.blob_manager = blob_manager
self.servers = servers or []
self.client = client or treq
self.blob_hashes = blob_hashes or []
self.missing_blob_hashes = []
self.downloaded_blob_hashes = []
self.sd_hashes = sd_hashes or []
self.head_blob_hashes = []
self.max_failures = 3
self.semaphore = defer.DeferredSemaphore(2)
self.deferreds = []
self.writers = []
self.retry = retry
self.looping_call = task.LoopingCall(self._download_lc)
self.looping_call.clock = self.clock
self.finished_deferred = defer.Deferred()
self.finished_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
self.short_delay = 30
self.long_delay = 600
self.delay = self.short_delay
self.last_missing = 10000000
self.lc_deferred = None
@defer.inlineCallbacks
def start(self):
if not self.looping_call.running:
self.lc_deferred = self.looping_call.start(self.short_delay, now=True)
self.lc_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
yield self.finished_deferred
def stop(self):
for d in reversed(self.deferreds):
d.cancel()
while self.writers:
writer = self.writers.pop()
writer.close(DownloadCanceledError())
self.blob_hashes = []
if self.looping_call.running:
self.looping_call.stop()
if self.lc_deferred and not self.lc_deferred.called:
self.lc_deferred.cancel()
if not self.finished_deferred.called:
self.finished_deferred.cancel()
@defer.inlineCallbacks
def _download_lc(self):
delay = yield self._download_and_get_retry_delay()
log.debug("delay: %s, missing: %i, downloaded from mirror: %i", delay, len(self.missing_blob_hashes),
len(self.downloaded_blob_hashes))
while self.missing_blob_hashes:
self.blob_hashes.append(self.missing_blob_hashes.pop())
if not delay:
if self.looping_call.running:
self.looping_call.stop()
if not self.finished_deferred.called:
log.debug("mirror finished")
self.finished_deferred.callback(None)
elif delay and delay != self.delay:
if delay == self.long_delay:
log.debug("mirror not making progress, trying less frequently")
elif delay == self.short_delay:
log.debug("queueing retry of %i blobs", len(self.missing_blob_hashes))
if self.looping_call.running:
self.looping_call.stop()
self.delay = delay
self.looping_call = task.LoopingCall(self._download_lc)
self.looping_call.clock = self.clock
self.lc_deferred = self.looping_call.start(self.delay, now=False)
self.lc_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
yield self.finished_deferred
@defer.inlineCallbacks
def _download_and_get_retry_delay(self):
if self.blob_hashes and self.servers:
if self.sd_hashes:
log.debug("trying to download stream from mirror (sd %s)", self.sd_hashes[0][:8])
else:
log.debug("trying to download %i blobs from mirror", len(self.blob_hashes))
blobs = yield DeferredDict(
{blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
)
self.deferreds = [self.download_blob(blobs[blob_hash]) for blob_hash in self.blob_hashes]
yield defer.DeferredList(self.deferreds)
if self.retry and self.missing_blob_hashes:
if not self.downloaded_blob_hashes:
defer.returnValue(self.long_delay)
if len(self.missing_blob_hashes) < self.last_missing:
self.last_missing = len(self.missing_blob_hashes)
defer.returnValue(self.short_delay)
if self.retry and self.last_missing and len(self.missing_blob_hashes) == self.last_missing:
defer.returnValue(self.long_delay)
defer.returnValue(None)
@defer.inlineCallbacks
def _download_blob(self, blob):
for _ in range(self.max_failures):
writer, finished_deferred = blob.open_for_writing('mirror')
self.writers.append(writer)
try:
downloaded = yield self._write_blob(writer, blob)
if downloaded:
yield finished_deferred # yield for verification errors, so we log them
if blob.verified:
log.info('Mirror completed download for %s', blob.blob_hash)
should_announce = blob.blob_hash in self.sd_hashes or blob.blob_hash in self.head_blob_hashes
yield self.blob_manager.blob_completed(blob, should_announce=should_announce)
self.downloaded_blob_hashes.append(blob.blob_hash)
break
except (IOError, Exception, defer.CancelledError, ConnectingCancelledError) as e:
if isinstance(
e, (DownloadCanceledError, defer.CancelledError, ConnectingCancelledError)
) or 'closed file' in str(e):
# some other downloader finished first or it was simply cancelled
log.info("Mirror download cancelled: %s", blob.blob_hash)
break
else:
log.exception('Mirror failed downloading')
finally:
finished_deferred.addBoth(lambda _: None) # suppress echoed errors
if 'mirror' in blob.writers:
writer.close()
self.writers.remove(writer)
def download_blob(self, blob):
if not blob.verified:
d = self.semaphore.run(self._download_blob, blob)
d.addErrback(lambda err: err.trap(defer.TimeoutError, defer.CancelledError))
return d
return defer.succeed(None)
@defer.inlineCallbacks
def _write_blob(self, writer, blob):
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
if response.code != 200:
log.debug('Missing a blob: %s', blob.blob_hash)
if blob.blob_hash in self.blob_hashes:
self.blob_hashes.remove(blob.blob_hash)
if blob.blob_hash not in self.missing_blob_hashes:
self.missing_blob_hashes.append(blob.blob_hash)
defer.returnValue(False)
log.debug('Download started: %s', blob.blob_hash)
blob.set_length(response.length)
yield self.client.collect(response, writer.write)
defer.returnValue(True)
@defer.inlineCallbacks
def download_stream(self, stream_hash, sd_hash):
stream_crypt_blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
self.blob_hashes.extend([
b.blob_hash for b in stream_crypt_blobs
if b.blob_hash and b.blob_hash not in self.blob_hashes
])
if sd_hash not in self.sd_hashes:
self.sd_hashes.append(sd_hash)
head_blob_hash = stream_crypt_blobs[0].blob_hash
if head_blob_hash not in self.head_blob_hashes:
self.head_blob_hashes.append(head_blob_hash)
yield self.start()
def url_for(server, blob_hash=''):
return 'http://{}/{}'.format(server, blob_hash)
-62
View File
@@ -1,62 +0,0 @@
from decimal import Decimal
class Offer:
"""A rate offer to download blobs from a host."""
RATE_ACCEPTED = "RATE_ACCEPTED"
RATE_TOO_LOW = "RATE_TOO_LOW"
RATE_UNSET = "RATE_UNSET"
def __init__(self, offer):
self._state = None
self.rate = None
if isinstance(offer, Decimal):
self.rate = round(offer, 5)
elif isinstance(offer, float):
self.rate = round(Decimal(offer), 5)
if self.rate is None or self.rate < Decimal(0.0):
self.unset()
@property
def is_accepted(self):
return self._state is Offer.RATE_ACCEPTED
@property
def is_too_low(self):
return self._state is Offer.RATE_TOO_LOW
@property
def is_unset(self):
return self._state is Offer.RATE_UNSET
@property
def message(self):
if self.is_accepted:
return Offer.RATE_ACCEPTED
elif self.is_too_low:
return Offer.RATE_TOO_LOW
elif self.is_unset:
return Offer.RATE_UNSET
return None
def accept(self):
if self.is_unset or self._state is None:
self._state = Offer.RATE_ACCEPTED
def reject(self):
if self.is_unset or self._state is None:
self._state = Offer.RATE_TOO_LOW
def unset(self):
self._state = Offer.RATE_UNSET
def handle(self, reply_message):
if reply_message == Offer.RATE_TOO_LOW:
self.reject()
elif reply_message == Offer.RATE_ACCEPTED:
self.accept()
elif reply_message == Offer.RATE_UNSET:
self.unset()
else:
raise Exception("Unknown offer reply %s" % str(reply_message))
-128
View File
@@ -1,128 +0,0 @@
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
from lbrynet import conf
from decimal import Decimal
class BasePaymentRateManager:
def __init__(self, rate=None, info_rate=None):
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
self.min_blob_info_payment_rate = (
info_rate if info_rate is not None else conf.settings['min_info_rate'])
class PaymentRateManager:
def __init__(self, base, rate=None):
"""
@param base: a BasePaymentRateManager
@param rate: the min blob data payment rate
"""
self.base = base
self.min_blob_data_payment_rate = rate
self.points_paid = 0.0
def get_rate_blob_data(self, peer):
return self.get_effective_min_blob_data_payment_rate()
def accept_rate_blob_data(self, peer, payment_rate):
return payment_rate >= self.get_effective_min_blob_data_payment_rate()
def get_effective_min_blob_data_payment_rate(self):
if self.min_blob_data_payment_rate is None:
return self.base.min_blob_data_payment_rate
return self.min_blob_data_payment_rate
def record_points_paid(self, amount):
self.points_paid += amount
class NegotiatedPaymentRateManager:
def __init__(self, base, availability_tracker, generous=None):
"""
@param base: a BasePaymentRateManager
@param availability_tracker: a BlobAvailabilityTracker
@param rate: the min blob data payment rate
"""
self.base = base
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
self.points_paid = 0.0
self.blob_tracker = availability_tracker
self.generous = generous if generous is not None else conf.settings['is_generous_host']
self.strategy = get_default_strategy(self.blob_tracker,
base_price=self.base.min_blob_data_payment_rate,
is_generous=generous)
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
return response.rate
def accept_rate_blob_data(self, peer, blobs, offer):
offer = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, offer)
return offer.is_accepted
def reply_to_offer(self, peer, blobs, offer):
reply = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, reply)
return reply
def get_rate_for_peer(self, peer):
return self.strategy.accepted_offers.get(peer, False)
def record_points_paid(self, amount):
self.points_paid += amount
def record_offer_reply(self, peer, offer):
self.strategy.update_accepted_offers(peer, offer)
def price_limit_reached(self, peer):
if peer in self.strategy.pending_sent_offers:
offer = self.strategy.pending_sent_offers[peer]
return (offer.is_too_low and
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
return False
class OnlyFreePaymentsManager:
def __init__(self, **kwargs):
"""
A payment rate manager that will only ever accept and offer a rate of 0.0,
Used for testing
"""
self.base = BasePaymentRateManager(0.0, 0.0)
self.points_paid = 0.0
self.min_blob_data_payment_rate = 0.0
self.generous = True
self.strategy = OnlyFreeStrategy()
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
return response.rate
def accept_rate_blob_data(self, peer, blobs, offer):
offer = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, offer)
return offer.is_accepted
def reply_to_offer(self, peer, blobs, offer):
reply = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, reply)
return reply
def get_rate_for_peer(self, peer):
return self.strategy.accepted_offers.get(peer, False)
def record_points_paid(self, amount):
self.points_paid += amount
def record_offer_reply(self, peer, offer):
self.strategy.update_accepted_offers(peer, offer)
def price_limit_reached(self, peer):
if peer in self.strategy.pending_sent_offers:
offer = self.strategy.pending_sent_offers[peer]
if offer.rate > 0.0:
return True
return False
-46
View File
@@ -1,46 +0,0 @@
import datetime
from collections import defaultdict
from lbrynet.core import utils
# Do not create this object except through PeerManager
class Peer:
def __init__(self, host, port):
self.host = host
self.port = port
# If a peer is reported down, we wait till this time till reattempting connection
self.attempt_connection_at = None
# Number of times this Peer has been reported to be down, resets to 0 when it is up
self.down_count = 0
# Number of succesful connections (with full protocol completion) to this peer
self.success_count = 0
self.score = 0
self.stats = defaultdict(float) # {string stat_type, float count}
def is_available(self):
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
return True
return False
def report_up(self):
self.down_count = 0
self.attempt_connection_at = None
def report_success(self):
self.success_count += 1
def report_down(self):
self.down_count += 1
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
self.attempt_connection_at = utils.today() + timeout_time
def update_score(self, score_change):
self.score += score_change
def update_stats(self, stat_type, count):
self.stats[stat_type] += count
def __str__(self):
return '{}:{}'.format(self.host, self.port)
def __repr__(self):
return 'Peer({!r}, {!r})'.format(self.host, self.port)
-14
View File
@@ -1,14 +0,0 @@
from lbrynet.core.Peer import Peer
class PeerManager:
def __init__(self):
self.peers = []
def get_peer(self, host, port):
for peer in self.peers:
if peer.host == host and peer.port == port:
return peer
peer = Peer(host, port)
self.peers.append(peer)
return peer
-62
View File
@@ -1,62 +0,0 @@
from zope.interface import implementer
from decimal import Decimal
from lbrynet.interfaces import IBlobPriceModel
from lbrynet import conf
def get_default_price_model(blob_tracker, base_price, **kwargs):
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
class ZeroPrice:
def __init__(self):
self.base_price = 0.0
def calculate_price(self, blob):
return 0.0
class MeanAvailabilityWeightedPrice:
"""Calculate mean-blob-availability and stream-position weighted price for a blob
Attributes:
base_price (float): base price
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
value blobs at the beginning of a stream.
alpha defaults to 1.0, which has a null effect
blob_tracker (BlobAvailabilityTracker): blob availability tracker
"""
implementer(IBlobPriceModel)
def __init__(self, tracker, base_price=None, alpha=1.0):
self.blob_tracker = tracker
base_price = base_price if base_price is not None else conf.settings['data_rate']
self.base_price = Decimal(base_price)
self.alpha = Decimal(alpha)
def calculate_price(self, blob):
mean_availability = self.blob_tracker.last_mean_availability
availability = self.blob_tracker.availability.get(blob, [])
index = 0 # blob.index
availability_mult = self._get_availability_multiplier(mean_availability, availability)
price = self.base_price * availability_mult / self._frontload(index)
return round(price, 5)
def _get_availability_multiplier(self, mean_availability, availability):
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
def _frontload(self, index):
"""Get front-load multiplier, used to weight prices of blobs in a
stream towards the front of the stream.
At index 0, returns 1.0
As index increases, return value approaches 2.0
@param index: blob position in stream
@return: front-load multiplier
"""
return Decimal(2.0) - (self.alpha ** index)
-154
View File
@@ -1,154 +0,0 @@
import logging
from twisted.internet import task
log = logging.getLogger(__name__)
class DummyRateLimiter:
def __init__(self):
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
self.total_dl_bytes = 0
self.total_ul_bytes = 0
self.target_dl = 0
self.target_ul = 0
self.tick_call = None
def start(self):
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(1)
def tick(self):
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
def stop(self):
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
pass
def set_ul_limit(self, limit):
pass
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_second += num_bytes
self.total_dl_bytes += num_bytes
def report_ul_bytes(self, num_bytes):
self.ul_bytes_this_second += num_bytes
self.total_ul_bytes += num_bytes
class RateLimiter:
"""This class ensures that upload and download rates don't exceed specified maximums"""
#implements(IRateLimiter)
#called by main application
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
self.max_dl_bytes = max_dl_bytes
self.max_ul_bytes = max_ul_bytes
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.total_dl_bytes = 0
self.total_ul_bytes = 0
self.tick_call = None
self.tick_interval = 0.1
self.dl_throttled = False
self.ul_throttled = False
self.protocols = []
def start(self):
log.info("Starting rate limiter.")
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(self.tick_interval)
def tick(self):
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.unthrottle_dl()
self.unthrottle_ul()
def stop(self):
log.info("Stopping rate limiter.")
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
self.max_dl_bytes = limit
def set_ul_limit(self, limit):
self.max_ul_bytes = limit
#throttling
def check_dl(self):
need_throttle = (self.max_dl_bytes is not None and
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
if need_throttle:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_dl)
def check_ul(self):
need_throttle = (self.max_ul_bytes is not None and
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
if need_throttle:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_ul)
def throttle_dl(self):
if self.dl_throttled is False:
for protocol in self.protocols:
protocol.throttle_download()
self.dl_throttled = True
def throttle_ul(self):
if self.ul_throttled is False:
for protocol in self.protocols:
protocol.throttle_upload()
self.ul_throttled = True
def unthrottle_dl(self):
if self.dl_throttled is True:
for protocol in self.protocols:
protocol.unthrottle_download()
self.dl_throttled = False
def unthrottle_ul(self):
if self.ul_throttled is True:
for protocol in self.protocols:
protocol.unthrottle_upload()
self.ul_throttled = False
#called by protocols
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_interval += num_bytes
self.total_dl_bytes += num_bytes
self.check_dl()
def report_ul_bytes(self, num_bytes):
self.ul_bytes_this_interval += num_bytes
self.total_ul_bytes += num_bytes
self.check_ul()
def register_protocol(self, protocol):
if protocol not in self.protocols:
self.protocols.append(protocol)
if self.dl_throttled is True:
protocol.throttle_download()
if self.ul_throttled is True:
protocol.throttle_upload()
def unregister_protocol(self, protocol):
if protocol in self.protocols:
self.protocols.remove(protocol)
-105
View File
@@ -1,105 +0,0 @@
import logging
import shutil
import tempfile
from twisted.internet import defer, threads, reactor
from lbrynet.blob import BlobFile
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.core.RateLimiter import DummyRateLimiter
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.dht.peerfinder import DummyPeerFinder
log = logging.getLogger(__name__)
class SinglePeerFinder(DummyPeerFinder):
def __init__(self, peer):
super().__init__()
self.peer = peer
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
return defer.succeed([self.peer])
class BlobCallback(BlobFile):
def __init__(self, blob_dir, blob_hash, timeout):
super().__init__(blob_dir, blob_hash)
self.callback = defer.Deferred()
reactor.callLater(timeout, self._cancel)
def _cancel(self):
if not self.callback.called:
self.callback.callback(False)
def save_verified_blob(self, writer):
result = BlobFile.save_verified_blob(self, writer)
if not self.callback.called:
self.callback.callback(True)
return result
class SingleBlobDownloadManager:
def __init__(self, blob):
self.blob = blob
def needed_blobs(self):
if self.blob.verified:
return []
else:
return [self.blob]
def get_head_blob_hash(self):
return self.blob.blob_hash
class SinglePeerDownloader:
def __init__(self):
self._payment_rate_manager = OnlyFreePaymentsManager()
self._rate_limiter = DummyRateLimiter()
self._wallet = None
self._blob_manager = None
def setup(self, wallet, blob_manager=None):
if not self._wallet:
self._wallet = wallet
if not self._blob_manager:
self._blob_manager = blob_manager
@defer.inlineCallbacks
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager):
log.debug("Try to download %s from %s", blob_hash, peer.host)
blob_manager = blob_manager
blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout)
download_manager = SingleBlobDownloadManager(blob)
peer_finder = SinglePeerFinder(peer)
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
self._wallet, download_manager)
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
self._rate_limiter, self._payment_rate_manager,
self._wallet, timeout=timeout)
info_exchanger = self._wallet.get_info_exchanger()
connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester],
[info_exchanger])
connection_manager.start()
result = yield blob.callback
if not result:
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
yield connection_manager.stop()
defer.returnValue(result)
@defer.inlineCallbacks
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_dir)
try:
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
finally:
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(result)
-152
View File
@@ -1,152 +0,0 @@
from zope.interface import implementer
from decimal import Decimal
from lbrynet import conf
from lbrynet.interfaces import INegotiationStrategy
from lbrynet.core.Offer import Offer
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
def get_default_strategy(blob_tracker, **kwargs):
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
class Strategy:
"""
Base for negotiation strategies
"""
implementer(INegotiationStrategy)
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
self.price_model = price_model
self.is_generous = (
is_generous if is_generous is not None else conf.settings['is_generous_host'])
self.accepted_offers = {}
self.pending_sent_offers = {}
self.offers_sent = {}
self.offers_received = {}
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
self.min_rate = Decimal(min_rate)
def _make_rate_offer(self, rates, offer_count):
return NotImplementedError()
def _get_response_rate(self, rates, offer_count):
return NotImplementedError()
def make_offer(self, peer, blobs):
offer_count = self.offers_sent.get(peer, 0)
self._add_offer_sent(peer)
if peer in self.accepted_offers:
# if there was a previous accepted offer, use that
offer = self.accepted_offers[peer]
if peer in self.pending_sent_offers:
del self.pending_sent_offers[peer]
elif offer_count == 0 and self.is_generous:
# Try asking for it for free
offer = Offer(Decimal(0.0))
self.pending_sent_offers.update({peer: offer})
else:
rates = [self.price_model.calculate_price(blob) for blob in blobs]
price = self._make_rate_offer(rates, offer_count)
offer = Offer(price)
self.pending_sent_offers.update({peer: offer})
return offer
def respond_to_offer(self, offer, peer, blobs):
offer_count = self.offers_received.get(peer, 0)
self._add_offer_received(peer)
rates = [self.price_model.calculate_price(blob) for blob in blobs]
price = self._get_response_rate(rates, offer_count)
if peer in self.accepted_offers:
offer = self.accepted_offers[peer]
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
# give blobs away for free by default on the first request
offer.accept()
self.accepted_offers.update({peer: offer})
elif offer.rate >= price:
offer.accept()
self.accepted_offers.update({peer: offer})
else:
offer.reject()
if peer in self.accepted_offers:
del self.accepted_offers[peer]
return offer
def update_accepted_offers(self, peer, offer):
if not offer.is_accepted and peer in self.accepted_offers:
del self.accepted_offers[peer]
if offer.is_accepted:
self.accepted_offers.update({peer: offer})
self.pending_sent_offers.update({peer: offer})
def _add_offer_sent(self, peer):
turn = self.offers_sent.get(peer, 0) + 1
self.offers_sent.update({peer: turn})
def _add_offer_received(self, peer):
turn = self.offers_received.get(peer, 0) + 1
self.offers_received.update({peer: turn})
def _bounded_price(self, price):
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
return price_for_return
class BasicAvailabilityWeightedStrategy(Strategy):
"""Basic strategy to target blob prices based on supply relative to mean supply
Discount price target with each incoming request, and raise it
with each outgoing from the modeled price until the rate is
accepted or a threshold is reached
"""
implementer(INegotiationStrategy)
def __init__(self, blob_tracker, acceleration=1.25,
deceleration=0.9, max_rate=None,
min_rate=0.0,
is_generous=None,
base_price=0.0001, alpha=1.0):
price_model = MeanAvailabilityWeightedPrice(
blob_tracker, base_price=base_price, alpha=alpha)
super().__init__(price_model, max_rate, min_rate, is_generous)
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
self._deceleration = Decimal(deceleration)
def _get_mean_rate(self, rates):
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
return mean_rate
def _premium(self, rate, turn):
return rate * (self._acceleration ** Decimal(turn))
def _discount(self, rate, turn):
return rate * (self._deceleration ** Decimal(turn))
def _get_response_rate(self, rates, offer_count):
rate = self._get_mean_rate(rates)
discounted = self._discount(rate, offer_count)
rounded_price = round(discounted, 5)
return self._bounded_price(rounded_price)
def _make_rate_offer(self, rates, offer_count):
rate = self._get_mean_rate(rates)
with_premium = self._premium(rate, offer_count)
rounded_price = round(with_premium, 5)
return self._bounded_price(rounded_price)
class OnlyFreeStrategy(Strategy):
implementer(INegotiationStrategy)
def __init__(self, *args, **kwargs):
price_model = ZeroPrice()
super().__init__(price_model, 0.0, 0.0, True)
def _get_mean_rate(self, rates):
return 0.0
def _get_response_rate(self, rates, offer_count):
return 0.0
def _make_rate_offer(self, rates, offer_count):
return 0.0
-472
View File
@@ -1,472 +0,0 @@
from binascii import unhexlify
import string
from collections import defaultdict
import json
import logging
from twisted.internet import threads, defer
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
from lbrynet.core.HTTPBlobDownloader import HTTPBlobDownloader
log = logging.getLogger(__name__)
class JSONBytesEncoder(json.JSONEncoder):
def default(self, obj): # pylint: disable=E0202
if isinstance(obj, bytes):
return obj.decode()
return super().default(obj)
class StreamDescriptorReader:
"""Classes which derive from this class read a stream descriptor file return
a dictionary containing the fields in the file"""
def __init__(self):
pass
def _get_raw_data(self):
"""This method must be overridden by subclasses. It should return a deferred
which fires with the raw data in the stream descriptor"""
pass
def get_info(self):
"""Return the fields contained in the file"""
d = self._get_raw_data()
d.addCallback(json.loads)
return d
class PlainStreamDescriptorReader(StreamDescriptorReader):
"""Read a stream descriptor file which is not a blob but a regular file"""
def __init__(self, stream_descriptor_filename):
super().__init__()
self.stream_descriptor_filename = stream_descriptor_filename
def _get_raw_data(self):
def get_data():
with open(self.stream_descriptor_filename) as file_handle:
raw_data = file_handle.read()
return raw_data
return threads.deferToThread(get_data)
class BlobStreamDescriptorReader(StreamDescriptorReader):
"""Read a stream descriptor file which is a blob"""
def __init__(self, blob):
super().__init__()
self.blob = blob
def _get_raw_data(self):
def get_data():
f = self.blob.open_for_reading()
if f is not None:
raw_data = f.read()
f.close()
return raw_data
else:
raise ValueError("Could not open the blob for reading")
return threads.deferToThread(get_data)
class StreamDescriptorWriter:
"""Classes which derive from this class write fields from a dictionary
of fields to a stream descriptor"""
def __init__(self):
pass
def create_descriptor(self, sd_info):
return self._write_stream_descriptor(
json.dumps(sd_info, sort_keys=True).encode()
)
def _write_stream_descriptor(self, raw_data):
"""This method must be overridden by subclasses to write raw data to
the stream descriptor
"""
pass
class PlainStreamDescriptorWriter(StreamDescriptorWriter):
def __init__(self, sd_file_name):
super().__init__()
self.sd_file_name = sd_file_name
def _write_stream_descriptor(self, raw_data):
def write_file():
log.info("Writing the sd file to disk")
with open(self.sd_file_name, 'w') as sd_file:
sd_file.write(raw_data)
return self.sd_file_name
return threads.deferToThread(write_file)
class BlobStreamDescriptorWriter(StreamDescriptorWriter):
def __init__(self, blob_manager):
super().__init__()
self.blob_manager = blob_manager
@defer.inlineCallbacks
def _write_stream_descriptor(self, raw_data):
log.debug("Creating the new blob for the stream descriptor")
blob_creator = self.blob_manager.get_blob_creator()
blob_creator.write(raw_data)
log.debug("Wrote the data to the new blob")
sd_hash = yield blob_creator.close()
yield self.blob_manager.creator_finished(blob_creator, should_announce=True)
defer.returnValue(sd_hash)
class StreamMetadata:
FROM_BLOB = 1
FROM_PLAIN = 2
def __init__(self, validator, options, factories):
self.validator = validator
self.options = options
self.factories = factories
self.metadata_source = None
self.source_blob_hash = None
self.source_file = None
class StreamDescriptorIdentifier:
"""Tries to determine the type of stream described by the stream descriptor using the
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
and returns the appropriate ones based on the type of the stream descriptor given
"""
def __init__(self):
# {stream_type: IStreamDescriptorValidator}
self._sd_info_validators = {}
# {stream_type: IStreamOptions
self._stream_options = {}
# {stream_type: [IStreamDownloaderFactory]}
self._stream_downloader_factories = defaultdict(list)
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
"""This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
There can only be one StreamDescriptorValidator for each type of stream.
@param stream_type: A string representing the type of stream
descriptor. This must be unique to this stream descriptor.
@param sd_info_validator: A class implementing the
IStreamDescriptorValidator interface. This class's
constructor will be passed the raw metadata in the stream
descriptor file and its 'validate' method will then be
called. If the validation step fails, an exception will be
thrown, preventing the stream descriptor from being
further processed.
@param stream_options: A class implementing the IStreamOptions
interface. This class's constructor will be passed the
sd_info_validator object containing the raw metadata from
the stream descriptor file.
@return: None
"""
self._sd_info_validators[stream_type] = sd_info_validator
self._stream_options[stream_type] = stream_options
def add_stream_downloader_factory(self, stream_type, factory):
"""Register a stream downloader factory with the StreamDescriptorIdentifier.
This is how the StreamDescriptorIdentifier determines what
factories may be used to process different stream descriptor
files. There must be at least one factory for each type of
stream added via "add_stream_info_validator".
@param stream_type: A string representing the type of stream
descriptor which the factory knows how to process.
@param factory: An object implementing the IStreamDownloaderFactory interface.
@return: None
"""
self._stream_downloader_factories[stream_type].append(factory)
def _return_metadata(self, options_validator_factories, source_type, source):
validator, options, factories = options_validator_factories
m = StreamMetadata(validator, options, factories)
m.metadata_source = source_type
if source_type == StreamMetadata.FROM_BLOB:
m.source_blob_hash = source
if source_type == StreamMetadata.FROM_PLAIN:
m.source_file = source
return m
def get_metadata_for_sd_file(self, sd_path):
sd_reader = PlainStreamDescriptorReader(sd_path)
d = sd_reader.get_info()
d.addCallback(self._return_options_and_validator_and_factories)
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
return d
def get_metadata_for_sd_blob(self, sd_blob):
sd_reader = BlobStreamDescriptorReader(sd_blob)
d = sd_reader.get_info()
d.addCallback(self._return_options_and_validator_and_factories)
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
return d
def _get_factories(self, stream_type):
if not stream_type in self._stream_downloader_factories:
raise UnknownStreamTypeError(stream_type)
return self._stream_downloader_factories[stream_type]
def _get_validator(self, stream_type):
if not stream_type in self._sd_info_validators:
raise UnknownStreamTypeError(stream_type)
return self._sd_info_validators[stream_type]
def _get_options(self, stream_type):
if not stream_type in self._stream_downloader_factories:
raise UnknownStreamTypeError(stream_type)
return self._stream_options[stream_type]
def _return_options_and_validator_and_factories(self, sd_info):
if not 'stream_type' in sd_info:
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
stream_type = sd_info['stream_type']
validator = self._get_validator(stream_type)(sd_info)
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
d = validator.validate()
def get_options():
options = self._get_options(stream_type)
return validator, options, factories
d.addCallback(lambda _: get_options())
return d
EncryptedFileStreamType = "lbryfile"
@defer.inlineCallbacks
def save_sd_info(blob_manager, sd_hash, sd_info):
if not blob_manager.blobs.get(sd_hash) or not blob_manager.blobs[sd_hash].get_is_verified():
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
calculated_sd_hash = yield descriptor_writer.create_descriptor(sd_info)
if calculated_sd_hash != sd_hash:
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
(sd_hash, calculated_sd_hash))
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
if not stream_hash:
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
stream_name = sd_info['stream_name']
key = sd_info['key']
stream_hash = sd_info['stream_hash']
stream_blobs = sd_info['blobs']
suggested_file_name = sd_info['suggested_file_name']
yield blob_manager.storage.add_known_blobs(stream_blobs)
yield blob_manager.storage.store_stream(
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
)
defer.returnValue(stream_hash)
def format_blobs(crypt_blob_infos):
formatted_blobs = []
for blob_info in crypt_blob_infos:
blob = {}
if blob_info.length != 0:
blob['blob_hash'] = blob_info.blob_hash
blob['blob_num'] = blob_info.blob_num
blob['iv'] = blob_info.iv
blob['length'] = blob_info.length
formatted_blobs.append(blob)
return formatted_blobs
def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_hash, blobs):
return {
"stream_type": stream_type,
"stream_name": stream_name,
"key": key,
"suggested_file_name": suggested_file_name,
"stream_hash": stream_hash,
"blobs": blobs
}
@defer.inlineCallbacks
def get_sd_info(storage, stream_hash, include_blobs):
"""
Get an sd info dictionary from storage
:param storage: (SQLiteStorage) storage instance
:param stream_hash: (str) stream hash
:param include_blobs: (bool) include stream blob infos
:return: {
"stream_type": "lbryfile",
"stream_name": <hex encoded stream name>,
"key": <stream key>,
"suggested_file_name": <hex encoded suggested file name>,
"stream_hash": <stream hash>,
"blobs": [
{
"blob_hash": <head blob_hash>,
"blob_num": 0,
"iv": <iv>,
"length": <head blob length>
}, ...
{
"blob_num": <stream length>,
"iv": <iv>,
"length": 0
}
]
}
"""
stream_info = yield storage.get_stream_info(stream_hash)
blobs = []
if include_blobs:
blobs = yield storage.get_blobs_for_stream(stream_hash)
defer.returnValue(
format_sd_info(
EncryptedFileStreamType, stream_info[0], stream_info[1],
stream_info[2], stream_hash, format_blobs(blobs)
)
)
def get_blob_hashsum(b):
length = b['length']
if length != 0:
blob_hash = b['blob_hash']
else:
blob_hash = None
blob_num = b['blob_num']
iv = b['iv']
blob_hashsum = get_lbry_hash_obj()
if length != 0:
blob_hashsum.update(blob_hash.encode())
blob_hashsum.update(str(blob_num).encode())
blob_hashsum.update(iv.encode())
blob_hashsum.update(str(length).encode())
return blob_hashsum.digest()
def get_stream_hash(hex_stream_name, key, hex_suggested_file_name, blob_infos):
h = get_lbry_hash_obj()
h.update(hex_stream_name.encode())
h.update(key.encode())
h.update(hex_suggested_file_name.encode())
blobs_hashsum = get_lbry_hash_obj()
for blob in blob_infos:
blobs_hashsum.update(get_blob_hashsum(blob))
h.update(blobs_hashsum.digest())
return h.hexdigest()
def verify_hex(text, field_name):
if not set(text).issubset(set(string.hexdigits)):
raise InvalidStreamDescriptorError("%s is not a hex-encoded string" % field_name)
def validate_descriptor(stream_info):
try:
hex_stream_name = stream_info['stream_name']
key = stream_info['key']
hex_suggested_file_name = stream_info['suggested_file_name']
stream_hash = stream_info['stream_hash']
blobs = stream_info['blobs']
except KeyError as e:
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
if stream_info['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([False if blob_info['length'] > 0 else True for blob_info in stream_info['blobs'][:-1]]):
raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in stream_info['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
verify_hex(key, "key")
verify_hex(hex_suggested_file_name, "suggested file name")
verify_hex(stream_hash, "stream_hash")
calculated_stream_hash = get_stream_hash(
hex_stream_name, key, hex_suggested_file_name, blobs
)
if calculated_stream_hash != stream_hash:
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
return True
class EncryptedFileStreamDescriptorValidator:
def __init__(self, raw_info):
self.raw_info = raw_info
def validate(self):
return defer.succeed(validate_descriptor(self.raw_info))
def info_to_show(self):
info = []
info.append(("stream_name", unhexlify(self.raw_info.get("stream_name"))))
size_so_far = 0
for blob_info in self.raw_info.get("blobs", []):
size_so_far += int(blob_info['length'])
info.append(("stream_size", str(self.get_length_of_stream())))
suggested_file_name = self.raw_info.get("suggested_file_name", None)
if suggested_file_name is not None:
suggested_file_name = unhexlify(suggested_file_name)
info.append(("suggested_file_name", suggested_file_name))
return info
def get_length_of_stream(self):
size_so_far = 0
for blob_info in self.raw_info.get("blobs", []):
size_so_far += int(blob_info['length'])
return size_so_far
@defer.inlineCallbacks
def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet, timeout=None,
download_mirrors=None):
"""
Downloads a single blob from the network
@param session:
@param blob_hash:
@param payment_rate_manager:
@return: An object of type HashBlob
"""
downloader = StandaloneBlobDownloader(blob_hash,
blob_manager,
peer_finder,
rate_limiter,
payment_rate_manager,
wallet,
timeout)
mirror = HTTPBlobDownloader(blob_manager, [blob_hash], download_mirrors or [], sd_hashes=[blob_hash], retry=False)
mirror.start()
sd_blob = yield downloader.download()
mirror.stop()
sd_reader = BlobStreamDescriptorReader(sd_blob)
sd_info = yield sd_reader.get_info()
try:
validate_descriptor(sd_info)
except InvalidStreamDescriptorError as err:
yield blob_manager.delete_blobs([blob_hash])
raise err
raw_sd = yield sd_reader._get_raw_data()
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
defer.returnValue(sd_blob)
-9
View File
@@ -1,9 +0,0 @@
"""
Classes and functions which can be used by any application wishing to make use of the LBRY network.
This includes classes for connecting to other peers and downloading blobs from them, listening for
connections from peers and responding to their requests, managing locally stored blobs, sending
and receiving payments, and locating peers in the DHT.
"""
from lbrynet import custom_logger
-81
View File
@@ -1,81 +0,0 @@
import logging
log = logging.getLogger()
MIN_DELAY = 0.0
MAX_DELAY = 0.01
DELAY_INCREMENT = 0.0001
QUEUE_SIZE_THRESHOLD = 100
class CallLaterManager:
def __init__(self, callLater):
"""
:param callLater: (IReactorTime.callLater)
"""
self._callLater = callLater
self._pendingCallLaters = []
self._delay = MIN_DELAY
def get_min_delay(self):
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
queue_size = len(self._pendingCallLaters)
if queue_size > QUEUE_SIZE_THRESHOLD:
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
else:
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
return self._delay
def _cancel(self, call_later):
"""
:param call_later: DelayedCall
:return: (callable) canceller function
"""
def cancel(reason=None):
"""
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
:return: reason
"""
if call_later.active():
call_later.cancel()
if call_later in self._pendingCallLaters:
self._pendingCallLaters.remove(call_later)
return reason
return cancel
def stop(self):
"""
Cancel any callLaters that are still running
"""
from twisted.internet import defer
while self._pendingCallLaters:
canceller = self._cancel(self._pendingCallLaters[0])
try:
canceller()
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
pass
def call_later(self, when, what, *args, **kwargs):
"""
Schedule a call later and get a canceller callback function
:param when: (float) delay in seconds
:param what: (callable)
:param args: (*tuple) args to be passed to the callable
:param kwargs: (**dict) kwargs to be passed to the callable
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
"""
call_later = self._callLater(when, what, *args, **kwargs)
canceller = self._cancel(call_later)
self._pendingCallLaters.append(call_later)
return call_later, canceller
def call_soon(self, what, *args, **kwargs):
delay = self.get_min_delay()
return self.call_later(delay, what, *args, **kwargs)
-590
View File
@@ -1,590 +0,0 @@
import logging
from collections import defaultdict
from decimal import Decimal
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.internet.error import ConnectionAborted
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
from lbrynet.core.Offer import Offer
log = logging.getLogger(__name__)
def get_points(num_bytes, rate):
if isinstance(rate, float):
return 1.0 * num_bytes * rate / 2**20
elif isinstance(rate, Decimal):
return 1.0 * num_bytes * float(rate) / 2**20
else:
raise Exception("Unknown rate type")
def cache(fn):
"""Caches the function call for each instance"""
attr = '__{}_value'.format(fn.__name__)
def helper(self):
if not hasattr(self, attr):
value = fn(self)
setattr(self, attr, value)
return getattr(self, attr)
return helper
class BlobRequester:
#implements(IRequestCreator)
def __init__(self, blob_manager, peer_finder, payment_rate_manager, wallet, download_manager):
self.blob_manager = blob_manager
self.peer_finder = peer_finder
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self._download_manager = download_manager
self._peers = defaultdict(int) # {Peer: score}
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
self._protocol_prices = {} # {ClientProtocol: price}
self._protocol_offers = {}
self._price_disagreements = [] # [Peer]
self._protocol_tries = {}
self._maxed_out_peers = []
self._incompatible_peers = []
######## IRequestCreator #########
def send_next_request(self, peer, protocol):
"""Makes an availability request, download request and price request"""
if not self.should_send_next_request(peer):
return defer.succeed(False)
return self._send_next_request(peer, protocol)
@defer.inlineCallbacks
def get_new_peers_for_head_blob(self):
""" look for peers for the head blob """
head_blob_hash = self._download_manager.get_head_blob_hash()
peers = yield self._find_peers_for_hash(head_blob_hash)
defer.returnValue(peers)
@defer.inlineCallbacks
def get_new_peers_for_next_unavailable(self):
"""
Look for peers for the next unavailable blob, if we have
all blobs, return an empty list
"""
blob_hash = yield self._get_hash_for_peer_search()
if blob_hash is None:
defer.returnValue([])
peers = yield self._find_peers_for_hash(blob_hash)
defer.returnValue(peers)
######### internal calls #########
def should_send_next_request(self, peer):
return (
self._blobs_to_download() and
self._should_send_request_to(peer)
)
def _send_next_request(self, peer, protocol):
log.debug('Sending a blob request for %s and %s', peer, protocol)
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
head_blob_hash = self._download_manager.get_head_blob_hash()
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager,
self.wallet, head_blob_hash)
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
sent_request = False
if availability.can_make_request():
availability.make_request_and_handle_response()
sent_request = True
if price.can_make_request():
# TODO: document why a PriceRequest is only made if an
# Availability or Download request was made
price.make_request_and_handle_response()
sent_request = True
if download.can_make_request():
try:
download.make_request_and_handle_response()
sent_request = True
except InsufficientFundsError as err:
return defer.fail(err)
return defer.succeed(sent_request)
def _get_hash_for_peer_search(self):
"""
Get next unavailable hash for blob,
returns None if there is nothing left to download
"""
r = None
blobs_to_download = self._blobs_to_download()
if blobs_to_download:
blobs_without_sources = self._blobs_without_sources()
if not blobs_without_sources:
blob_hash = blobs_to_download[0].blob_hash
else:
blob_hash = blobs_without_sources[0].blob_hash
r = blob_hash
log.debug("Blob requester peer search response: %s", str(r))
return defer.succeed(r)
def _find_peers_for_hash(self, h):
d = self.peer_finder.find_peers_for_blob(h, filter_self=True)
def choose_best_peers(peers):
bad_peers = self._get_bad_peers()
without_bad_peers = [p for p in peers if not p in bad_peers]
without_maxed_out_peers = [
p for p in without_bad_peers if p not in self._maxed_out_peers]
return without_maxed_out_peers
d.addCallback(choose_best_peers)
def lookup_failed(err):
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
return []
d.addErrback(lookup_failed)
return d
def _should_send_request_to(self, peer):
if self._peers[peer] < -5.0:
return False
if peer in self._price_disagreements:
return False
if peer in self._incompatible_peers:
return False
return True
def _get_bad_peers(self):
return [p for p in self._peers.keys() if not self._should_send_request_to(p)]
def _hash_available(self, blob_hash):
for peer in self._available_blobs:
if blob_hash in self._available_blobs[peer]:
return True
return False
def _hash_available_on(self, blob_hash, peer):
if blob_hash in self._available_blobs[peer]:
return True
return False
def _blobs_to_download(self):
needed_blobs = self._download_manager.needed_blobs()
return sorted(needed_blobs, key=lambda b: b.is_downloading())
def _blobs_without_sources(self):
return [
b for b in self._download_manager.needed_blobs()
if not self._hash_available(b.blob_hash)
]
def _price_settled(self, protocol):
if protocol in self._protocol_prices:
return True
return False
def _update_local_score(self, peer, amount):
self._peers[peer] += amount
class RequestHelper:
def __init__(self, requestor, peer, protocol, payment_rate_manager):
self.requestor = requestor
self.peer = peer
self.protocol = protocol
self.payment_rate_manager = payment_rate_manager
@property
def protocol_prices(self):
return self.requestor._protocol_prices
@property
def protocol_offers(self):
return self.requestor._protocol_offers
@property
def available_blobs(self):
return self.requestor._available_blobs[self.peer]
@property
def unavailable_blobs(self):
return self.requestor._unavailable_blobs[self.peer]
@property
def maxed_out_peers(self):
return self.requestor._maxed_out_peers
def update_local_score(self, score):
self.requestor._update_local_score(self.peer, score)
def _request_failed(self, reason, request_type):
if reason.check(DownloadCanceledError, RequestCanceledError, ConnectionAborted,
ConnectionClosedBeforeResponseError, ValueError):
return
if reason.check(NoResponseError):
self.requestor._incompatible_peers.append(self.peer)
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
request_type, reason.getErrorMessage(), reason.type)
self.update_local_score(-10.0)
if isinstance(reason, (InvalidResponseError, NoResponseError)):
self.peer.update_score(-10.0)
else:
self.peer.update_score(-2.0)
return reason
def get_rate(self):
if self.payment_rate_manager.price_limit_reached(self.peer):
if self.peer not in self.maxed_out_peers:
self.maxed_out_peers.append(self.peer)
return None
rate = self.protocol_prices.get(self.protocol)
if rate is None:
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
if not pending.is_too_low and not pending.is_accepted:
return pending.rate
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
return rate
def _handle_incoming_blob(response_dict, peer, request):
if request.response_identifier not in response_dict:
return InvalidResponseError("response identifier not in response")
if not isinstance(response_dict[request.response_identifier], dict):
return InvalidResponseError("response not a dict. got %s" %
type(response_dict[request.response_identifier]))
response = response_dict[request.response_identifier]
if 'error' in response:
# This means we're not getting our blob for some reason
if response['error'] == "RATE_UNSET":
# Stop the download with an error that won't penalize the peer
request.cancel(PriceDisagreementError())
else:
# The peer has done something bad so we should get out of here
return InvalidResponseError("Got an unknown error from the peer: %s" %
(response['error'],))
else:
if 'blob_hash' not in response:
return InvalidResponseError("Missing the required field 'blob_hash'")
if not response['blob_hash'] == request.request_dict['requested_blob']:
return InvalidResponseError(
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
(response['blob_hash'], request.request_dict['requested_blob'])
)
if 'length' not in response:
return InvalidResponseError("Missing the required field 'length'")
if not request.blob.set_length(response['length']):
return InvalidResponseError("Could not set the length of the blob")
return True
def _handle_download_error(err, peer, blob_to_download):
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError,
ConnectionClosedBeforeResponseError):
log.warning("An error occurred while downloading %s from %s. Error: %s",
blob_to_download.blob_hash, str(peer), err.getTraceback())
if err.check(PriceDisagreementError):
# Don't kill the whole connection just because a price couldn't be agreed upon.
# Other information might be desired by other request creators at a better rate.
return True
return err
class AvailabilityRequest(RequestHelper):
"""Ask a peer what blobs it has available.
Results are saved in `_available_blobs` and `_unavailable_blobs`
on the parent BlobRequester.
"""
def can_make_request(self):
return self.get_top_needed_blobs()
def make_request_and_handle_response(self):
request = self._get_request()
self._handle_request(request)
def _get_request(self):
to_request = self.get_top_needed_blobs()
if not to_request:
raise Exception('Unable to make a request without available blobs')
return self._make_request(to_request)
@cache
def get_top_needed_blobs(self, limit=20):
all_needed = [
b.blob_hash for b in self.requestor._blobs_to_download()
if not self.is_available(b)
]
# sort them so that the peer will be asked first for blobs it
# hasn't said it doesn't have
sorted_needed = sorted(
all_needed,
key=lambda b: b in self.unavailable_blobs
)
return sorted_needed[:limit]
def is_available(self, blob):
return blob.blob_hash in self.available_blobs
def _make_request(self, to_request):
log.debug('Requesting blobs: %s', to_request)
r_dict = {'requested_blobs': to_request}
response_identifier = 'available_blobs'
request = ClientRequest(r_dict, response_identifier)
return request
def _handle_request(self, a_r):
log.debug('making an availability request')
d1 = self.protocol.add_request(a_r)
d1.addCallback(self._handle_availability, a_r)
d1.addErrback(self._request_failed, "availability request")
def _handle_availability(self, response_dict, request):
assert request.response_identifier == 'available_blobs'
if 'available_blobs' not in response_dict:
raise InvalidResponseError("response identifier not in response")
log.debug("Received a response to the availability request")
# save available blobs
blob_hashes = response_dict['available_blobs']
if not blob_hashes:
# should not send any more requests as it doesnt have any blob we need
self.update_local_score(-10.0)
return True
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
self.process_available_blob_hash(blob_hash, request)
# everything left in the request is missing
for blob_hash in request.request_dict['requested_blobs']:
self.unavailable_blobs.append(blob_hash)
return True
def process_available_blob_hash(self, blob_hash, request):
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
self.available_blobs.append(blob_hash)
self.remove_from_unavailable_blobs(blob_hash)
request.request_dict['requested_blobs'].remove(blob_hash)
def remove_from_unavailable_blobs(self, blob_hash):
if blob_hash in self.unavailable_blobs:
self.unavailable_blobs.remove(blob_hash)
class PriceRequest(RequestHelper):
"""Ask a peer if a certain price is acceptable"""
def can_make_request(self):
if len(self.available_blobs) and self.protocol not in self.protocol_prices:
return self.get_rate() is not None
return False
def make_request_and_handle_response(self):
request = self._get_price_request()
self._handle_price_request(request)
def _get_price_request(self):
rate = self.get_rate()
if rate is None:
log.debug("No blobs to request from %s", self.peer)
raise Exception('Cannot make a price request without a payment rate')
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
request_dict = {'blob_data_payment_rate': rate}
assert self.protocol not in self.protocol_offers
self.protocol_offers[self.protocol] = rate
return ClientRequest(request_dict, 'blob_data_payment_rate')
def _handle_price_request(self, price_request):
d = self.protocol.add_request(price_request)
d.addCallback(self._handle_price_response, price_request)
d.addErrback(self._request_failed, "price request")
def _handle_price_response(self, response_dict, request):
assert request.response_identifier == 'blob_data_payment_rate'
if 'blob_data_payment_rate' not in response_dict:
return InvalidResponseError("response identifier not in response")
offer_value = self.protocol_offers.pop(self.protocol)
offer = Offer(offer_value)
offer.handle(response_dict['blob_data_payment_rate'])
self.payment_rate_manager.record_offer_reply(self.peer, offer)
if offer.is_accepted:
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
self.protocol_prices[self.protocol] = offer.rate
return True
elif offer.is_too_low:
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
return not self.payment_rate_manager.price_limit_reached(self.peer)
else:
log.warning("Price disagreement")
self.requestor._price_disagreements.append(self.peer)
return False
class DownloadRequest(RequestHelper):
"""Choose a blob and download it from a peer and also pay the peer for the data."""
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet, head_blob_hash):
super().__init__(requester, peer, protocol, payment_rate_manager)
self.wallet = wallet
self.head_blob_hash = head_blob_hash
def can_make_request(self):
if self.protocol in self.protocol_prices:
return self.get_blob_details()
return False
def make_request_and_handle_response(self):
request = self._get_request()
self._handle_download_request(request)
def _get_request(self):
blob_details = self.get_blob_details()
if not blob_details:
raise Exception('No blobs available to download')
return self._make_request(blob_details)
@cache
def get_blob_details(self):
"""Open a blob for writing and return the details.
If no blob can be opened, returns None.
"""
to_download = self.get_available_blobs()
return self.find_blob(to_download)
def get_available_blobs(self):
available_blobs = [
b for b in self.requestor._blobs_to_download()
if self.requestor._hash_available_on(b.blob_hash, self.peer)
]
log.debug('available blobs: %s', available_blobs)
return available_blobs
def find_blob(self, to_download):
"""Return the first blob in `to_download` that is successfully opened for write."""
for blob in to_download:
if blob.get_is_verified():
log.debug('Skipping blob %s as its already validated', blob)
continue
writer, d = blob.open_for_writing(self.peer)
if d is not None:
return BlobDownloadDetails(blob, d, writer.write, writer.close, self.peer)
log.warning('Skipping blob %s as there was an issue opening it for writing', blob)
return None
def _make_request(self, blob_details):
blob = blob_details.blob
request = ClientBlobRequest(
{'requested_blob': blob.blob_hash},
'incoming_blob',
blob_details.counting_write_func,
blob_details.deferred,
blob_details.cancel_func,
blob
)
log.info("Requesting blob %s from %s", blob.blob_hash, self.peer)
return request
def _handle_download_request(self, client_blob_request):
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
self.create_add_blob_request(client_blob_request)
def reserve_funds_or_cancel(self, client_blob_request):
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
if reserved_points is not None:
return reserved_points
client_blob_request.cancel(InsufficientFundsError())
client_blob_request.finished_deferred.addErrback(lambda _: True)
raise InsufficientFundsError()
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
# Note: The following three callbacks will be called when the blob has been
# fully downloaded or canceled
client_blob_request.finished_deferred.addCallbacks(
self._download_succeeded,
self._download_failed,
callbackArgs=(client_blob_request.blob,),
)
client_blob_request.finished_deferred.addBoth(self._pay_or_cancel_payment,
reserved_points, client_blob_request.blob)
client_blob_request.finished_deferred.addErrback(_handle_download_error, self.peer,
client_blob_request.blob)
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
if self._can_pay_peer(blob, arg):
self._pay_peer(blob.length, reserved_points)
else:
self._cancel_points(reserved_points)
return arg
def _can_pay_peer(self, blob, arg):
return (
blob.length != 0 and
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
)
def _pay_peer(self, num_bytes, reserved_points):
assert num_bytes != 0
rate = self.get_rate()
point_amount = get_points(num_bytes, rate)
self.wallet.send_points(reserved_points, point_amount)
self.payment_rate_manager.record_points_paid(point_amount)
def _cancel_points(self, reserved_points):
self.wallet.cancel_point_reservation(reserved_points)
def create_add_blob_request(self, client_blob_request):
d = self.protocol.add_blob_request(client_blob_request)
# Note: The following two callbacks will be called as soon as the peer sends its
# response, which will be before the blob has finished downloading, but may be
# after the blob has been canceled. For example,
# 1) client sends request to Peer A
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
# 3) client receives response from Peer A
# Therefore, these callbacks shouldn't rely on there being a blob about to be
# downloaded.
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
d.addErrback(self._request_failed, "download request")
def _reserve_points(self, num_bytes):
# jobevers: there was an assertion here, but I don't think it
# was a valid assertion to make. It is possible for a rate to
# not yet been set for this protocol or for it to have been
# removed so instead I switched it to check if a rate has been set
# and calculate it if it has not
rate = self.get_rate()
points_to_reserve = get_points(num_bytes, rate)
return self.wallet.reserve_points(self.peer, points_to_reserve)
def _download_succeeded(self, arg, blob):
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
self.update_local_score(5.0)
self.peer.update_stats('blobs_downloaded', 1)
self.peer.update_score(5.0)
should_announce = blob.blob_hash == self.head_blob_hash
d = self.requestor.blob_manager.blob_completed(blob, should_announce=should_announce)
d.addCallback(lambda _: arg)
return d
def _download_failed(self, reason):
if not reason.check(DownloadCanceledError, PriceDisagreementError):
self.update_local_score(-10.0)
return reason
class BlobDownloadDetails:
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
def __init__(self, blob, deferred, write_func, cancel_func, peer):
self.blob = blob
self.deferred = deferred
self.write_func = write_func
self.cancel_func = cancel_func
self.peer = peer
def counting_write_func(self, data):
self.peer.update_stats('blob_bytes_downloaded', len(data))
return self.write_func(data)
-284
View File
@@ -1,284 +0,0 @@
import json
import logging
from decimal import Decimal
from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.policies import TimeoutMixin
from twisted.python import failure
from lbrynet import conf
from lbrynet.core import utils
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
from lbrynet.core.Error import RequestCanceledError
log = logging.getLogger(__name__)
def encode_decimal(obj):
if isinstance(obj, Decimal):
return float(obj)
raise TypeError(repr(obj) + " is not JSON serializable")
class ClientProtocol(Protocol, TimeoutMixin):
#implements(IRequestSender, IRateLimited)
######### Protocol #########
PROTOCOL_TIMEOUT = 30
def connectionMade(self):
log.debug("Connection made to %s", self.factory.peer)
self._connection_manager = self.factory.connection_manager
self._rate_limiter = self.factory.rate_limiter
self.peer = self.factory.peer
self._response_deferreds = {}
self._response_buff = b''
self._downloading_blob = False
self._blob_download_request = None
self._next_request = {}
self.connection_closed = False
self.connection_closing = False
# This needs to be set for TimeoutMixin
self.callLater = utils.call_later
self.peer.report_up()
self._ask_for_request()
def dataReceived(self, data):
log.debug("Received %d bytes from %s", len(data), self.peer)
self.setTimeout(None)
self._rate_limiter.report_dl_bytes(len(data))
if self._downloading_blob is True:
self._blob_download_request.write(data)
else:
self._response_buff += data
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
log.warning("Response is too large from %s. Size %s",
self.peer, len(self._response_buff))
self.transport.loseConnection()
response, extra_data = self._get_valid_response(self._response_buff)
if response is not None:
self._response_buff = b''
self._handle_response(response)
if self._downloading_blob is True and len(extra_data) != 0:
self._blob_download_request.write(extra_data)
def timeoutConnection(self):
log.info("Connection timed out to %s", self.peer)
self.peer.report_down()
self.transport.abortConnection()
def connectionLost(self, reason=None):
log.debug("Connection lost to %s: %s", self.peer, reason)
self.setTimeout(None)
self.connection_closed = True
if reason is None or reason.check(error.ConnectionDone):
err = failure.Failure(ConnectionClosedBeforeResponseError())
else:
err = reason
for key, d in self._response_deferreds.items():
d.errback(err)
self._response_deferreds.clear()
if self._blob_download_request is not None:
self._blob_download_request.cancel(err)
self.factory.connection_was_made_deferred.callback(True)
######### IRequestSender #########
def add_request(self, request):
if request.response_identifier in self._response_deferreds:
raise ValueError("There is already a request for that response active")
self._next_request.update(request.request_dict)
d = defer.Deferred()
log.debug("Adding a request for %s. Request: %s", self.peer, request.request_dict)
self._response_deferreds[request.response_identifier] = d
return d
def add_blob_request(self, blob_request):
if self._blob_download_request is None:
d = self.add_request(blob_request)
self._blob_download_request = blob_request
blob_request.finished_deferred.addCallbacks(self._downloading_finished,
self._handle_response_error)
return d
else:
return defer.fail(ValueError("There is already a blob download request active"))
def cancel_requests(self):
self.connection_closing = True
ds = []
err = RequestCanceledError()
for key, d in list(self._response_deferreds.items()):
del self._response_deferreds[key]
d.errback(err)
ds.append(d)
if self._blob_download_request is not None:
ds.append(self._blob_download_request.finished_deferred)
self._blob_download_request.cancel(err)
self._blob_download_request = None
self._downloading_blob = False
return defer.DeferredList(ds)
######### Internal request handling #########
def _handle_request_error(self, err):
log.error("An unexpected error occurred creating or sending a request to %s. %s: %s",
self.peer, err.type, err)
self.transport.loseConnection()
def _ask_for_request(self):
if self.connection_closed is True or self.connection_closing is True:
return
def send_request_or_close(do_request):
if do_request is True:
request_msg, self._next_request = self._next_request, {}
self._send_request_message(request_msg)
else:
# The connection manager has indicated that this connection should be terminated
log.debug("Closing the connection to %s due to having no further requests to send",
self.peer)
self.peer.report_success()
self.transport.loseConnection()
d = self._connection_manager.get_next_request(self.peer, self)
d.addCallback(send_request_or_close)
d.addErrback(self._handle_request_error)
def _send_request_message(self, request_msg):
self.setTimeout(self.PROTOCOL_TIMEOUT)
# TODO: compare this message to the last one. If they're the same,
# TODO: incrementally delay this message.
m = json.dumps(request_msg, default=encode_decimal).encode()
self.transport.write(m)
def _get_valid_response(self, response_msg):
extra_data = None
response = None
curr_pos = 0
while 1:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren != -1:
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
except ValueError:
pass
else:
extra_data = response_msg[curr_pos:]
break
else:
break
return response, extra_data
def _handle_response_error(self, err):
# If an error gets to this point, log it and kill the connection.
if err.check(DownloadCanceledError, RequestCanceledError, error.ConnectionAborted,
ConnectionClosedBeforeResponseError):
# TODO: (wish-list) it seems silly to close the connection over this, and it shouldn't
# TODO: always be this way. it's done this way now because the client has no other way
# TODO: of telling the server it wants the download to stop. It would be great if the
# TODO: protocol had such a mechanism.
log.info("Closing the connection to %s because the download of blob %s was canceled",
self.peer, self._blob_download_request.blob)
result = None
elif err.check(MisbehavingPeerError):
log.warning("The connection to %s is closing due to: %s", self.peer, err)
result = err
else:
log.error("The connection to %s is closing due to an unexpected error: %s",
self.peer, err)
result = err
self._blob_download_request = None
self._downloading_blob = False
self.transport.loseConnection()
return result
def _handle_response(self, response):
ds = []
log.debug(
"Handling a response from %s. Expected responses: %s. Actual responses: %s",
self.peer, self._response_deferreds.keys(), response.keys())
for key, val in response.items():
if key in self._response_deferreds:
d = self._response_deferreds.pop(key)
d.callback({key: val})
ds.append(d)
for k, d in self._response_deferreds.items():
del self._response_deferreds[k]
d.errback(failure.Failure(NoResponseError()))
ds.append(d)
if self._blob_download_request is not None:
self._downloading_blob = True
d = self._blob_download_request.finished_deferred
d.addErrback(self._handle_response_error)
ds.append(d)
# TODO: are we sure we want to consume errors here
dl = defer.DeferredList(ds, consumeErrors=True)
def get_next_request(results):
failed = False
for success, result in results:
if success is False:
failed = True
if not isinstance(result.value, DownloadCanceledError):
log.info(result.value)
log.info("The connection to %s is closing due to an error: %s",
self.peer, result.getTraceback())
self.peer.report_down()
if failed is False:
log.debug("Asking for another request from %s", self.peer)
self._ask_for_request()
else:
log.warning("Not asking for another request from %s", self.peer)
self.transport.loseConnection()
dl.addCallback(get_next_request)
def _downloading_finished(self, arg):
log.debug("The blob has finished downloading from %s", self.peer)
self._blob_download_request = None
self._downloading_blob = False
return arg
######### IRateLimited #########
def throttle_upload(self):
pass
def unthrottle_upload(self):
pass
def throttle_download(self):
self.transport.pauseProducing()
def unthrottle_download(self):
self.transport.resumeProducing()
class ClientProtocolFactory(ClientFactory):
protocol = ClientProtocol
def __init__(self, peer, rate_limiter, connection_manager):
self.peer = peer
self.rate_limiter = rate_limiter
self.connection_manager = connection_manager
self.p = None
# This defer fires and returns True when connection was
# made and completed, or fires and returns False if
# connection failed
self.connection_was_made_deferred = defer.Deferred()
def clientConnectionFailed(self, connector, reason):
log.debug("Connection failed to %s: %s", self.peer, reason)
self.peer.report_down()
self.connection_was_made_deferred.callback(False)
def buildProtocol(self, addr):
p = self.protocol()
p.factory = self
self.p = p
return p
-27
View File
@@ -1,27 +0,0 @@
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
class ClientRequest:
def __init__(self, request_dict, response_identifier=None):
self.request_dict = request_dict
self.response_identifier = response_identifier
class ClientPaidRequest(ClientRequest):
def __init__(self, request_dict, response_identifier, max_pay_units):
super().__init__(request_dict, response_identifier)
self.max_pay_units = max_pay_units
class ClientBlobRequest(ClientPaidRequest):
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
cancel_func, blob):
if blob.length is None:
max_pay_units = MAX_BLOB_SIZE
else:
max_pay_units = blob.length
super().__init__(request_dict, response_identifier, max_pay_units)
self.write = write_func
self.finished_deferred = finished_deferred
self.cancel = cancel_func
self.blob = blob
-227
View File
@@ -1,227 +0,0 @@
import random
import logging
from twisted.internet import defer, reactor
from lbrynet import conf
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.core import utils
log = logging.getLogger(__name__)
class PeerConnectionHandler:
def __init__(self, request_creators, factory):
self.request_creators = request_creators
self.factory = factory
self.connection = None
class ConnectionManager:
#implements(interfaces.IConnectionManager)
MANAGE_CALL_INTERVAL_SEC = 5
TCP_CONNECT_TIMEOUT = 15
def __init__(self, downloader, rate_limiter,
primary_request_creators, secondary_request_creators):
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
self.downloader = downloader
self.rate_limiter = rate_limiter
self._primary_request_creators = primary_request_creators
self._secondary_request_creators = secondary_request_creators
self._peer_connections = {} # {Peer: PeerConnectionHandler}
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
self._next_manage_call = None
# a deferred that gets fired when a _manage call is set
self._manage_deferred = None
self.stopped = True
log.debug("%s initialized", self._get_log_name())
# this identifies what the connection manager is for,
# used for logging purposes only
def _get_log_name(self):
out = 'Connection Manager Unknown'
if hasattr(self.downloader, 'stream_name'):
out = 'Connection Manager '+self.downloader.stream_name
elif hasattr(self.downloader, 'blob_hash'):
out = 'Connection Manager '+self.downloader.blob_hash
return out
def _start(self):
self.stopped = False
if self._next_manage_call is not None and self._next_manage_call.active() is True:
self._next_manage_call.cancel()
def start(self):
log.debug("%s starting", self._get_log_name())
self._start()
self._next_manage_call = utils.call_later(0, self.manage)
return defer.succeed(True)
@defer.inlineCallbacks
def stop(self):
log.debug("%s stopping", self._get_log_name())
self.stopped = True
# wait for the current manage call to finish
if self._manage_deferred:
yield self._manage_deferred
# in case we stopped between manage calls, cancel the next one
if self._next_manage_call and self._next_manage_call.active():
self._next_manage_call.cancel()
self._next_manage_call = None
yield self._close_peers()
def num_peer_connections(self):
return len(self._peer_connections)
def _close_peers(self):
def disconnect_peer(p):
d = defer.Deferred()
self._connections_closing[p] = d
self._peer_connections[p].connection.disconnect()
if p in self._peer_connections:
del self._peer_connections[p]
return d
def close_connection(p):
log.debug("%s Abruptly closing a connection to %s due to downloading being paused",
self._get_log_name(), p)
if self._peer_connections[p].factory.p is not None:
d = self._peer_connections[p].factory.p.cancel_requests()
else:
d = defer.succeed(True)
d.addBoth(lambda _: disconnect_peer(p))
return d
# fixme: stop modifying dict during iteration
closing_deferreds = [close_connection(peer) for peer in list(self._peer_connections)]
return defer.DeferredList(closing_deferreds)
@defer.inlineCallbacks
def get_next_request(self, peer, protocol):
log.debug("%s Trying to get the next request for peer %s", self._get_log_name(), peer)
if not peer in self._peer_connections or self.stopped is True:
log.debug("%s The peer %s has already been told to shut down.",
self._get_log_name(), peer)
defer.returnValue(False)
requests = yield self._send_primary_requests(peer, protocol)
have_request = any(r[1] for r in requests if r[0] is True)
if have_request:
yield self._send_secondary_requests(peer, protocol)
defer.returnValue(have_request)
def _send_primary_requests(self, peer, protocol):
def handle_error(err):
err.trap(InsufficientFundsError)
self.downloader.insufficient_funds(err)
return False
def check_if_request_sent(request_sent, request_creator):
if peer not in self._peer_connections:
# This can happen if the connection is told to close
return False
if request_sent is False:
if request_creator in self._peer_connections[peer].request_creators:
self._peer_connections[peer].request_creators.remove(request_creator)
else:
if not request_creator in self._peer_connections[peer].request_creators:
self._peer_connections[peer].request_creators.append(request_creator)
return request_sent
ds = []
for p_r_c in self._primary_request_creators:
d = p_r_c.send_next_request(peer, protocol)
d.addErrback(handle_error)
d.addCallback(check_if_request_sent, p_r_c)
ds.append(d)
return defer.DeferredList(ds, fireOnOneErrback=True)
def _send_secondary_requests(self, peer, protocol):
ds = [
s_r_c.send_next_request(peer, protocol)
for s_r_c in self._secondary_request_creators
]
return defer.DeferredList(ds)
@defer.inlineCallbacks
def manage(self, schedule_next_call=True):
self._manage_deferred = defer.Deferred()
if len(self._peer_connections) < self.max_connections_per_stream:
log.debug("%s have %d connections, looking for %d",
self._get_log_name(), len(self._peer_connections),
self.max_connections_per_stream)
peers = yield self._get_new_peers()
for peer in peers:
self._connect_to_peer(peer)
self._manage_deferred.callback(None)
self._manage_deferred = None
if not self.stopped and schedule_next_call:
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
def return_shuffled_peers_not_connected_to(self, peers, new_conns_needed):
out = [peer for peer in peers if peer not in self._peer_connections]
random.shuffle(out)
return out[0:new_conns_needed]
@defer.inlineCallbacks
def _get_new_peers(self):
new_conns_needed = self.max_connections_per_stream - len(self._peer_connections)
if new_conns_needed < 1:
defer.returnValue([])
# we always get the peer from the first request creator
# must be a type BlobRequester...
request_creator = self._primary_request_creators[0]
log.debug("%s Trying to get a new peer to connect to", self._get_log_name())
# find peers for the head blob if configured to do so
if self.seek_head_blob_first:
try:
peers = yield request_creator.get_new_peers_for_head_blob()
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
except KeyError:
log.warning("%s does not have a head blob", self._get_log_name())
peers = []
else:
peers = []
# we didn't find any new peers on the head blob,
# we have to look for the first unavailable blob
if not peers:
peers = yield request_creator.get_new_peers_for_next_unavailable()
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
log.debug("%s Got a list of peers to choose from: %s",
self._get_log_name(), peers)
log.debug("%s Current connections: %s",
self._get_log_name(), self._peer_connections.keys())
log.debug("%s List of connection states: %s", self._get_log_name(),
[p_c_h.connection.state for p_c_h in self._peer_connections.values()])
defer.returnValue(peers)
def _connect_to_peer(self, peer):
if self.stopped:
return
log.debug("%s Trying to connect to %s", self._get_log_name(), peer)
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
factory.connection_was_made_deferred.addCallback(
lambda c_was_made: self._peer_disconnected(c_was_made, peer))
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
factory)
connection = reactor.connectTCP(peer.host, peer.port, factory,
timeout=self.TCP_CONNECT_TIMEOUT)
self._peer_connections[peer].connection = connection
def _peer_disconnected(self, connection_was_made, peer):
log.debug("%s protocol disconnected for %s",
self._get_log_name(), peer)
if peer in self._peer_connections:
del self._peer_connections[peer]
if peer in self._connections_closing:
d = self._connections_closing[peer]
del self._connections_closing[peer]
d.callback(True)
return connection_was_made
-101
View File
@@ -1,101 +0,0 @@
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class DownloadManager:
#implements(interfaces.IDownloadManager)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
self.blob_info_finder = None
self.progress_manager = None
self.blob_handler = None
self.connection_manager = None
self.blobs = {}
self.blob_infos = {}
######### IDownloadManager #########
def start_downloading(self):
d = self.blob_info_finder.get_initial_blobs()
log.debug("Requested the initial blobs from the info finder")
d.addCallback(self.add_blobs_to_download)
d.addCallback(lambda _: self.resume_downloading())
return d
@defer.inlineCallbacks
def resume_downloading(self):
yield self.connection_manager.start()
yield self.progress_manager.start()
defer.returnValue(True)
@defer.inlineCallbacks
def stop_downloading(self):
yield self.progress_manager.stop()
yield self.connection_manager.stop()
defer.returnValue(True)
def add_blobs_to_download(self, blob_infos):
log.debug("Adding %s blobs to blobs", len(blob_infos))
def add_blob_to_list(blob, blob_num):
self.blobs[blob_num] = blob
log.debug(
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
def error_during_add(err):
log.warning(
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
return err
ds = []
for blob_info in blob_infos:
if not blob_info.blob_num in self.blobs:
self.blob_infos[blob_info.blob_num] = blob_info
log.debug(
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
d.addCallback(add_blob_to_list, blob_info.blob_num)
d.addErrback(error_during_add)
ds.append(d)
dl = defer.DeferredList(ds)
return dl
def stream_position(self):
return self.progress_manager.stream_position()
def needed_blobs(self):
return self.progress_manager.needed_blobs()
def final_blob_num(self):
return self.blob_info_finder.final_blob_num()
def handle_blob(self, blob_num):
return self.blob_handler.handle_blob(self.blobs[blob_num], self.blob_infos[blob_num])
def calculate_total_bytes(self):
return sum([bi.length for bi in self.blob_infos.values()])
def calculate_bytes_left_to_output(self):
if not self.blobs:
return self.calculate_total_bytes()
else:
to_be_outputted = [
b for n, b in self.blobs.items()
if n >= self.progress_manager.last_blob_outputted
]
return sum([b.length for b in to_be_outputted if b.length is not None])
def calculate_bytes_left_to_download(self):
if not self.blobs:
return self.calculate_total_bytes()
else:
return sum([b.length for b in self.needed_blobs() if b.length is not None])
def get_head_blob_hash(self):
return self.blobs[0].blob_hash
@@ -1,142 +0,0 @@
import logging
from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.Error import InvalidBlobHashError, DownloadSDTimeout
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
from twisted.python.failure import Failure
from twisted.internet import defer
from twisted.internet.task import LoopingCall
log = logging.getLogger(__name__)
class SingleBlobMetadataHandler:
#implements(interfaces.IMetadataHandler)
def __init__(self, blob_hash, download_manager):
self.blob_hash = blob_hash
self.download_manager = download_manager
######## IMetadataHandler #########
def get_initial_blobs(self):
log.debug("Returning the blob info")
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
def final_blob_num(self):
return 0
class SingleProgressManager:
def __init__(self, download_manager, finished_callback, timeout_callback, timeout):
self.finished_callback = finished_callback
self.timeout_callback = timeout_callback
self.download_manager = download_manager
self.timeout = timeout
self.timeout_counter = 0
self.checker = LoopingCall(self._check_if_finished)
def start(self):
safe_start_looping_call(self.checker, 1)
return defer.succeed(True)
def stop(self):
safe_stop_looping_call(self.checker)
return defer.succeed(True)
def _check_if_finished(self):
if self.stream_position() == 1:
blob_downloaded = self.download_manager.blobs[0]
log.debug("The blob %s has been downloaded. Calling the finished callback",
str(blob_downloaded))
safe_stop_looping_call(self.checker)
self.finished_callback(blob_downloaded)
elif self.timeout is not None:
self.timeout_counter += 1
if self.timeout_counter >= self.timeout:
safe_stop_looping_call(self.checker)
self.timeout_callback()
def stream_position(self):
blobs = self.download_manager.blobs
if blobs and blobs[0].get_is_verified():
return 1
return 0
def needed_blobs(self):
blobs = self.download_manager.blobs
assert len(blobs) == 1
return [b for b in blobs.values() if not b.get_is_verified()]
class DummyBlobHandler:
def __init__(self):
pass
def handle_blob(self, blob, blob_info):
pass
class StandaloneBlobDownloader:
def __init__(self, blob_hash, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, wallet,
timeout=None):
self.blob_hash = blob_hash
self.blob_manager = blob_manager
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self.timeout = timeout
self.download_manager = None
self.finished_deferred = None
def download(self):
if not is_valid_blobhash(self.blob_hash):
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
def cancel_download(d):
self.stop()
self.finished_deferred = defer.Deferred(canceller=cancel_download)
self.download_manager = DownloadManager(self.blob_manager)
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
self.payment_rate_manager, self.wallet,
self.download_manager)
self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash,
self.download_manager)
self.download_manager.progress_manager = SingleProgressManager(self.download_manager,
self._blob_downloaded,
self._download_timedout,
self.timeout)
self.download_manager.blob_handler = DummyBlobHandler()
self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
self.download_manager.connection_manager = ConnectionManager(
self, self.rate_limiter,
[self.download_manager.blob_requester],
[self.download_manager.wallet_info_exchanger]
)
d = self.download_manager.start_downloading()
d.addCallback(lambda _: self.finished_deferred)
return d
def stop(self):
return self.download_manager.stop_downloading()
def _blob_downloaded(self, blob):
self.stop()
if not self.finished_deferred.called:
self.finished_deferred.callback(blob)
def _download_timedout(self):
self.stop()
if not self.finished_deferred.called:
self.finished_deferred.errback(DownloadSDTimeout(self.blob_hash))
def insufficient_funds(self, err):
self.stop()
if not self.finished_deferred.called:
self.finished_deferred.errback(err)
@@ -1,165 +0,0 @@
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class StreamProgressManager:
#implements(IProgressManager)
def __init__(self, finished_callback, blob_manager,
download_manager, delete_blob_after_finished=False):
self.finished_callback = finished_callback
self.blob_manager = blob_manager
self.delete_blob_after_finished = delete_blob_after_finished
self.download_manager = download_manager
self.provided_blob_nums = []
self.last_blob_outputted = -1
self.stopped = True
self._next_try_to_output_call = None
self.outputting_d = None
######### IProgressManager #########
def start(self):
from twisted.internet import reactor
self.stopped = False
self._next_try_to_output_call = reactor.callLater(0, self._try_to_output)
return defer.succeed(True)
def stop(self):
self.stopped = True
if self._next_try_to_output_call is not None and self._next_try_to_output_call.active():
self._next_try_to_output_call.cancel()
self._next_try_to_output_call = None
return self._stop_outputting()
def blob_downloaded(self, blob, blob_num):
if self.outputting_d is None:
self._output_loop()
######### internal #########
def _finished_outputting(self):
self.finished_callback(True)
def _try_to_output(self):
from twisted.internet import reactor
self._next_try_to_output_call = reactor.callLater(1, self._try_to_output)
if self.outputting_d is None:
self._output_loop()
def _output_loop(self):
pass
def _stop_outputting(self):
if self.outputting_d is not None:
return self.outputting_d
return defer.succeed(None)
def _finished_with_blob(self, blob_num):
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
if self.delete_blob_after_finished is True:
log.debug("delete_blob_after_finished is True")
blobs = self.download_manager.blobs
if blob_num in blobs:
log.debug("Telling the blob manager, %s, to delete blob %s",
self.blob_manager, blobs[blob_num].blob_hash)
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
else:
log.debug("Blob number %s was not in blobs", str(blob_num))
else:
log.debug("delete_blob_after_finished is False")
class FullStreamProgressManager(StreamProgressManager):
def __init__(self, finished_callback, blob_manager,
download_manager, delete_blob_after_finished=False):
super().__init__(finished_callback, blob_manager, download_manager,
delete_blob_after_finished)
self.outputting_d = None
######### IProgressManager #########
def _done(self, i, blobs):
"""Return true if `i` is a blob number we don't have"""
return (
i not in blobs or
(
not blobs[i].get_is_verified() and
i not in self.provided_blob_nums
)
)
def stream_position(self):
blobs = self.download_manager.blobs
if not blobs:
return 0
else:
for i in range(max(blobs.keys())):
if self._done(i, blobs):
return i
return max(blobs.keys()) + 1
def needed_blobs(self):
blobs = self.download_manager.blobs
return [
b for n, b in blobs.items()
if not b.get_is_verified() and not n in self.provided_blob_nums
]
######### internal #########
def _output_loop(self):
from twisted.internet import reactor
if self.stopped:
if self.outputting_d is not None:
self.outputting_d.callback(True)
self.outputting_d = None
return
if self.outputting_d is None:
self.outputting_d = defer.Deferred()
blobs = self.download_manager.blobs
def finished_outputting_blob():
self.last_blob_outputted += 1
def check_if_finished():
final_blob_num = self.download_manager.final_blob_num()
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
self._finished_outputting()
self.outputting_d.callback(True)
self.outputting_d = None
else:
reactor.callLater(0, self._output_loop)
current_blob_num = self.last_blob_outputted + 1
if current_blob_num in blobs and blobs[current_blob_num].get_is_verified():
log.debug("Outputting blob %s", str(self.last_blob_outputted + 1))
self.provided_blob_nums.append(self.last_blob_outputted + 1)
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
d.addCallback(lambda _: finished_outputting_blob())
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
d.addCallback(lambda _: check_if_finished())
def log_error(err):
log.warning("Error outputting blob %s: %s", blobs[current_blob_num].blob_hash,
err.getErrorMessage())
if self.outputting_d is not None and not self.outputting_d.called:
self.outputting_d.callback(True)
self.outputting_d = None
self.stop()
d.addErrback(log_error)
else:
self.outputting_d.callback(True)
self.outputting_d = None
-5
View File
@@ -1,5 +0,0 @@
import hashlib
def get_lbry_hash_obj():
return hashlib.sha384()
-273
View File
@@ -1,273 +0,0 @@
import json
import logging
import logging.handlers
import sys
import traceback
import treq
from twisted.internet import defer
import twisted.python.log
from lbrynet import __version__ as lbrynet_version, build_type, conf
from lbrynet.core import utils
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
@defer.inlineCallbacks
def _emit(self, record):
payload = self.format(record)
response = yield treq.post(self.url, data=payload, cookies=self.cookies)
self.cookies.update(response.cookies())
def emit(self, record):
return self._emit(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
"""Configure a logging handler.
`fn` is a function that returns a logging handler. The returned
handler has its log-level set and is attached to the specified
logger or the root logger.
"""
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
if not isinstance(level, int):
# despite the name, getLevelName returns
# the numeric level when passed a text level
level = logging.getLevelName(level)
handler = fn(*args, **kwargs)
configure_handler(handler, log, level)
return handler
return helper
def configure_handler(handler, log, level):
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
# need to reduce the logger's level down to the
# handler's level or else the handler won't
# get those messages
if log.level > level:
log.setLevel(level)
return handler
def disable_third_party_loggers():
logging.getLogger('requests').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
logging.getLogger('lbryum').setLevel(logging.WARNING)
logging.getLogger('twisted').setLevel(logging.CRITICAL)
logging.getLogger('txupnp').setLevel(logging.WARNING)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a log-handler that outputs to stdout"""
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'console'
return handler
@_log_decorator
def configure_file_handler(file_name, **kwargs):
"""Convenience function to configure a log-handler that writes to `file_name`"""
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'file'
return handler
def get_loggly_url(token=None, version=None):
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
version = version or lbrynet_version
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
def configure_loggly_handler():
if build_type.BUILD == 'dev':
return
if not conf.settings['share_usage_data']:
return
level = logging.ERROR
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
session_id=conf.settings.get_session_id())
log = logging.getLogger("lbrynet")
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
# need to reduce the logger's level down to the
# handler's level or else the handler won't
# get those messages
if log.level > level:
log.setLevel(level)
def get_loggly_handler(level, installation_id, session_id):
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
handler = HTTPSHandler(get_loggly_url())
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
def failure(failure, log, msg, *args):
"""Log a failure message from a deferred.
Args:
failure: twisted.python.failure.Failure
log: a python logger instance
msg: the message to log. Can use normal logging string interpolation.
the last argument will be set to the error message from the failure.
args: values to substitute into `msg`
"""
args += (failure.getErrorMessage(),)
exc_info = (failure.type, failure.value, failure.getTracebackObject())
log.error(msg, *args, exc_info=exc_info)
def convert_verbose(verbose):
"""Convert the results of the --verbose flag into a list of logger names
if --verbose is not provided, args.verbose will be None and logging
should be at the info level.
if --verbose is provided, but not followed by any arguments, then
args.verbose = [] and debug logging should be enabled for all of lbrynet
along with info logging on lbryum.
if --verbose is provided and followed by arguments, those arguments
will be in a list
"""
if verbose is None:
return []
if verbose == []:
return ['lbrynet', 'lbryum']
return verbose
def configure_logging(file_name, console, verbose=None):
"""Apply the default logging configuration.
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
Optionally turns on a console logger that defaults to the INFO level, with
specified loggers being set to the DEBUG level.
Args:
file_name: the file to which logs should be saved
console: If true, enable a console logger
verbose: a list of loggers to set to debug level.
See `convert_verbose` for more details.
"""
verbose = convert_verbose(verbose)
configure_twisted()
configure_file_handler(file_name)
disable_third_party_loggers()
if console:
# if there are some loggers at the debug level, we need
# to enable the console to allow debug. Otherwise, only
# allow info.
level = 'DEBUG' if verbose else 'INFO'
handler = configure_console(level=level)
if 'lbryum' in verbose:
# TODO: this enables lbryum logging on the other handlers
# too which isn't consistent with how verbose logging
# happens with other loggers. Should change the configuration
# so that its only logging at the INFO level for the console.
logging.getLogger('lbryum').setLevel(logging.INFO)
verbose.remove('lbryum')
if verbose:
handler.addFilter(LoggerNameFilter(verbose))
def configure_twisted():
"""Setup twisted logging to output events to the python stdlib logger"""
# I tried using the new logging api
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
# and it simply didn't work
observer = twisted.python.log.PythonLoggingObserver()
observer.start()
class LoggerNameFilter:
"""Filter a log record based on its name.
Allows all info level and higher records to pass thru.
Debug records pass if the log record name (or a parent) match
the input list of logger names.
"""
def __init__(self, logger_names):
self.logger_names = logger_names
def filter(self, record):
if record.levelno >= logging.INFO:
return True
name = record.name
while name:
if name in self.logger_names:
return True
name = get_parent(name)
return False
def get_parent(logger_name):
names = logger_name.split('.')
if len(names) == 1:
return ''
names = names[:-1]
return '.'.join(names)
-20
View File
@@ -1,20 +0,0 @@
class LoopingCallManager:
def __init__(self, calls=None):
self.calls = calls or {}
def register_looping_call(self, name, call):
assert name not in self.calls, '{} is already registered'.format(name)
self.calls[name] = call
def start(self, name, *args):
lcall = self.calls[name]
if not lcall.running:
lcall.start(*args)
def stop(self, name):
self.calls[name].stop()
def shutdown(self):
for lcall in self.calls.values():
if lcall.running:
lcall.stop()
@@ -1,57 +0,0 @@
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class BlobAvailabilityHandlerFactory:
# implements(IQueryHandlerFactory)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
######### IQueryHandlerFactory #########
def build_query_handler(self):
q_h = BlobAvailabilityHandler(self.blob_manager)
return q_h
def get_primary_query_identifier(self):
return 'requested_blobs'
def get_description(self):
return "Blob Availability - blobs that are available to be uploaded"
class BlobAvailabilityHandler:
#implements(IQueryHandler)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
self.query_identifiers = ['requested_blobs']
######### IQueryHandler #########
def register_with_request_handler(self, request_handler, peer):
request_handler.register_query_handler(self, self.query_identifiers)
def handle_queries(self, queries):
if self.query_identifiers[0] in queries:
log.info("Received the client's list of requested blobs")
d = self._get_available_blobs(queries[self.query_identifiers[0]])
def set_field(available_blobs):
log.debug("available blobs: %s", str(available_blobs))
return {'available_blobs': available_blobs}
d.addCallback(set_field)
return d
return defer.succeed({})
######### internal #########
def _get_available_blobs(self, requested_blobs):
d = self.blob_manager.completed_blobs(requested_blobs)
return d
-227
View File
@@ -1,227 +0,0 @@
import logging
from twisted.internet import defer
from twisted.protocols.basic import FileSender
from twisted.python.failure import Failure
from lbrynet import analytics
from lbrynet.core.Offer import Offer
log = logging.getLogger(__name__)
class BlobRequestHandlerFactory:
#implements(IQueryHandlerFactory)
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
self.blob_manager = blob_manager
self.wallet = wallet
self.payment_rate_manager = payment_rate_manager
self.analytics_manager = analytics_manager
######### IQueryHandlerFactory #########
def build_query_handler(self):
q_h = BlobRequestHandler(
self.blob_manager, self.wallet, self.payment_rate_manager, self.analytics_manager)
return q_h
def get_primary_query_identifier(self):
return 'requested_blob'
def get_description(self):
return "Blob Uploader - uploads blobs"
class BlobRequestHandler:
#implements(IQueryHandler, IBlobSender)
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
BLOB_QUERY = 'requested_blob'
AVAILABILITY_QUERY = 'requested_blobs'
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
self.analytics_manager = analytics_manager
self.peer = None
self.blob_data_payment_rate = None
self.read_handle = None
self.currently_uploading = None
self.file_sender = None
self.blob_bytes_uploaded = 0
self._blobs_requested = []
######### IQueryHandler #########
def register_with_request_handler(self, request_handler, peer):
self.peer = peer
request_handler.register_query_handler(self, self.query_identifiers)
request_handler.register_blob_sender(self)
def handle_queries(self, queries):
response = defer.succeed({})
log.debug("Handle query: %s", str(queries))
if self.AVAILABILITY_QUERY in queries:
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
if self.PAYMENT_RATE_QUERY in queries:
offered_rate = queries[self.PAYMENT_RATE_QUERY]
offer = Offer(offered_rate)
if offer.rate is None:
log.warning("Empty rate offer")
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
if self.BLOB_QUERY in queries:
incoming = queries[self.BLOB_QUERY]
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
return response
######### IBlobSender #########
def send_blob_if_requested(self, consumer):
if self.currently_uploading is not None:
return self.send_file(consumer)
return defer.succeed(True)
def cancel_send(self, err):
if self.currently_uploading is not None:
self.read_handle.close()
self.read_handle = None
self.currently_uploading = None
return err
######### internal #########
def _reply_to_availability(self, request, blobs):
d = self._get_available_blobs(blobs)
def set_available(available_blobs):
log.debug("available blobs: %s", str(available_blobs))
request.update({'available_blobs': available_blobs})
return request
d.addCallback(set_available)
return d
def _handle_payment_rate_query(self, offer, request):
blobs = self._blobs_requested
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
if reply.is_accepted:
self.blob_data_payment_rate = offer.rate
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
log.debug("Accepted rate: %f", offer.rate)
elif reply.is_too_low:
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
log.debug("Reject rate: %f", offer.rate)
elif reply.is_unset:
log.warning("Rate unset")
request['incoming_blob'] = {'error': 'RATE_UNSET'}
log.debug("Returning rate query result: %s", str(request))
return request
def _handle_blob_query(self, response, query):
log.debug("Received the client's request to send a blob")
response['incoming_blob'] = {}
if self.blob_data_payment_rate is None:
response['incoming_blob'] = {'error': "RATE_UNSET"}
return response
else:
return self._send_blob(response, query)
def _send_blob(self, response, query):
d = self.blob_manager.get_blob(query)
d.addCallback(self.open_blob_for_reading, response)
return d
def open_blob_for_reading(self, blob, response):
response_fields = {}
d = defer.succeed(None)
if blob.get_is_verified():
read_handle = blob.open_for_reading()
if read_handle is not None:
self.currently_uploading = blob
self.read_handle = read_handle
log.info("Sending %s to %s", str(blob), self.peer)
response_fields['blob_hash'] = blob.blob_hash
response_fields['length'] = blob.length
response['incoming_blob'] = response_fields
d.addCallback(lambda _: response)
return d
log.debug("We can not send %s", str(blob))
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
d.addCallback(lambda _: response)
return d
def _reply_to_send_request(self, response, incoming):
response_fields = {}
response['incoming_blob'] = response_fields
if self.blob_data_payment_rate is None:
log.debug("Rate not set yet")
response['incoming_blob'] = {'error': 'RATE_UNSET'}
return defer.succeed(response)
else:
log.debug("Requested blob: %s", str(incoming))
d = self.blob_manager.get_blob(incoming)
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
return d
def _get_available_blobs(self, requested_blobs):
d = self.blob_manager.completed_blobs(requested_blobs)
return d
def send_file(self, consumer):
def _send_file():
inner_d = start_transfer()
# TODO: if the transfer fails, check if it's because the connection was cut off.
# TODO: if so, perhaps bill the client
inner_d.addCallback(lambda _: set_expected_payment())
inner_d.addBoth(set_not_uploading)
return inner_d
def count_bytes(data):
uploaded = len(data)
self.blob_bytes_uploaded += uploaded
self.peer.update_stats('blob_bytes_uploaded', uploaded)
if self.analytics_manager is not None:
self.analytics_manager.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
return data
def start_transfer():
self.file_sender = FileSender()
log.debug("Starting the file upload")
assert self.read_handle is not None, \
"self.read_handle was None when trying to start the transfer"
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
return d
def set_expected_payment():
log.debug("Setting expected payment")
if (
self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None
and self.blob_data_payment_rate > 0
):
# TODO: explain why 2**20
self.wallet.add_expected_payment(self.peer,
self.currently_uploading.length * 1.0 *
self.blob_data_payment_rate / 2 ** 20)
self.blob_bytes_uploaded = 0
self.peer.update_stats('blobs_uploaded', 1)
return None
def set_not_uploading(reason=None):
if self.currently_uploading is not None:
self.read_handle.close()
self.read_handle = None
self.currently_uploading = None
self.file_sender = None
if reason is not None and isinstance(reason, Failure):
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
return _send_file()
-92
View File
@@ -1,92 +0,0 @@
import logging
from twisted.internet import error
from twisted.internet.protocol import Protocol, ServerFactory
from twisted.python import failure
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
log = logging.getLogger(__name__)
class ServerProtocol(Protocol):
"""ServerProtocol needs to:
1) Receive requests from its transport
2) Pass those requests on to its request handler
3) Tell the request handler to pause/resume producing
4) Tell its transport to pause/resume producing
5) Hang up when the request handler is done producing
6) Tell the request handler to stop producing if the connection is lost
7) Upon creation, register with the rate limiter
8) Upon connection loss, unregister with the rate limiter
9) Report all uploaded and downloaded bytes to the rate limiter
10) Pause/resume production when told by the rate limiter
"""
#implements(interfaces.IConsumer)
#Protocol stuff
def connectionMade(self):
log.debug("Got a connection")
peer_info = self.transport.getPeer()
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
self.request_handler = ServerRequestHandler(self)
for query_handler_factory in self.factory.query_handler_factories.values():
query_handler = query_handler_factory.build_query_handler()
query_handler.register_with_request_handler(self.request_handler, self.peer)
log.debug("Setting the request handler")
self.factory.rate_limiter.register_protocol(self)
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
if self.request_handler is not None:
self.request_handler.stopProducing()
self.factory.rate_limiter.unregister_protocol(self)
if not reason.check(error.ConnectionDone):
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
def dataReceived(self, data):
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
self.factory.rate_limiter.report_dl_bytes(len(data))
if self.request_handler is not None:
self.request_handler.data_received(data)
#IConsumer stuff
def registerProducer(self, producer, streaming):
log.debug("Registering the producer")
assert streaming is True
def unregisterProducer(self):
self.request_handler = None
self.transport.loseConnection()
def write(self, data):
log.trace("Writing %s bytes of data to the transport", len(data))
self.transport.write(data)
self.factory.rate_limiter.report_ul_bytes(len(data))
#Rate limiter stuff
def throttle_upload(self):
if self.request_handler is not None:
self.request_handler.pauseProducing()
def unthrottle_upload(self):
if self.request_handler is not None:
self.request_handler.resumeProducing()
def throttle_download(self):
self.transport.pauseProducing()
def unthrottle_download(self):
self.transport.resumeProducing()
class ServerProtocolFactory(ServerFactory):
protocol = ServerProtocol
def __init__(self, rate_limiter, query_handler_factories, peer_manager):
self.rate_limiter = rate_limiter
self.query_handler_factories = query_handler_factories
self.peer_manager = peer_manager
-184
View File
@@ -1,184 +0,0 @@
import json
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class ServerRequestHandler:
"""This class handles requests from clients. It can upload blobs and
return request for information about more blobs that are
associated with streams.
"""
#implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
def __init__(self, consumer):
self.consumer = consumer
self.production_paused = False
self.request_buff = b''
self.response_buff = b''
self.producer = None
self.request_received = False
self.CHUNK_SIZE = 2**14
self.query_handlers = {} # {IQueryHandler: [query_identifiers]}
self.blob_sender = None
self.consumer.registerProducer(self, True)
#IPushProducer stuff
def pauseProducing(self):
self.production_paused = True
def stopProducing(self):
if self.producer is not None:
self.producer.stopProducing()
self.producer = None
self.production_paused = True
self.consumer.unregisterProducer()
def resumeProducing(self):
from twisted.internet import reactor
self.production_paused = False
self._produce_more()
if self.producer is not None:
reactor.callLater(0, self.producer.resumeProducing)
def _produce_more(self):
from twisted.internet import reactor
if self.production_paused:
return
chunk = self.response_buff[:self.CHUNK_SIZE]
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
if chunk == b'':
return
log.trace("writing %s bytes to the client", len(chunk))
self.consumer.write(chunk)
reactor.callLater(0, self._produce_more)
#IConsumer stuff
def registerProducer(self, producer, streaming):
self.producer = producer
assert streaming is False
producer.resumeProducing()
def unregisterProducer(self):
self.producer = None
def write(self, data):
from twisted.internet import reactor
self.response_buff = self.response_buff + data
self._produce_more()
def get_more_data():
if self.producer is not None:
log.trace("Requesting more data from the producer")
self.producer.resumeProducing()
reactor.callLater(0, get_more_data)
#From Protocol
def data_received(self, data):
log.debug("Received data")
log.debug("%s", str(data))
if self.request_received is False:
return self._parse_data_and_maybe_send_blob(data)
else:
log.warning(
"The client sent data when we were uploading a file. This should not happen")
def _parse_data_and_maybe_send_blob(self, data):
self.request_buff = self.request_buff + data
msg = self.try_to_parse_request(self.request_buff)
if msg:
self.request_buff = b''
self._process_msg(msg)
else:
log.debug("Request buff not a valid json message")
log.debug("Request buff: %s", self.request_buff)
def _process_msg(self, msg):
d = self.handle_request(msg)
if self.blob_sender:
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
######### IRequestHandler #########
def register_query_handler(self, query_handler, query_identifiers):
self.query_handlers[query_handler] = query_identifiers
def register_blob_sender(self, blob_sender):
self.blob_sender = blob_sender
#response handling
def request_failure_handler(self, err):
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
self.stopProducing()
return err
def finished_response(self):
self.request_received = False
self._produce_more()
def send_response(self, msg):
m = json.dumps(msg).encode()
log.debug("Sending a response of length %s", str(len(m)))
log.debug("Response: %s", str(m))
self.response_buff = self.response_buff + m
self._produce_more()
return True
def handle_request(self, msg):
log.debug("Handling a request")
log.debug(str(msg))
def create_response_message(results):
response = {}
for success, result in results:
if success is True:
response.update(result)
else:
# result is a Failure
return result
log.debug("Finished making the response message. Response: %s", str(response))
return response
def log_errors(err):
log.warning(
"An error occurred handling a client request. Error message: %s",
err.getErrorMessage())
return err
def send_response(response):
self.send_response(response)
return True
ds = []
for query_handler, query_identifiers in self.query_handlers.items():
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
d = query_handler.handle_queries(queries)
d.addErrback(log_errors)
ds.append(d)
dl = defer.DeferredList(ds)
dl.addCallback(create_response_message)
dl.addCallback(send_response)
return dl
def try_to_parse_request(self, request_buff):
try:
msg = json.loads(request_buff)
return msg
except ValueError:
return None
-59
View File
@@ -1,59 +0,0 @@
import platform
import json
import subprocess
import os
from six.moves.urllib import request
from six.moves.urllib.error import URLError
from lbryschema import __version__ as lbryschema_version
from lbrynet import build_type, __version__ as lbrynet_version
from lbrynet.conf import ROOT_DIR
import logging.handlers
log = logging.getLogger(__name__)
def get_lbrynet_version():
if build_type.BUILD == "dev":
try:
with open(os.devnull, 'w') as devnull:
git_dir = ROOT_DIR + '/.git'
return subprocess.check_output(
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
stderr=devnull
).decode().strip().lstrip('v')
except (subprocess.CalledProcessError, OSError):
log.debug("failed to get version from git")
return lbrynet_version
def get_platform(get_ip=True):
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": get_lbrynet_version(),
"lbryschema_version": lbryschema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
try:
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
except ModuleNotFoundError:
pass
# TODO: remove this from get_platform and add a get_external_ip function using treq
if get_ip:
try:
response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
if not response['success']:
raise URLError("failed to get external ip")
p['ip'] = response['data']['ip']
except (URLError, AssertionError):
p['ip'] = "Could not determine IP"
return p
-248
View File
@@ -1,248 +0,0 @@
import base64
import codecs
import datetime
import random
import socket
import string
import json
import traceback
import functools
import logging
import pkg_resources
from twisted.python.failure import Failure
from twisted.internet import defer
from lbryschema.claim import ClaimDict
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
# defining these time functions here allows for easier overriding in testing
def now():
return datetime.datetime.now()
def utcnow():
return datetime.datetime.utcnow()
def isonow():
"""Return utc now in isoformat with timezone"""
return utcnow().isoformat() + 'Z'
def today():
return datetime.datetime.today()
def timedelta(**kwargs):
return datetime.timedelta(**kwargs)
def datetime_obj(*args, **kwargs):
return datetime.datetime(*args, **kwargs)
def call_later(delay, func, *args, **kwargs):
# Import here to ensure that it gets called after installing a reactor
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def safe_start_looping_call(looping_call, interval_sec):
if not looping_call.running:
looping_call.start(interval_sec)
def safe_stop_looping_call(looping_call):
if looping_call.running:
looping_call.stop()
def generate_id(num=None):
h = get_lbry_hash_obj()
if num is not None:
h.update(str(num).encode())
else:
h.update(str(random.getrandbits(512)).encode())
return h.digest()
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def version_is_greater_than(a, b):
"""Returns True if version a is more recent than version b"""
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
def rot13(some_str):
return codecs.encode(some_str, 'rot_13')
def deobfuscate(obfustacated):
return base64.b64decode(rot13(obfustacated))
def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5):
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
server = socket.gethostbyname(server)
socket.create_connection((server, port), timeout)
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
log.warning("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
server, port)
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout)
log.debug('Connection successful')
return True
except Exception as ex:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
except Exception as ex:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
def short_hash(hash_str):
return hash_str[:6]
def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, ClaimDict):
return stream_info.source_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\
get('source', {}).\
get('source')
if not result:
log.warning("Unable to get sd_hash")
return result
def json_dumps_pretty(obj, **kwargs):
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
class DeferredLockContextManager:
def __init__(self, lock):
self._lock = lock
def __enter__(self):
yield self._lock.aquire()
def __exit__(self, exc_type, exc_val, exc_tb):
yield self._lock.release()
@defer.inlineCallbacks
def DeferredDict(d, consumeErrors=False):
keys = []
dl = []
response = {}
for k, v in d.items():
keys.append(k)
dl.append(v)
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
for k, (success, result) in zip(keys, results):
if success:
response[k] = result
defer.returnValue(response)
class DeferredProfiler:
def __init__(self):
self.profile_results = {}
def add_result(self, fn, start_time, finished_time, stack, success):
self.profile_results[fn].append((start_time, finished_time, stack, success))
def show_profile_results(self, fn):
profile_results = list(self.profile_results[fn])
call_counts = {
caller: [(start, finished, finished - start, success)
for (start, finished, _caller, success) in profile_results
if _caller == caller]
for caller in set(result[2] for result in profile_results)
}
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
call_info = call_counts[caller]
times = [r[2] for r in call_info]
own_time = sum(times)
times.sort()
longest = 0 if not times else times[-1]
shortest = 0 if not times else times[0]
log.info(
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
len([r for r in call_info if r[3]]),
len([r for r in call_info if not r[3]]),
longest, shortest, own_time / float(len(call_info)), caller
)
def profiled_deferred(self, reactor=None):
if not reactor:
from twisted.internet import reactor
def _cb(result, fn, start, caller_info):
if isinstance(result, (Failure, Exception)):
error = result
result = None
else:
error = None
self.add_result(fn, start, reactor.seconds(), caller_info, error is None)
if error is None:
return result
raise error
def _profiled_deferred(fn):
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
self.profile_results[fn] = []
@functools.wraps(fn)
def _wrapper(*args, **kwargs):
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
start = reactor.seconds()
d = defer.maybeDeferred(fn, *args, **kwargs)
d.addBoth(_cb, fn, start, caller_info)
return d
return _wrapper
return _profiled_deferred
_profiler = DeferredProfiler()
profile_deferred = _profiler.profiled_deferred
+9
View File
@@ -0,0 +1,9 @@
import hashlib
from cryptography.hazmat.backends import default_backend
backend = default_backend()
def get_lbry_hash_obj():
return hashlib.sha384()
-166
View File
@@ -1,166 +0,0 @@
import binascii
import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
log = logging.getLogger(__name__)
backend = default_backend()
class CryptBlobInfo(BlobInfo):
def __init__(self, blob_hash, blob_num, length, iv):
super().__init__(blob_hash, blob_num, length)
self.iv = iv
def get_dict(self):
info = {
"blob_num": self.blob_num,
"length": self.length,
"iv": self.iv.decode()
}
if self.blob_hash:
info['blob_hash'] = self.blob_hash
return info
class StreamBlobDecryptor:
def __init__(self, blob, key, iv, length):
"""
This class decrypts blob
blob - object which implements read() function.
key = encryption_key
iv = initialization vector
blob_num = blob number (has no effect on encryption)
length = length in bytes of blob
"""
self.blob = blob
self.key = key
self.iv = iv
self.length = length
self.buff = b''
self.len_read = 0
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
self.unpadder = PKCS7(AES.block_size).unpadder()
self.cipher = cipher.decryptor()
def decrypt(self, write_func):
"""
Decrypt blob and write its content useing write_func
write_func - function that takes decrypted string as
arugment and writes it somewhere
Returns:
deferred that returns after decrypting blob and writing content
"""
def remove_padding(data):
return self.unpadder.update(data) + self.unpadder.finalize()
def write_bytes():
if self.len_read < self.length:
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
write_func(self.cipher.update(data_to_decrypt))
def finish_decrypt():
bytes_left = len(self.buff) % (AES.block_size // 8)
if bytes_left != 0:
log.warning(self.buff[-1 * (AES.block_size // 8):].encode('hex'))
raise Exception("blob %s has incorrect padding: %i bytes left" %
(self.blob.blob_hash, bytes_left))
data_to_decrypt, self.buff = self.buff, b''
last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize()
write_func(remove_padding(last_chunk))
read_handle = self.blob.open_for_reading()
@defer.inlineCallbacks
def decrypt_bytes():
producer = FileBodyProducer(read_handle)
buff = BytesIO()
yield producer.startProducing(buff)
self.buff = buff.getvalue()
self.len_read += len(self.buff)
write_bytes()
finish_decrypt()
d = decrypt_bytes()
return d
class CryptStreamBlobMaker:
def __init__(self, key, iv, blob_num, blob):
"""
This class encrypts data and writes it to a new blob
key = encryption_key
iv = initialization vector
blob_num = blob number (has no effect on encryption)
blob = object which implements write(), close() function , close() function must
be a deferred. (Will generally be of HashBlobCreator type)
"""
self.key = key
self.iv = iv
self.blob_num = blob_num
self.blob = blob
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
self.padder = PKCS7(AES.block_size).padder()
self.cipher = cipher.encryptor()
self.length = 0
def write(self, data):
"""
encrypt and write string data
Returns:
tuple (done, num_bytes_to_write) where done is True if
max bytes are written. num_bytes_to_write is the number
of bytes that will be written from data in this call
"""
max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1
done = False
if max_bytes_to_write <= len(data):
num_bytes_to_write = max_bytes_to_write
done = True
else:
num_bytes_to_write = len(data)
data_to_write = data[:num_bytes_to_write]
self.length += len(data_to_write)
padded_data = self.padder.update(data_to_write)
encrypted_data = self.cipher.update(padded_data)
self.blob.write(encrypted_data)
return done, num_bytes_to_write
@defer.inlineCallbacks
def close(self):
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
if self.length != 0:
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
padded_data = self.padder.finalize()
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
self.blob.write(encrypted_data)
blob_hash = yield self.blob.close()
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
blob = CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
defer.returnValue(blob)
def greatest_multiple(a, b):
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
return (a // b) * b
def split(buff, cutoff):
return buff[:cutoff], buff[cutoff:]
-149
View File
@@ -1,149 +0,0 @@
"""
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
"""
import os
import logging
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from twisted.internet import defer
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
log = logging.getLogger(__name__)
class CryptStreamCreator:
"""
Create a new stream with blobs encrypted by a symmetric cipher.
Each blob is encrypted with the same key, but each blob has its
own initialization vector which is associated with the blob when
the blob is associated with the stream.
"""
#implements(interfaces.IConsumer)
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
"""@param blob_manager: Object that stores and provides access to blobs.
@type blob_manager: BlobManager
@param name: the name of the stream, which will be presented to the user
@type name: string
@param key: the raw AES key which will be used to encrypt the
blobs. If None, a random key will be generated.
@type key: string
@param iv_generator: a generator which yields initialization
vectors for the blobs. Will be called once for each blob.
@type iv_generator: a generator function which yields strings
@return: None
"""
self.blob_manager = blob_manager
self.name = name
self.key = key
if iv_generator is None:
self.iv_generator = self.random_iv_generator()
else:
self.iv_generator = iv_generator
self.stopped = True
self.producer = None
self.streaming = None
self.blob_count = -1
self.current_blob = None
self.finished_deferreds = []
def registerProducer(self, producer, streaming):
from twisted.internet import reactor
self.producer = producer
self.streaming = streaming
self.stopped = False
if streaming is False:
reactor.callLater(0, self.producer.resumeProducing)
def unregisterProducer(self):
self.stopped = True
self.producer = None
def _close_current_blob(self):
# close the blob that was being written to
# and save it to blob manager
should_announce = self.blob_count == 0
d = self.current_blob.close()
d.addCallback(self._blob_finished)
d.addCallback(lambda blob_info: self.blob_manager.creator_finished(blob_info,
should_announce))
self.finished_deferreds.append(d)
self.current_blob = None
def stop(self):
"""Stop creating the stream. Create the terminating zero-length blob."""
log.debug("stop has been called for StreamCreator")
self.stopped = True
if self.current_blob is not None:
self._close_current_blob()
d = self._finalize()
d.addCallback(lambda _: self._finished())
return d
# TODO: move the stream creation process to its own thread and
# remove the reactor from this process.
def write(self, data):
from twisted.internet import reactor
self._write(data)
if self.stopped is False and self.streaming is False:
reactor.callLater(0, self.producer.resumeProducing)
@staticmethod
def random_iv_generator():
while 1:
yield os.urandom(AES.block_size // 8)
def setup(self):
"""Create the symmetric key if it wasn't provided"""
if self.key is None:
self.key = os.urandom(AES.block_size // 8)
return defer.succeed(True)
@defer.inlineCallbacks
def _finalize(self):
"""
Finalize a stream by adding an empty
blob at the end, this is to indicate that
the stream has ended. This empty blob is not
saved to the blob manager
"""
yield defer.DeferredList(self.finished_deferreds)
self.blob_count += 1
iv = next(self.iv_generator)
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
stream_terminator = yield final_blob.close()
terminator_info = yield self._blob_finished(stream_terminator)
defer.returnValue(terminator_info)
def _write(self, data):
while len(data) > 0:
if self.current_blob is None:
self.next_blob_creator = self.blob_manager.get_blob_creator()
self.blob_count += 1
iv = next(self.iv_generator)
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
done, num_bytes_written = self.current_blob.write(data)
data = data[num_bytes_written:]
if done is True:
self._close_current_blob()
def _get_blob_maker(self, iv, blob_creator):
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
def _finished(self):
raise NotImplementedError()
def _blob_finished(self, blob_info):
raise NotImplementedError()
-8
View File
@@ -1,8 +0,0 @@
"""
Classes and functions for dealing with Crypt Streams.
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
metadata is generally stored in a Stream Descriptor File, for example containing a public key
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
may or may not be present.
"""
@@ -1,22 +0,0 @@
import binascii
from twisted.internet import defer
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
class CryptBlobHandler:
#implements(IBlobHandler)
def __init__(self, key, write_func):
self.key = key
self.write_func = write_func
######## IBlobHandler #########
def handle_blob(self, blob, blob_info):
try:
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv),
blob_info.length)
except ValueError as err:
return defer.fail(err)
d = blob_decryptor.decrypt(self.write_func)
return d
@@ -1,219 +0,0 @@
from binascii import unhexlify
import logging
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
from twisted.internet import defer
from twisted.python.failure import Failure
log = logging.getLogger(__name__)
class StartFailedError(Exception):
pass
class AlreadyRunningError(Exception):
pass
class AlreadyStoppedError(Exception):
pass
class CurrentlyStoppingError(Exception):
pass
class CurrentlyStartingError(Exception):
pass
class CryptStreamDownloader:
#implements(IStreamDownloader)
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
key, stream_name):
"""Initialize a CryptStreamDownloader
@param peer_finder: An object which implements the IPeerFinder
interface. Used to look up peers by a hashsum.
@param rate_limiter: An object which implements the IRateLimiter interface
@param blob_manager: A BlobManager object
@param payment_rate_manager: A NegotiatedPaymentRateManager object
@param wallet: An object which implements the IWallet interface
@return:
"""
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self.key = unhexlify(key)
self.stream_name = unhexlify(stream_name).decode()
self.completed = False
self.stopped = True
self.stopping = False
self.starting = False
self.download_manager = None
self.finished_deferred = None
self.points_paid = 0.0
self.blob_requester = None
def __str__(self):
return str(self.stream_name)
def toggle_running(self):
if self.stopped is True:
return self.start()
else:
return self.stop()
def start(self):
if self.starting is True:
raise CurrentlyStartingError()
if self.stopping is True:
raise CurrentlyStoppingError()
if self.stopped is False:
raise AlreadyRunningError()
assert self.download_manager is None
self.starting = True
self.completed = False
self.finished_deferred = defer.Deferred()
d = self._start()
d.addCallback(lambda _: self.finished_deferred)
return d
@defer.inlineCallbacks
def stop(self, err=None):
if self.stopped is True:
raise AlreadyStoppedError()
if self.stopping is True:
raise CurrentlyStoppingError()
assert self.download_manager is not None
self.stopping = True
success = yield self.download_manager.stop_downloading()
self.stopping = False
if success is True:
self.stopped = True
self._remove_download_manager()
yield self._fire_completed_deferred(err)
def _start_failed(self):
def set_stopped():
self.stopped = True
self.stopping = False
self.starting = False
if self.download_manager is not None:
d = self.download_manager.stop_downloading()
d.addCallback(lambda _: self._remove_download_manager())
else:
d = defer.succeed(True)
d.addCallback(lambda _: set_stopped())
d.addCallback(lambda _: Failure(StartFailedError()))
return d
def _start(self):
def check_start_succeeded(success):
if success:
self.starting = False
self.stopped = False
self.completed = False
return True
else:
return self._start_failed()
self.download_manager = self._get_download_manager()
d = self.download_manager.start_downloading()
d.addCallbacks(check_start_succeeded)
return d
def _get_download_manager(self):
assert self.blob_requester is None
download_manager = DownloadManager(self.blob_manager)
# TODO: can we get rid of these circular references. I'm not
# smart enough to handle thinking about the interactions
# between them and have hope that there is a simpler way
# to accomplish what we want
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
download_manager.progress_manager = self._get_progress_manager(download_manager)
download_manager.blob_handler = self._get_blob_handler(download_manager)
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
# blob_requester needs to be set before the connection manager is setup
self.blob_requester = self._get_blob_requester(download_manager)
download_manager.connection_manager = self._get_connection_manager(download_manager)
return download_manager
def _remove_download_manager(self):
self.download_manager.blob_info_finder = None
self.download_manager.progress_manager = None
self.download_manager.blob_handler = None
self.download_manager.wallet_info_exchanger = None
self.blob_requester = None
self.download_manager.connection_manager = None
self.download_manager = None
def _get_primary_request_creators(self, download_manager):
return [self.blob_requester]
def _get_secondary_request_creators(self, download_manager):
return [download_manager.wallet_info_exchanger]
def _get_metadata_handler(self, download_manager):
pass
def _get_blob_requester(self, download_manager):
return BlobRequester(self.blob_manager, self.peer_finder,
self.payment_rate_manager, self.wallet,
download_manager)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
def _get_write_func(self):
pass
def _get_blob_handler(self, download_manager):
return CryptBlobHandler(self.key, self._get_write_func())
def _get_connection_manager(self, download_manager):
return ConnectionManager(self, self.rate_limiter,
self._get_primary_request_creators(download_manager),
self._get_secondary_request_creators(download_manager))
def _fire_completed_deferred(self, err=None):
self.finished_deferred, d = None, self.finished_deferred
if d is not None:
if err is not None:
d.errback(err)
else:
value = self._get_finished_deferred_callback_value()
d.callback(value)
else:
log.debug("Not firing the completed deferred because d is None")
def _get_finished_deferred_callback_value(self):
return None
def _finished_downloading(self, finished):
if finished is True:
self.completed = True
return self.stop()
def insufficient_funds(self, err):
return self.stop(err=err)
-106
View File
@@ -1,106 +0,0 @@
import os
import sys
import inspect
import logging
TRACE = 5
####
# This code is copied from logging/__init__.py in the python source code
####
#
# _srcfile is used when walking the stack to check when we've got the first
# caller stack frame.
#
if hasattr(sys, 'frozen'): # support for py2exe
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
_srcfile = __file__[:-4] + '.py'
else:
_srcfile = __file__
_srcfile = os.path.normcase(_srcfile)
def findCaller(srcfile=None):
"""Returns the filename, line number and function name of the caller"""
srcfile = srcfile or _srcfile
f = inspect.currentframe()
# On some versions of IronPython, currentframe() returns None if
# IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
rv = "(unknown file)", 0, "(unknown function)"
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
# ignore any function calls that are in this file
if filename == srcfile:
f = f.f_back
continue
rv = (filename, f.f_lineno, co.co_name)
break
return rv
###
class Logger(logging.Logger):
"""A logger that has an extra `fail` method useful for handling twisted failures."""
def fail(self, callback=None, *args, **kwargs):
"""Returns a function to log a failure from an errback.
The returned function appends the error message and extracts
the traceback from `err`.
Example usage:
d.addErrback(log.fail(), 'This is an error message')
Although odd, making the method call is necessary to extract
out useful filename and line number information; otherwise the
reported values are from inside twisted's deferred handling
code.
Args:
callback: callable to call after making the log. The first argument
will be the `err` from the deferred
args: extra arguments to pass into `callback`
Returns: a function that takes the following arguments:
err: twisted.python.failure.Failure
msg: the message to log, using normal logging string iterpolation.
msg_args: the values to subtitute into `msg`
msg_kwargs: set `level` to change from the default ERROR severity. Other
keywoards are treated as normal log kwargs.
"""
fn, lno, func = findCaller()
def _fail(err, msg, *msg_args, **msg_kwargs):
level = msg_kwargs.pop('level', logging.ERROR)
msg += ": %s"
msg_args += (err.getErrorMessage(),)
exc_info = (err.type, err.value, err.getTracebackObject())
record = self.makeRecord(
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
self.handle(record)
if callback:
try:
return callback(err, *args, **kwargs)
except Exception:
# log.fail is almost always called within an
# errback. If callback fails and we didn't catch
# the exception we would need to attach a second
# errback to deal with that, which we will almost
# never do and then we end up with an unhandled
# error that will get swallowed by twisted
self.exception('Failed to run callback')
return _fail
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(TRACE):
self._log(TRACE, msg, args, **kwargs)
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')
-737
View File
@@ -1,737 +0,0 @@
import os
import logging
import treq
import math
import binascii
from hashlib import sha256
from types import SimpleNamespace
from twisted.internet import defer, threads, reactor, error
import lbryschema
from txupnp.upnp import UPnP
from lbrynet import conf
from lbrynet.core.utils import DeferredDict
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.core.RateLimiter import RateLimiter
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.wallet.manager import LbryWalletManager
from lbrynet.wallet.network import Network
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
from lbrynet.daemon.Component import Component
from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
from lbrynet.database.storage import SQLiteStorage
from lbrynet.dht import node, hashannouncer
from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.reflector import ServerFactory as reflector_server_factory
from lbrynet.core.utils import generate_id
log = logging.getLogger(__name__)
# settings must be initialized before this file is imported
DATABASE_COMPONENT = "database"
BLOB_COMPONENT = "blob_manager"
HEADERS_COMPONENT = "blockchain_headers"
WALLET_COMPONENT = "wallet"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
FILE_MANAGER_COMPONENT = "file_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
REFLECTOR_COMPONENT = "reflector"
UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
RATE_LIMITER_COMPONENT = "rate_limiter"
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
def get_wallet_config():
wallet_type = GCS('wallet')
if wallet_type == conf.LBRYCRD_WALLET:
raise ValueError('LBRYcrd Wallet is no longer supported')
elif wallet_type != conf.LBRYUM_WALLET:
raise ValueError('Wallet Type {} is not valid'.format(wallet_type))
lbryum_servers = {address: {'t': str(port)}
for address, port in GCS('lbryum_servers')}
config = {
'auto_connect': True,
'chain': GCS('blockchain_name'),
'default_servers': lbryum_servers
}
if 'use_keyring' in conf.settings:
config['use_keyring'] = GCS('use_keyring')
if conf.settings['lbryum_wallet_dir']:
config['lbryum_path'] = GCS('lbryum_wallet_dir')
return config
class ConfigSettings:
@staticmethod
def get_conf_setting(setting_name):
return conf.settings[setting_name]
@staticmethod
def get_blobfiles_dir():
if conf.settings['BLOBFILES_DIR'] == "blobfiles":
return os.path.join(GCS("data_dir"), "blobfiles")
else:
log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR'])
return conf.settings['BLOBFILES_DIR']
@staticmethod
def get_node_id():
return conf.settings.node_id
@staticmethod
def get_external_ip():
from lbrynet.core.system_info import get_platform
platform = get_platform(get_ip=True)
return platform['ip']
# Shorthand for common ConfigSettings methods
CS = ConfigSettings
GCS = ConfigSettings.get_conf_setting
class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.storage = None
@property
def component(self):
return self.storage
@staticmethod
def get_current_db_revision():
return 9
@staticmethod
def get_revision_filename():
return conf.settings.get_db_revision_filename()
@staticmethod
def _write_db_revision_file(version_num):
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
db_revision.write(str(version_num))
@defer.inlineCallbacks
def start(self):
# check directories exist, create them if they don't
log.info("Loading databases")
if not os.path.exists(GCS('download_directory')):
os.mkdir(GCS('download_directory'))
if not os.path.exists(GCS('data_dir')):
os.mkdir(GCS('data_dir'))
self._write_db_revision_file(self.get_current_db_revision())
log.debug("Created the db revision file: %s", self.get_revision_filename())
if not os.path.exists(CS.get_blobfiles_dir()):
os.mkdir(CS.get_blobfiles_dir())
log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir()))
if not os.path.exists(self.get_revision_filename()):
log.warning("db_revision file not found. Creating it")
self._write_db_revision_file(self.get_current_db_revision())
# check the db migration and run any needed migrations
with open(self.get_revision_filename(), "r") as revision_read_handle:
old_revision = int(revision_read_handle.read().strip())
if old_revision > self.get_current_db_revision():
raise Exception('This version of lbrynet is not compatible with the database\n'
'Your database is revision %i, expected %i' %
(old_revision, self.get_current_db_revision()))
if old_revision < self.get_current_db_revision():
from lbrynet.database.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
yield threads.deferToThread(
dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
# start SQLiteStorage
self.storage = SQLiteStorage(GCS('data_dir'))
yield self.storage.setup()
@defer.inlineCallbacks
def stop(self):
yield self.storage.stop()
self.storage = None
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
HEADER_SIZE = 112
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.headers_dir = os.path.join(conf.settings['lbryum_wallet_dir'], 'lbc_mainnet')
self.headers_file = os.path.join(self.headers_dir, 'headers')
self.old_file = os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')
self._downloading_headers = None
self._headers_progress_percent = None
@property
def component(self):
return self
def get_status(self):
return {} if not self._downloading_headers else {
'downloading_headers': self._downloading_headers,
'download_progress': self._headers_progress_percent
}
@defer.inlineCallbacks
def fetch_headers_from_s3(self):
def collector(data, h_file):
h_file.write(data)
local_size = float(h_file.tell())
final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size()
resume_header = {"Range": "bytes={}-".format(local_header_size)}
response = yield treq.get(HEADERS_URL, headers=resume_header)
got_406 = response.code == 406 # our file is bigger
final_size_after_download = response.length + local_header_size
if got_406:
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.length)
with open(self.headers_file, "a+b") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
else:
with open(self.headers_file, "wb") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
def local_header_file_size(self):
if os.path.isfile(self.headers_file):
return os.stat(self.headers_file).st_size
return 0
@defer.inlineCallbacks
def get_remote_height(self):
ledger = SimpleNamespace()
ledger.config = {
'default_servers': conf.settings['lbryum_servers'],
'data_path': conf.settings['lbryum_wallet_dir']
}
net = Network(ledger)
net.start()
yield net.on_connected.first
remote_height = yield net.get_server_height()
yield net.stop()
defer.returnValue(remote_height)
@defer.inlineCallbacks
def should_download_headers_from_s3(self):
if conf.settings['blockchain_name'] != "lbrycrd_main":
defer.returnValue(False)
self._check_header_file_integrity()
s3_headers_depth = conf.settings['s3_headers_depth']
if not s3_headers_depth:
defer.returnValue(False)
local_height = self.local_header_file_height()
remote_height = yield self.get_remote_height()
log.info("remote height: %i, local height: %s", remote_height, local_height)
if remote_height > (local_height + s3_headers_depth):
defer.returnValue(True)
defer.returnValue(False)
def _check_header_file_integrity(self):
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
if conf.settings['blockchain_name'] != "lbrycrd_main":
return
hashsum = sha256()
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
checksum_length_in_bytes = checksum_height * HEADER_SIZE
if self.local_header_file_size() < checksum_length_in_bytes:
return
with open(self.headers_file, "rb") as headers_file:
hashsum.update(headers_file.read(checksum_length_in_bytes))
current_checksum = hashsum.hexdigest()
if current_checksum != checksum:
msg = "Expected checksum {}, got {}".format(checksum, current_checksum)
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
log.warning("Deleting header file so it can be downloaded again.")
os.unlink(self.headers_file)
elif (self.local_header_file_size() % HEADER_SIZE) != 0:
log.warning("Header file is good up to checkpoint height, but incomplete. Truncating to checkpoint.")
with open(self.headers_file, "rb+") as headers_file:
headers_file.truncate(checksum_length_in_bytes)
@defer.inlineCallbacks
def start(self):
conf.settings.ensure_wallet_dir()
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
self._downloading_headers = yield self.should_download_headers_from_s3()
if self._downloading_headers:
try:
yield self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self._downloading_headers = False
def stop(self):
return defer.succeed(None)
class WalletComponent(Component):
component_name = WALLET_COMPONENT
depends_on = [DATABASE_COMPONENT, HEADERS_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.wallet_manager = None
@property
def component(self):
return self.wallet_manager
@defer.inlineCallbacks
def get_status(self):
if self.wallet_manager:
local_height = self.wallet_manager.network.get_local_height()
remote_height = self.wallet_manager.network.get_server_height()
best_hash = yield self.wallet_manager.get_best_blockhash()
defer.returnValue({
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
'best_blockhash': best_hash,
'is_encrypted': self.wallet_manager.wallet.use_encryption,
'is_locked': not self.wallet_manager.is_wallet_unlocked,
})
@defer.inlineCallbacks
def start(self):
conf.settings.ensure_wallet_dir()
log.info("Starting torba wallet")
storage = self.component_manager.get_component(DATABASE_COMPONENT)
lbryschema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
self.wallet_manager = LbryWalletManager.from_lbrynet_config(conf.settings, storage)
self.wallet_manager.old_db = storage
yield self.wallet_manager.start()
@defer.inlineCallbacks
def stop(self):
yield self.wallet_manager.stop()
self.wallet_manager = None
class BlobComponent(Component):
component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_manager = None
@property
def component(self):
return self.blob_manager
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
return self.blob_manager.setup()
def stop(self):
return self.blob_manager.stop()
@defer.inlineCallbacks
def get_status(self):
count = 0
if self.blob_manager:
count = yield self.blob_manager.storage.count_finished_blobs()
defer.returnValue({
'finished_blobs': count
})
class DHTComponent(Component):
component_name = DHT_COMPONENT
depends_on = [UPNP_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.dht_node = None
self.upnp_component = None
self.external_udp_port = None
self.external_peer_port = None
@property
def component(self):
return self.dht_node
def get_status(self):
return {
'node_id': binascii.hexlify(CS.get_node_id()),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
}
@defer.inlineCallbacks
def start(self):
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port"))
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port"))
node_id = CS.get_node_id()
if node_id is None:
node_id = generate_id()
self.dht_node = node.Node(
node_id=node_id,
udpPort=GCS('dht_node_port'),
externalUDPPort=self.external_udp_port,
externalIP=self.upnp_component.external_ip,
peerPort=self.external_peer_port
)
self.dht_node.start_listening()
yield self.dht_node._protocol._listening
d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
d.addCallback(lambda _: self.dht_node.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
log.info("Started the dht")
@defer.inlineCallbacks
def stop(self):
yield self.dht_node.stop()
class HashAnnouncerComponent(Component):
component_name = HASH_ANNOUNCER_COMPONENT
depends_on = [DHT_COMPONENT, DATABASE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.hash_announcer = None
@property
def component(self):
return self.hash_announcer
@defer.inlineCallbacks
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
yield self.hash_announcer.start()
@defer.inlineCallbacks
def stop(self):
yield self.hash_announcer.stop()
def get_status(self):
return {
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
}
class RateLimiterComponent(Component):
component_name = RATE_LIMITER_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.rate_limiter = RateLimiter()
@property
def component(self):
return self.rate_limiter
def start(self):
self.rate_limiter.start()
return defer.succeed(None)
def stop(self):
self.rate_limiter.stop()
return defer.succeed(None)
class StreamIdentifierComponent(Component):
component_name = STREAM_IDENTIFIER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.sd_identifier = StreamDescriptorIdentifier()
@property
def component(self):
return self.sd_identifier
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
add_lbry_file_to_sd_identifier(self.sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
dht_node.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
GCS('download_directory')
)
yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
def stop(self):
pass
class PaymentRateComponent(Component):
component_name = PAYMENT_RATE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.payment_rate_manager = OnlyFreePaymentsManager()
@property
def component(self):
return self.payment_rate_manager
def start(self):
return defer.succeed(None)
def stop(self):
return defer.succeed(None)
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.file_manager = None
@property
def component(self):
return self.file_manager
def get_status(self):
if not self.file_manager:
return
return {
'managed_files': len(self.file_manager.lbry_files)
}
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager')
self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier)
yield self.file_manager.setup()
log.info('Done setting up file manager')
@defer.inlineCallbacks
def stop(self):
yield self.file_manager.stop()
class PeerProtocolServerComponent(Component):
component_name = PEER_PROTOCOL_SERVER_COMPONENT
depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.lbry_server_port = None
@property
def component(self):
return self.lbry_server_port
@defer.inlineCallbacks
def start(self):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
upnp = self.component_manager.get_component(UPNP_COMPONENT)
peer_port = GCS('peer_port')
query_handlers = {
handler.get_primary_query_identifier(): handler for handler in [
BlobRequestHandlerFactory(
self.component_manager.get_component(BLOB_COMPONENT),
wallet,
self.component_manager.get_component(PAYMENT_RATE_COMPONENT),
self.component_manager.analytics_manager
),
wallet.get_wallet_info_query_handler_factory(),
]
}
server_factory = ServerProtocolFactory(
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
self.component_manager.get_component(DHT_COMPONENT).peer_manager
)
try:
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
upnp.upnp_redirects.get("TCP", peer_port))
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
" more details.", peer_port)
log.error("%s", traceback.format_exc())
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
@defer.inlineCallbacks
def stop(self):
if self.lbry_server_port is not None:
self.lbry_server_port, old_port = None, self.lbry_server_port
log.info('Stop listening on port %s', old_port.port)
yield old_port.stopListening()
class ReflectorComponent(Component):
component_name = REFLECTOR_COMPONENT
depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.reflector_server_port = GCS('reflector_port')
self.reflector_server = None
@property
def component(self):
return self.reflector_server
@defer.inlineCallbacks
def start(self):
log.info("Starting reflector server")
dht_node = self.component_manager.get_component(DHT_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
try:
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
log.info('Started reflector on port %s', self.reflector_server_port)
except error.CannotListenError as e:
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
raise ValueError("{} lbrynet may already be running on your computer.".format(e))
@defer.inlineCallbacks
def stop(self):
if self.reflector_server is not None:
log.info("Stopping reflector server")
self.reflector_server, p = None, self.reflector_server
yield p.stopListening
class UPnPComponent(Component):
component_name = UPNP_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = GCS('peer_port')
self._int_dht_node_port = GCS('dht_node_port')
self.use_upnp = GCS('use_upnp')
self.upnp = None
self.upnp_redirects = {}
self.external_ip = None
@property
def component(self):
return self
@defer.inlineCallbacks
def _setup_redirects(self):
upnp_redirects = yield DeferredDict({
"UDP": self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"),
"TCP": self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
})
self.upnp_redirects.update(upnp_redirects)
@defer.inlineCallbacks
def start(self):
if not self.use_upnp:
self.external_ip = CS.get_external_ip()
return
self.upnp = UPnP(self.component_manager.reactor, try_miniupnpc_fallback=True)
try:
found = yield self.upnp.discover()
except Exception as err:
log.warning("upnp discovery failed: %s", err)
found = False
if found and not self.upnp.miniupnpc_runner:
log.info("set up redirects using txupnp")
elif found and self.upnp.miniupnpc_runner:
log.warning("failed to set up redirect with txupnp, miniupnpc fallback was successful")
if found:
try:
self.external_ip = yield self.upnp.get_external_ip()
yield self._setup_redirects()
except Exception as err:
log.warning("error trying to set up upnp: %s", err)
self.external_ip = CS.get_external_ip()
else:
self.external_ip = CS.get_external_ip()
def stop(self):
return defer.DeferredList(
[self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()]
)
class ExchangeRateManagerComponent(Component):
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
def __init__(self, component_manager):
Component.__init__(self, component_manager)
self.exchange_rate_manager = ExchangeRateManager()
@property
def component(self):
return self.exchange_rate_manager
@defer.inlineCallbacks
def start(self):
yield self.exchange_rate_manager.start()
@defer.inlineCallbacks
def stop(self):
yield self.exchange_rate_manager.stop()
File diff suppressed because it is too large. Load diff
-232
View File
@@ -1,232 +0,0 @@
import sys
import code
import argparse
import asyncio
import logging.handlers
from twisted.internet import defer, reactor, threads
from aiohttp import client_exceptions
from lbrynet import analytics
from lbrynet import conf
from lbrynet.core import utils
from lbrynet.core import log_support
from lbrynet.daemon.auth.client import LBRYAPIClient
from lbrynet.daemon.Daemon import Daemon
log = logging.getLogger(__name__)
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
def color(msg, c="white"):
_colors = {
"normal": (0, 37),
"underlined": (2, 37),
"red": (1, 31),
"green": (1, 32),
"yellow": (1, 33),
"blue": (1, 33),
"magenta": (1, 34),
"cyan": (1, 35),
"white": (1, 36),
"grey": (1, 37)
}
i, j = _colors[c]
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
logo = """\
╓▄█▄ç
,▄█▓▓▀▀▀▓▓▓▌▄,
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
▄█▓▓█▀` ╙▀█▓▓▓▄▄
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
▄█▓▓█▀ ╙▀▓▓
╓▄▓▓▓▀` ▄█▓▓▓▀
▓▓█▀ ,▄▓▓▓▀╙
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
╙▀█▓▓█▓▓▓▀▀
╙▀▀`
"""
else:
def color(msg, c=None):
return msg
logo = """\
'.
++++.
+++,;+++,
:+++ :+++:
+++ ,+++;
'++; .+++'
`+++ `++++
+++. `++++
;+++ ++++
+++ +++
+++: '+
,+++ +++
+++` +++:
`+' ,+++
`+ + +++
`+ +++ '++' :'+++:
`+ ++++ `+++ ++++
`+ ++++ +++. :+++'
`+, ++++ ;+++ +++++
`+++, ++++ +++ +++; +
,+++, ++++ +++: .+++
,+++: '++++++ +++`
,+++: '++ '++'
,+++: `+++
.+++; +++,
.+++; ;+++
.+++; +++
`+++++:
`++
"""
welcometext = """\
For a list of available commands:
>>>help()
To see the documentation for a given command:
>>>help("resolve")
To exit:
>>>exit()
"""
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
welcome += "\n".join(["{:<60}".format(w) for w in welcometext.splitlines()])
welcome += "\n%s" % ("*" * 60)
welcome = color(welcome, "grey")
banner = color(logo, "green") + color(welcome, "grey")
def get_methods(daemon):
locs = {}
def wrapped(name, fn):
client = LBRYAPIClient.get_client()
_fn = getattr(client, name)
_fn.__doc__ = fn.__doc__
return {name: _fn}
for method_name, method in daemon.callable_methods.items():
locs.update(wrapped(method_name, method))
return locs
def run_terminal(callable_methods, started_daemon, quiet=False):
locs = {}
locs.update(callable_methods)
def help(method_name=None):
if not method_name:
print("Available api functions: ")
for name in callable_methods:
print("\t%s" % name)
return
if method_name not in callable_methods:
print("\"%s\" is not a recognized api function")
return
print(callable_methods[method_name].__doc__)
return
locs.update({'help': help})
if started_daemon:
def exit(status=None):
if not quiet:
print("Stopping lbrynet-daemon...")
callable_methods['daemon_stop']()
return sys.exit(status)
locs.update({'exit': exit})
else:
def exit(status=None):
try:
reactor.callLater(0, reactor.stop)
except Exception as err:
print("error stopping reactor: {}".format(err))
return sys.exit(status)
locs.update({'exit': exit})
code.interact(banner if not quiet else "", local=locs)
@defer.inlineCallbacks
def start_server_and_listen(use_auth, analytics_manager, quiet):
log_support.configure_console()
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
logging.getLogger("requests").setLevel(logging.CRITICAL)
analytics_manager.send_server_startup()
yield Daemon().start_listening()
def threaded_terminal(started_daemon, quiet):
callable_methods = get_methods(Daemon)
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
d.addErrback(lambda err: err.trap(SystemExit))
d.addErrback(log.exception)
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
if not utils.check_connection():
print("Not connected to internet, unable to start")
raise Exception("Not connected to internet, unable to start")
if not quiet:
print("Starting lbrynet-console...")
try:
await LBRYAPIClient.get_client().status()
d = defer.succeed(False)
if not quiet:
print("lbrynet-daemon is already running, connecting to it...")
except client_exceptions.ClientConnectorError:
if not use_existing_daemon:
if not quiet:
print("Starting lbrynet-daemon...")
analytics_manager = analytics.Manager.new_instance()
d = start_server_and_listen(useauth, analytics_manager, quiet)
else:
raise Exception("cannot connect to an existing daemon instance, "
"and set to not start a new one")
d.addCallback(threaded_terminal, quiet)
d.addErrback(log.exception)
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
parser.add_argument(
"--use_existing_daemon",
help="Start lbrynet-daemon if it isn't already running",
action="store_true",
default=False,
dest="use_existing_daemon"
)
parser.add_argument(
"--quiet", dest="quiet", action="store_true", default=False
)
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
)
args = parser.parse_args()
loop = asyncio.get_event_loop()
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
reactor.run()
if __name__ == "__main__":
main()
-74
View File
@@ -1,74 +0,0 @@
import os
import sys
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
# This needs to happen before anything else
if 'win' in sys.platform:
import certifi
os.environ['SSL_CERT_FILE'] = certifi.where()
from lbrynet.core import log_support
import argparse
import logging.handlers
from twisted.internet import reactor
from lbrynet import conf
from lbrynet.core import utils, system_info
from lbrynet.daemon.Daemon import Daemon
log = logging.getLogger(__name__)
def test_internet_connection():
return utils.check_connection()
def start(argv=None, conf_path=None):
if conf_path is not None:
conf.conf_file = conf_path
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
)
parser.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
parser.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
parser.add_argument(
'--version', action="store_true",
help='Show daemon version and quit'
)
args = parser.parse_args(argv)
if args.useauth:
conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
if args.version:
version = system_info.get_platform(get_ip=False)
version['installation_id'] = conf.settings.installation_id
print(utils.json_dumps_pretty(version))
return
lbrynet_log = conf.settings.get_log_filename()
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
log_support.configure_loggly_handler()
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
log.info("Starting lbrynet-daemon from command line")
if test_internet_connection():
daemon = Daemon()
daemon.start_listening()
reactor.run()
else:
log.info("Not connected to internet, unable to start")
-230
View File
@@ -1,230 +0,0 @@
import logging
import os
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbryschema.fee import Fee
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
from lbrynet.core.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet import conf
from torba.constants import COIN
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
DOWNLOAD_TIMEOUT_CODE = 'timeout'
DOWNLOAD_RUNNING_CODE = 'running'
DOWNLOAD_STOPPED_CODE = 'stopped'
STREAM_STAGES = [
(INITIALIZING_CODE, 'Initializing'),
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
]
log = logging.getLogger(__name__)
class GetStream:
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None):
self.timeout = timeout or conf.settings['download_timeout']
self.data_rate = data_rate or conf.settings['data_rate']
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
self.download_directory = conf.settings['download_directory']
self.timeout_counter = 0
self.code = None
self.sd_hash = None
self.blob_manager = blob_manager
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.wallet = wallet
self.exchange_rate_manager = exchange_rate_manager
self.payment_rate_manager = payment_rate_manager
self.sd_identifier = sd_identifier
self.storage = storage
self.downloader = None
self.checker = LoopingCall(self.check_status)
# fired when the download is complete
self.finished_deferred = None
# fired after the metadata and the first data blob have been downloaded
self.data_downloading_deferred = defer.Deferred(None)
@property
def download_path(self):
return os.path.join(self.download_directory, self.downloader.file_name)
def _check_status(self, status):
if status.num_completed > 0 and not self.data_downloading_deferred.called:
self.data_downloading_deferred.callback(True)
if self.data_downloading_deferred.called:
safe_stop_looping_call(self.checker)
else:
log.debug("Waiting for stream data (%i seconds)", self.timeout_counter)
def check_status(self):
"""
Check if we've got the first data blob in the stream yet
"""
self.timeout_counter += 1
if self.timeout_counter > self.timeout:
if not self.data_downloading_deferred.called:
if self.downloader:
err = DownloadDataTimeout(self.sd_hash)
else:
err = DownloadSDTimeout(self.sd_hash)
self.data_downloading_deferred.errback(err)
safe_stop_looping_call(self.checker)
elif self.downloader:
d = self.downloader.status()
d.addCallback(self._check_status)
else:
log.debug("Waiting for stream descriptor (%i seconds)", self.timeout_counter)
def convert_max_fee(self):
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
def set_status(self, status, name):
log.info("Download lbry://%s status changed to %s" % (name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
@defer.inlineCallbacks
def check_fee_and_convert(self, fee):
max_key_fee_amount = self.convert_max_fee()
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
fee.amount)
if converted_fee_amount > (yield self.wallet.default_account.get_balance()):
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
raise KeyFeeAboveMaxAllowed('Key fee %s above max allowed %s' % (converted_fee_amount,
max_key_fee_amount))
converted_fee = {
'currency': 'LBC',
'amount': converted_fee_amount,
'address': fee.address
}
return Fee(converted_fee)
def get_downloader_factory(self, factories):
for factory in factories:
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
return factory
raise Exception('No suitable factory was found in {}'.format(factories))
@defer.inlineCallbacks
def get_downloader(self, factory, stream_metadata, file_name=None):
# TODO: we should use stream_metadata.options.get_downloader_options
# instead of hard-coding the options to be [self.data_rate]
downloader = yield factory.make_downloader(
stream_metadata,
self.data_rate,
self.payment_rate_manager,
self.download_directory,
file_name=file_name
)
defer.returnValue(downloader)
def _pay_key_fee(self, address, fee_lbc, name):
log.info("Pay key fee %f --> %s", fee_lbc, address)
reserved_points = self.wallet.reserve_points(address, fee_lbc)
if reserved_points is None:
raise InsufficientFundsError('Unable to pay the key fee of %s for %s' % (fee_lbc, name))
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
@defer.inlineCallbacks
def pay_key_fee(self, fee, name):
if fee is not None:
yield self._pay_key_fee(fee.address.decode(), int(fee.amount * COIN), name)
else:
defer.returnValue(None)
@defer.inlineCallbacks
def finish(self, results, name):
self.set_status(DOWNLOAD_STOPPED_CODE, name)
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
self.download_path)
safe_stop_looping_call(self.checker)
status = yield self.downloader.status()
self._check_status(status)
defer.returnValue(self.download_path)
def fail(self, err):
safe_stop_looping_call(self.checker)
raise err
@defer.inlineCallbacks
def _initialize(self, stream_info):
# Set sd_hash and return key_fee from stream_info
self.sd_hash = stream_info.source_hash.decode()
key_fee = None
if stream_info.has_fee:
key_fee = yield self.check_fee_and_convert(stream_info.source_fee)
defer.returnValue(key_fee)
@defer.inlineCallbacks
def _create_downloader(self, sd_blob, file_name=None):
stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
factory = self.get_downloader_factory(stream_metadata.factories)
downloader = yield self.get_downloader(factory, stream_metadata, file_name)
defer.returnValue(downloader)
@defer.inlineCallbacks
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, conf.settings['download_mirrors']
)
defer.returnValue(sd_blob)
@defer.inlineCallbacks
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
yield self.pay_key_fee(key_fee, name)
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.finished_deferred = self.downloader.start()
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
@defer.inlineCallbacks
def start(self, stream_info, name, txid, nout, file_name=None):
"""
Start download
Returns:
(tuple) Tuple containing (downloader, finished_deferred)
downloader - instance of ManagedEncryptedFileDownloader
finished_deferred - deferred callbacked when download is finished
"""
self.set_status(INITIALIZING_CODE, name)
key_fee = yield self._initialize(stream_info)
safe_start_looping_call(self.checker, 1)
self.set_status(DOWNLOAD_METADATA_CODE, name)
try:
sd_blob = yield self._download_sd_blob()
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
yield self.data_downloading_deferred
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
safe_stop_looping_call(self.checker)
raise err
defer.returnValue((self.downloader, self.finished_deferred))
def cancel(self, reason=None):
if reason:
msg = "download stream cancelled: %s" % reason
else:
msg = "download stream cancelled"
if self.data_downloading_deferred and not self.data_downloading_deferred.called:
self.data_downloading_deferred.errback(DownloadCanceledError(msg))
-78
View File
@@ -1,78 +0,0 @@
import logging
import mimetypes
import os
from twisted.internet import defer
from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
log = logging.getLogger(__name__)
class Publisher:
def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate):
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.storage = storage
self.lbry_file_manager = lbry_file_manager
self.wallet = wallet
self.certificate = certificate
self.lbry_file = None
@defer.inlineCallbacks
def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
"""Create lbry file and make claim"""
log.info('Starting publish for %s', name)
if not os.path.isfile(file_path):
raise Exception("File {} not found".format(file_path))
if os.path.getsize(file_path) == 0:
raise Exception("Cannot publish empty file {}".format(file_path))
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as read_handle:
self.lbry_file = yield create_lbry_file(
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
read_handle
)
if 'source' not in claim_dict['stream']:
claim_dict['stream']['source'] = {}
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
tx = yield self.wallet.claim_name(
name, bid, claim_dict, self.certificate, holding_address
)
# check if we have a file already for this claim (if this is a publish update with a new stream)
old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(
tx.outputs[0].claim_id, self.lbry_file.stream_hash
)
if old_stream_hashes:
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
list(self.lbry_file_manager.lbry_files)):
yield self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False)
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
yield self.storage.save_content_claim(
self.lbry_file.stream_hash, tx.outputs[0].id
)
defer.returnValue(tx)
@defer.inlineCallbacks
def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
"""Make a claim without creating a lbry file"""
tx = yield self.wallet.claim_name(
name, bid, claim_dict, self.certificate, holding_address
)
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
yield self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
defer.returnValue(tx)
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
-1
View File
@@ -1 +0,0 @@
from . import Components # register Component classes
-34
View File
@@ -1,34 +0,0 @@
import logging
from zope.interface import implementer
from twisted.cred import portal, checkers, credentials, error as cred_error
from twisted.internet import defer
from twisted.web import resource
from .keyring import Keyring
log = logging.getLogger(__name__)
@implementer(portal.IRealm)
class HttpPasswordRealm:
def __init__(self, resource):
self.resource = resource
def requestAvatar(self, avatarId, mind, *interfaces):
log.debug("Processing request for %s", avatarId)
if resource.IResource in interfaces:
return (resource.IResource, self.resource, lambda: None)
raise NotImplementedError()
@implementer(checkers.ICredentialsChecker)
class PasswordChecker:
credentialInterfaces = (credentials.IUsernamePassword,)
def __init__(self, keyring: Keyring):
self.api_key = keyring.api_key
def requestAvatarId(self, creds):
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
return defer.succeed(creds.username)
log.warning('Incorrect username or password')
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
-133
View File
@@ -1,133 +0,0 @@
import json
import aiohttp
import logging
from urllib.parse import urlparse
from lbrynet import conf
from lbrynet.daemon.auth.keyring import Keyring, APIKey
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
class UnAuthAPIClient:
def __init__(self, host, port, session):
self.host = host
self.port = port
self.session = session
def __getattr__(self, method):
async def f(*args, **kwargs):
return await self.call(method, [args, kwargs])
return f
@classmethod
async def from_url(cls, url):
url_fragment = urlparse(url)
host = url_fragment.hostname
port = url_fragment.port
connector = aiohttp.TCPConnector(
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
)
session = aiohttp.ClientSession(connector=connector)
return cls(host, port, session)
async def call(self, method, params=None):
message = {'method': method, 'params': params}
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
return await resp.json()
class AuthAPIClient:
def __init__(self, key, session, cookies, url, login_url):
self.session = session
self.__api_key = key
self.__login_url = login_url
self.__id_count = 0
self.__url = url
self.__cookies = cookies
def __getattr__(self, name):
if name.startswith('__') and name.endswith('__'):
raise AttributeError(name)
def f(*args, **kwargs):
return self.call(name, [args, kwargs])
return f
async def call(self, method, params=None):
params = params or {}
self.__id_count += 1
pre_auth_post_data = {
'version': '2',
'method': method,
'params': params,
'id': self.__id_count
}
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
auth_msg = self.__api_key.get_hmac(to_auth).decode()
pre_auth_post_data.update({'hmac': auth_msg})
post_data = json.dumps(pre_auth_post_data)
headers = {
'Host': self.__url.hostname,
'User-Agent': USER_AGENT,
'Content-type': 'application/json'
}
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
if resp is None:
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
resp.raise_for_status()
next_secret = resp.headers.get(LBRY_SECRET, False)
if next_secret:
self.__api_key.secret = next_secret
return await resp.json()
@classmethod
async def get_client(cls, key_name=None):
api_key_name = key_name or "api"
keyring = Keyring.load_from_disk()
api_key = keyring.api_key
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
url = urlparse(login_url)
headers = {
'Host': url.hostname,
'User-Agent': USER_AGENT,
'Content-type': 'application/json'
}
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
session = aiohttp.ClientSession(connector=connector)
async with session.post(login_url, headers=headers) as r:
cookies = r.cookies
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
api_key = APIKey.create(seed=uid.encode())
return cls(api_key, session, cookies, url, login_url)
class LBRYAPIClient:
@staticmethod
def get_client(conf_path=None):
conf.conf_file = conf_path
if not conf.settings:
conf.initialize_settings()
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
-50
View File
@@ -1,50 +0,0 @@
import logging
from twisted.web import server, guard, resource
from twisted.cred import portal
from lbrynet import conf
from .auth import PasswordChecker, HttpPasswordRealm
from ..auth.keyring import Keyring
log = logging.getLogger(__name__)
class HTTPJSONRPCFactory(server.Site):
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
self.use_ssl = False
class HTTPSJSONRPCFactory(server.Site):
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
self.options = keyring.private_certificate.options()
self.use_ssl = True
class AuthJSONRPCResource(resource.Resource):
def __init__(self, protocol):
resource.Resource.__init__(self)
self.putChild(b"", protocol)
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
def getChild(self, name, request):
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
request.setHeader('expires', '0')
return self if name == '' else resource.Resource.getChild(self, name, request)
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
if use_authentication:
log.info("Using authenticated API")
checker = PasswordChecker(keyring)
realm = HttpPasswordRealm(self)
portal_to_realm = portal.Portal(realm, [checker, ])
root = guard.HTTPAuthSessionWrapper(
portal_to_realm, [guard.BasicCredentialFactory('Login to lbrynet api'), ]
)
else:
log.info("Using non-authenticated API")
root = self
return factory_class(root, keyring)
-130
View File
@@ -1,130 +0,0 @@
import os
import datetime
import hmac
import hashlib
import base58
from OpenSSL.crypto import FILETYPE_PEM
from ssl import create_default_context, SSLContext
from cryptography.hazmat.backends import default_backend
from cryptography import x509
from cryptography.hazmat.primitives import hashes
from cryptography.x509.name import NameOID, NameAttribute
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from twisted.internet import ssl
import logging
from lbrynet import conf
log = logging.getLogger(__name__)
def sha(x: bytes) -> str:
h = hashlib.sha256(x).digest()
return base58.b58encode(h).decode()
def generate_key(x: bytes = None) -> str:
if not x:
return sha(os.urandom(256))
else:
return sha(x)
class APIKey:
def __init__(self, secret: str, name: str):
self.secret = secret
self.name = name
@classmethod
def create(cls, seed=None, name=None):
secret = generate_key(seed)
return APIKey(secret, name)
def _raw_key(self) -> str:
return base58.b58decode(self.secret)
def get_hmac(self, message) -> str:
decoded_key = self._raw_key()
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
return base58.b58encode(signature.digest())
def compare_hmac(self, message, token) -> bool:
decoded_token = base58.b58decode(token)
target = base58.b58decode(self.get_hmac(message))
try:
if len(decoded_token) != len(target):
return False
return hmac.compare_digest(decoded_token, target)
except:
return False
class Keyring:
encoding = serialization.Encoding.PEM
filetype = FILETYPE_PEM
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
self.api_key: APIKey = api_key
self.public_certificate: str = public_certificate
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
@classmethod
def load_from_disk(cls):
api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token')
api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem')
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
return
with open(api_key_path, 'rb') as f:
api_key = APIKey(f.read().decode(), "api")
with open(api_ssl_cert_path, 'rb') as f:
public_cert = f.read().decode()
return cls(api_key, public_cert)
@classmethod
def generate_and_save(cls):
dns = conf.settings['api_host']
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=4096,
backend=default_backend()
)
subject = issuer = x509.Name([
NameAttribute(NameOID.COUNTRY_NAME, "US"),
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
])
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
certificate = x509.CertificateBuilder(
subject_name=subject,
issuer_name=issuer,
public_key=private_key.public_key(),
serial_number=x509.random_serial_number(),
not_valid_before=datetime.datetime.utcnow(),
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
).sign(private_key, hashes.SHA256(), default_backend())
public_certificate = certificate.public_bytes(cls.encoding).decode()
private_certificate = ssl.PrivateCertificate.load(
public_certificate,
ssl.KeyPair.load(
private_key.private_bytes(
encoding=cls.encoding,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()
).decode(),
cls.filetype
),
cls.filetype
)
auth_token = APIKey.create(seed=None, name="api")
with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f:
f.write(auth_token.secret.encode())
with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f:
f.write(public_certificate.encode())
return cls(auth_token, public_certificate, private_certificate)
-598
View File
@@ -1,598 +0,0 @@
import logging
from six.moves.urllib import parse as urlparse
import json
import inspect
import signal
from functools import wraps
from twisted.web import server
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.internet.error import ConnectionDone, ConnectionLost
from txjsonrpc import jsonrpclib
from traceback import format_exc
from lbrynet import conf, analytics
from lbrynet.core.Error import InvalidAuthenticationToken
from lbrynet.core import utils
from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
from lbrynet.core.looping_call_manager import LoopingCallManager
from lbrynet.daemon.ComponentManager import ComponentManager
from .keyring import APIKey, Keyring
from .undecorated import undecorated
from .factory import AuthJSONRPCResource
from lbrynet.daemon.json_response_encoder import JSONResponseEncoder
log = logging.getLogger(__name__)
EMPTY_PARAMS = [{}]
LBRY_SECRET = "LBRY_SECRET"
class JSONRPCError:
# http://www.jsonrpc.org/specification#error_object
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
MESSAGES = {
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
CODE_METHOD_NOT_FOUND: "Method Not Found",
CODE_INVALID_PARAMS: "Invalid Params",
CODE_INTERNAL_ERROR: "Internal Error",
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
}
HTTP_CODES = {
CODE_INVALID_REQUEST: 400,
CODE_PARSE_ERROR: 400,
CODE_INVALID_PARAMS: 400,
CODE_METHOD_NOT_FOUND: 404,
CODE_INTERNAL_ERROR: 500,
CODE_APPLICATION_ERROR: 500,
CODE_AUTHENTICATION_ERROR: 401,
}
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
assert isinstance(code, int), "'code' must be an int"
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
self.code = code
if message is None:
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
self.message = message
self.data = {} if data is None else data
self.traceback = []
if traceback is not None:
trace_lines = traceback.split("\n")
for i, t in enumerate(trace_lines):
if "--- <exception caught here> ---" in t:
if len(trace_lines) > i + 1:
self.traceback = [j for j in trace_lines[i+1:] if j]
break
def to_dict(self):
return {
'code': self.code,
'message': self.message,
'data': self.traceback
}
@classmethod
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
return cls(message, code=code, traceback=traceback)
class UnknownAPIMethodError(Exception):
pass
def trap(err, *to_trap):
err.trap(*to_trap)
def jsonrpc_dumps_pretty(obj, **kwargs):
try:
id_ = kwargs.pop("id")
except KeyError:
id_ = None
if isinstance(obj, JSONRPCError):
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
else:
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
class JSONRPCServerType(type):
def __new__(mcs, name, bases, newattrs):
klass = type.__new__(mcs, name, bases, newattrs)
klass.callable_methods = {}
klass.deprecated_methods = {}
for methodname in dir(klass):
if methodname.startswith("jsonrpc_"):
method = getattr(klass, methodname)
if not hasattr(method, '_deprecated'):
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
else:
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
return klass
class AuthorizedBase(metaclass=JSONRPCServerType):
@staticmethod
def deprecated(new_command=None):
def _deprecated_wrapper(f):
f.new_command = new_command
f._deprecated = True
return f
return _deprecated_wrapper
@staticmethod
def requires(*components, **conditions):
if conditions and ["conditions"] != list(conditions.keys()):
raise SyntaxError("invalid conditions argument")
condition_names = conditions.get("conditions", [])
def _wrap(fn):
@defer.inlineCallbacks
@wraps(fn)
def _inner(*args, **kwargs):
component_manager = args[0].component_manager
for condition_name in condition_names:
condition_result, err_msg = yield component_manager.evaluate_condition(condition_name)
if not condition_result:
raise ComponentStartConditionNotMet(err_msg)
if not component_manager.all_components_running(*components):
raise ComponentsNotStarted("the following required components have not yet started: "
"%s" % json.dumps(components))
result = yield fn(*args, **kwargs)
defer.returnValue(result)
return _inner
return _wrap
class AuthJSONRPCServer(AuthorizedBase):
"""
Authorized JSONRPC server used as the base class for the LBRY API
API methods are named with a leading "jsonrpc_"
Attributes:
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
callable_methods (dict): {<api method name>: <api method>}
Authentication:
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
service url will require a username and password.
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
client uses to generate the token for their next request.
"""
#implements(resource.IResource)
isLeaf = True
allowed_during_startup = []
component_attributes = {}
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
to_skip=None, looping_calls=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
self.component_manager = component_manager or ComponentManager(
analytics_manager=self.analytics_manager,
skip_components=to_skip or [],
reactor=reactor
)
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
self._use_authentication = use_authentication or conf.settings['use_auth_http']
self._use_https = use_https or conf.settings['use_https']
self.listening_port = None
self._component_setup_deferred = None
self.announced_startup = False
self.sessions = {}
self.server = None
self.keyring = Keyring.generate_and_save()
@defer.inlineCallbacks
def start_listening(self):
from twisted.internet import reactor, error as tx_error
try:
self.server = self.get_server_factory()
if self.server.use_ssl:
log.info("Using SSL")
self.listening_port = reactor.listenSSL(
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
)
else:
log.info("Not using SSL")
self.listening_port = reactor.listenTCP(
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
)
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
yield self.setup()
self.analytics_manager.send_server_startup_success()
except tx_error.CannotListenError:
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
reactor.fireSystemEvent("shutdown")
except defer.CancelledError:
log.info("shutting down before finished starting")
reactor.fireSystemEvent("shutdown")
except Exception as err:
self.analytics_manager.send_server_startup_error(str(err))
log.exception('Failed to start lbrynet-daemon')
reactor.fireSystemEvent("shutdown")
def setup(self):
from twisted.internet import reactor
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
if not self.analytics_manager.is_started:
self.analytics_manager.start()
self.analytics_manager.send_server_startup()
for lc_name, lc_time in self._looping_call_times.items():
self.looping_call_manager.start(lc_name, lc_time)
def update_attribute(setup_result, component):
setattr(self, self.component_attributes[component.component_name], component.component)
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
self._component_setup_deferred = self.component_manager.setup(**kwargs)
return self._component_setup_deferred
@staticmethod
def _already_shutting_down(sig_num, frame):
log.info("Already shutting down")
def _shutdown(self):
# ignore INT/TERM signals once shutdown has started
signal.signal(signal.SIGINT, self._already_shutting_down)
signal.signal(signal.SIGTERM, self._already_shutting_down)
if self.listening_port:
self.listening_port.stopListening()
self.looping_call_manager.shutdown()
if self.server is not None:
for session in list(self.server.sessions.values()):
session.expire()
if self.analytics_manager:
self.analytics_manager.shutdown()
try:
self._component_setup_deferred.cancel()
except (AttributeError, defer.CancelledError):
pass
if self.component_manager is not None:
d = self.component_manager.stop()
d.addErrback(log.fail(), 'Failure while shutting down')
else:
d = defer.succeed(None)
return d
def get_server_factory(self):
return AuthJSONRPCResource(self).getServerFactory(self.keyring, self._use_authentication, self._use_https)
def _set_headers(self, request, data, update_secret=False):
if conf.settings['allowed_origin']:
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
request.setHeader("Content-Type", "application/json")
request.setHeader("Accept", "application/json-rpc")
request.setHeader("Content-Length", str(len(data)))
if update_secret:
session_id = request.getSession().uid
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
@staticmethod
def _render_message(request, message: str):
request.write(message.encode())
request.finish()
def _render_error(self, failure, request, id_):
if isinstance(failure, JSONRPCError):
error = failure
elif isinstance(failure, Failure):
# maybe failure is JSONRPCError wrapped in a twisted Failure
error = failure.check(JSONRPCError)
if error is None:
# maybe its a twisted Failure with another type of error
if hasattr(failure.type, "code"):
error_code = failure.type.code
else:
error_code = JSONRPCError.CODE_APPLICATION_ERROR
error = JSONRPCError.create_from_exception(
failure.getErrorMessage() or failure.type.__name__,
code=error_code,
traceback=failure.getTraceback()
)
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
log.warning("error processing api request: %s\ntraceback: %s", error.message,
"\n".join(error.traceback))
else:
# last resort, just cast it as a string
error = JSONRPCError(str(failure))
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
self._set_headers(request, response_content)
request.setResponseCode(200)
self._render_message(request, response_content)
@staticmethod
def _handle_dropped_request(result, d, function_name):
if not d.called:
log.warning("Cancelling dropped api request %s", function_name)
d.cancel()
def render(self, request):
try:
return self._render(request)
except BaseException as e:
log.error(e)
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
self._render_error(error, request, None)
return server.NOT_DONE_YET
def _render(self, request):
time_in = utils.now()
# if not self._check_headers(request):
# self._render_error(Failure(InvalidHeaderError()), request, None)
# return server.NOT_DONE_YET
session = request.getSession()
session_id = session.uid
finished_deferred = request.notifyFinish()
if self._use_authentication:
# if this is a new session, send a new secret and set the expiration
# otherwise, session.touch()
if self._initialize_session(session_id):
def expire_session():
self._unregister_user_session(session_id)
session.notifyOnExpire(expire_session)
message = "OK"
request.setResponseCode(200)
self._set_headers(request, message, True)
self._render_message(request, message)
return server.NOT_DONE_YET
else:
session.touch()
request.content.seek(0, 0)
content = request.content.read().decode()
try:
parsed = jsonrpclib.loads(content)
except json.JSONDecodeError:
log.warning("Unable to decode request json")
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
return server.NOT_DONE_YET
request_id = None
try:
function_name = parsed.get('method')
args = parsed.get('params', {})
request_id = parsed.get('id', None)
token = parsed.pop('hmac', None)
except AttributeError as err:
log.warning(err)
self._render_error(
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
)
return server.NOT_DONE_YET
reply_with_next_secret = False
if self._use_authentication:
try:
self._verify_token(session_id, parsed, token)
except InvalidAuthenticationToken as err:
log.warning("API validation failed")
self._render_error(
JSONRPCError.create_from_exception(
str(err),
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
traceback=format_exc()
),
request, request_id
)
return server.NOT_DONE_YET
request.addCookie("TWISTED_SESSION", session_id)
self._update_session_secret(session_id)
reply_with_next_secret = True
try:
fn = self._get_jsonrpc_method(function_name)
except UnknownAPIMethodError as err:
log.warning('Failed to get function %s: %s', function_name, err)
self._render_error(
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
request, request_id
)
return server.NOT_DONE_YET
if args in (EMPTY_PARAMS, []):
_args, _kwargs = (), {}
elif isinstance(args, dict):
_args, _kwargs = (), args
elif len(args) == 1 and isinstance(args[0], dict):
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
# TODO: also delete EMPTY_PARAMS then
_args, _kwargs = (), args[0]
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
_args, _kwargs = args
else:
raise ValueError('invalid args format')
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
if params_error is not None:
params_error_message = '{} for {} command: {}'.format(
params_error, function_name, ', '.join(erroneous_params)
)
log.warning(params_error_message)
self._render_error(
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
request, request_id
)
return server.NOT_DONE_YET
d = defer.maybeDeferred(fn, self, *_args, **_kwargs)
# finished_deferred will callback when the request is finished
# and errback if something went wrong. If the errback is
# called, cancel the deferred stack. This is to prevent
# request.finish() from being called on a closed request.
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
d.addErrback(self._render_error, request, request_id)
d.addBoth(lambda _: log.debug("%s took %f",
function_name,
(utils.now() - time_in).total_seconds()))
return server.NOT_DONE_YET
def _register_user_session(self, session_id):
"""
Add or update a HMAC secret for a session
@param session_id:
@return: secret
"""
log.info("Started new api session")
token = APIKey.create(seed=session_id)
self.sessions.update({session_id: token})
def _unregister_user_session(self, session_id):
log.info("Unregister API session")
del self.sessions[session_id]
def _check_headers(self, request):
return (
self._check_header_source(request, 'Origin') and
self._check_header_source(request, 'Referer')
)
def _check_header_source(self, request, header):
"""Check if the source of the request is allowed based on the header value."""
source = request.getHeader(header)
if not self._check_source_of_request(source):
log.warning("Attempted api call from invalid %s: %s", header, source)
return False
return True
def _check_source_of_request(self, source):
if source is None:
return True
if conf.settings['api_host'] == '0.0.0.0':
return True
server, port = self.get_server_port(source)
return self._check_server_port(server, port)
def _check_server_port(self, server, port):
api = (conf.settings['api_host'], conf.settings['api_port'])
return (server, port) == api or self._is_from_allowed_origin(server, port)
def _is_from_allowed_origin(self, server, port):
allowed_origin = conf.settings['allowed_origin']
if not allowed_origin:
return False
if allowed_origin == '*':
return True
allowed_server, allowed_port = self.get_server_port(allowed_origin)
return (allowed_server, allowed_port) == (server, port)
def get_server_port(self, origin):
parsed = urlparse.urlparse(origin)
server_port = parsed.netloc.split(':')
assert len(server_port) <= 2
if len(server_port) == 2:
return server_port[0], int(server_port[1])
else:
return server_port[0], 80
def _verify_method_is_callable(self, function_path):
if function_path not in self.callable_methods:
raise UnknownAPIMethodError(function_path)
def _get_jsonrpc_method(self, function_path):
if function_path in self.deprecated_methods:
new_command = self.deprecated_methods[function_path].new_command
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
function_path, new_command)
function_path = new_command
self._verify_method_is_callable(function_path)
return self.callable_methods.get(function_path)
@staticmethod
def _check_params(function, args_tup, args_dict):
argspec = inspect.getfullargspec(undecorated(function))
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
duplicate_params = [
duplicate_param
for duplicate_param in argspec.args[1:len(args_tup) + 1]
if duplicate_param in args_dict
]
if duplicate_params:
return 'Duplicate parameters', duplicate_params
missing_required_params = [
required_param
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
if required_param not in args_dict
]
if len(missing_required_params):
return 'Missing required parameters', missing_required_params
extraneous_params = [] if argspec.varkw is not None else [
extra_param
for extra_param in args_dict
if extra_param not in argspec.args[1:]
]
if len(extraneous_params):
return 'Extraneous parameters', extraneous_params
return None, None
def _initialize_session(self, session_id):
if not self.sessions.get(session_id):
self._register_user_session(session_id)
return True
return False
def _verify_token(self, session_id, message, token):
if token is None:
raise InvalidAuthenticationToken('Authentication token not found')
to_auth = json.dumps(message, sort_keys=True)
api_key = self.sessions.get(session_id)
if not api_key.compare_hmac(to_auth, token):
raise InvalidAuthenticationToken('Invalid authentication token')
def _update_session_secret(self, session_id):
self.sessions.update({session_id: APIKey.create(name=session_id)})
def _callback_render(self, result, request, id_, auth_required=False):
try:
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
request.setResponseCode(200)
self._set_headers(request, message, auth_required)
self._render_message(request, message)
except Exception as err:
log.exception("Failed to render API response: %s", result)
self._render_error(err, request, id_)
@staticmethod
def _render_response(result):
return defer.succeed(result)
-57
View File
@@ -1,57 +0,0 @@
from decimal import Decimal
from binascii import hexlify
from datetime import datetime
from json import JSONEncoder
from lbrynet.wallet.transaction import Transaction, Output
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
return float(obj)
if isinstance(obj, bytes):
return obj.decode()
return super().default(obj)
def encode_transaction(self, tx):
return {
'txid': tx.id,
'height': tx.height,
'inputs': [self.encode_input(txo) for txo in tx.inputs],
'outputs': [self.encode_output(txo) for txo in tx.outputs],
'total_input': tx.input_sum,
'total_output': tx.input_sum - tx.fee,
'total_fee': tx.fee,
'hex': hexlify(tx.raw).decode(),
}
def encode_output(self, txo):
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': txo.amount,
'address': txo.get_address(self.ledger),
'is_claim': txo.script.is_claim_name,
'is_support': txo.script.is_support_claim,
'is_update': txo.script.is_update_claim,
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
return output
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
Loaded 100 of 351 files, more files were not shown because too many files have changed in this diff. Show more