Compare commits

...
Author SHA1 Message Date
Victor Shyba 3a0affe8ba show status while downloading headers 2019-06-11 02:25:49 -03:00
Jack Robison 06a9770504 Merge pull request #2216 from lbryio/fix-connection-status-startup-error
fix connections status raising an error during startup
2019-06-07 13:55:54 -04:00
Jack Robison 0da43d3e05 fix connections status raising an error during startup 2019-06-07 12:14:05 -04:00
Lex Berezhny 077dbb3138 better handle claims which fail to decode 2019-06-07 08:42:17 -04:00
Jack Robison 3b8fd65fa9 Merge pull request #2212 from lbryio/add-torba-version-to-version-rpc
update `version` docstring, add `torba_version`
2019-06-05 20:59:58 -04:00
Jack Robison d0ddaa5518 update version docstring, add torba_version 2019-06-05 20:33:21 -04:00
Jack Robison a60ab97552 Merge pull request #2211 from lbryio/pin-aioupnp
pin aioupnp requirement to 0.0.12
2019-06-05 19:41:45 -04:00
Jack Robison 078d09976c pin aioupnp requirement 2019-06-05 18:50:36 -04:00
Jack Robison 3770558694 Merge pull request #2210 from lbryio/fix-wine-build
fix torba version on travis
2019-06-05 18:46:24 -04:00
Jack Robison 4409b8c4f8 fix torba version on travis 2019-06-05 18:19:56 -04:00
Lex Berezhny f5b7aaac18 v0.38.0rc6 2019-06-04 22:32:33 -04:00
Lex Berezhny 6dbb6b9079 fix claim expiration calculation 2019-06-04 22:28:35 -04:00
Lex Berezhny 16b774a69e add normalized to claims_in_channel index to make update claim faster 2019-06-04 22:28:35 -04:00
Lex Berezhny 72ea19532e added index specifically for claim_in_channel computation 2019-06-04 22:28:35 -04:00
Lex Berezhny ad24946af3 add improved profiling to validate_channel_signatures 2019-06-04 22:28:35 -04:00
Lex Berezhny b676907fd7 added index for claim expiration_height 2019-06-04 22:28:35 -04:00
Lex Berezhny 6965971613 delete claims and re-calculate winner when claim is expired 2019-06-04 22:28:35 -04:00
Lex Berezhny 6651111522 pylint 2019-06-04 18:44:21 -04:00
Lex Berezhny d8630a02d9 Strings setting works like Server from CLI 2019-06-04 18:44:21 -04:00
Lex Berezhny 69345b4b0e integrationt tests fixed 2019-06-04 16:04:46 -04:00
Lex Berezhny f23aea9951 better handling for claims in abandoned channels 2019-06-04 16:04:46 -04:00
Lex Berezhny f96c46e84a claim search fixes for filtering with valid_channel_signatures 2019-06-04 16:04:46 -04:00
Alex Grintsvayg bae2939943 change a few instances of lbry.io to lbry.com. fixes #2135 2019-06-04 10:23:04 -04:00
Alex Grintsvayg 7063138e3c small improvements from our cli investigation 2019-06-04 09:39:33 -04:00
Lex Berezhny 798ba9acf8 v0.38.0rc5 2019-06-04 01:20:32 -04:00
Jack Robison c8dd1987e6 add connection status 2019-06-04 01:20:05 -04:00
Lex Berezhny 8d2c9e5785 claim_type/stream_type/media_type claim search integration test 2019-06-04 00:51:25 -04:00
Lex Berezhny 9dbf47916b add public_key_id to channel integration tests 2019-06-03 21:56:17 -04:00
Lex Berezhny 565ef5e454 docopt doc string fix 2019-06-03 21:56:17 -04:00
Lex Berezhny 9f1f2b1781 claim search with --public_key_id 2019-06-03 21:56:17 -04:00
Lex Berezhny 76df239d1a start command accepts server lists consistent with other commands 2019-06-03 21:22:24 -04:00
Thomas Zarebczan 86ee6a78f5 add MKV to mime types 2019-06-03 19:51:48 -04:00
Lex Berezhny 4fb6f2bb27 review fixes 2019-06-03 06:51:09 -04:00
Oleg Silkin ec039825e8 Fixes non-subscribtable object error 2019-06-03 06:51:09 -04:00
Oleg Silkin a10f09d169 Updates signature return field 2019-06-03 06:51:09 -04:00
Oleg Silkin 76409600e1 Fixes stuff up 2019-06-03 06:51:09 -04:00
Oleg Silkin d06a0f876a Patches comment server interface in the SDK 2019-06-03 06:51:09 -04:00
Oleg Silkin 8e857de62a Fixes non-subscribtable object error 2019-06-03 06:51:09 -04:00
Oleg Silkin 3d37f7c735 Updates signature return field 2019-06-03 06:51:09 -04:00
Oleg Silkin 552b94bc65 Fixes stuff up 2019-06-03 06:51:09 -04:00
Oleg Silkin 17164601a1 Patches comment server interface in the SDK 2019-06-03 06:51:09 -04:00
Lex Berezhny 26dd6d2ac7 claim search test fix 2019-05-30 21:57:49 -04:00
Lex Berezhny 5d5e36a2c2 short_url and canonical_url returned from wallet server use original claim name instead of normalized 2019-05-30 21:32:35 -04:00
Lex Berezhny 77afbcded6 wallet server, skip table join for count(*) query 2019-05-30 21:07:07 -04:00
Lex Berezhny ae11d72f66 added claim.effective_amount index on wallet server 2019-05-30 16:13:46 -04:00
Lex Berezhny d2773f9713 v0.38.0rc4 2019-05-29 22:48:28 -04:00
Lex Berezhny a7fba569cd update tests to use valid_channel_signatures argument 2019-05-29 22:47:33 -04:00
Lex Berezhny 0559c64038 fixed claim search bug related to signed claims 2019-05-29 22:47:33 -04:00
Lex Berezhny 5ab2754bc6 shutdown daemon during tests teardown 2019-05-29 19:27:05 -04:00
Lex Berezhny 6bfb8384e7 channel import/export working tests 2019-05-29 19:27:05 -04:00
Lex Berezhny 0156aa8b00 channel import/export wip 2019-05-29 19:27:05 -04:00
Lex Berezhny ad411cc157 trying to fix rebase 2019-05-29 19:27:05 -04:00
hackrush 6d462ad6d8 channel export import 2019-05-29 19:27:05 -04:00
Lex Berezhny 1b35cef77a fix wallet.server.test_sqldb test 2019-05-28 20:51:22 -04:00
Lex Berezhny a357208a77 cleaned up with passing tests 2019-05-28 20:51:22 -04:00
hackrush 7e1f8a56d5 Changed txo.private_key to be an object 2019-05-28 20:51:22 -04:00
hackrush 37665f2df7 Review Fixes 2019-05-28 20:51:22 -04:00
hackrush 8f3b596385 Add consolidated field to migration result 2019-05-28 20:51:22 -04:00
hackrush e45c6fe42c Only store pubkey-private key pairs 2019-05-28 20:51:22 -04:00
hackrush 56edf62cb9 Temporary fix for github.com/PyCQA/pylint/issues/2820 2019-05-28 20:51:22 -04:00
hackrush d367ff6ac1 Update tests and code 2019-05-28 20:51:22 -04:00
hackrush 7a78d49e13 Store public-private mappings in wallet 2019-05-28 20:51:22 -04:00
Lex Berezhny 4dae1ae05f regenerate docs 2019-05-27 23:18:26 -04:00
Lex Berezhny 9004f240e6 bug fixes 2019-05-27 23:18:26 -04:00
Lex Berezhny 1848462ad9 v0.38.0rc3 2019-05-26 14:04:45 -04:00
Lex Berezhny 786efa06dd fixed resolve to work with partial claim_ids 2019-05-26 14:01:06 -04:00
Lex Berezhny b6811d55cf no longer log skipped claim insertions on wallet server, happens too often 2019-05-26 14:01:06 -04:00
Lex Berezhny 42dc0eccd6 wallet server channel key change error 2019-05-26 14:01:06 -04:00
Lex Berezhny 29bd936181 wallet server tracks claim signature 2019-05-26 14:01:06 -04:00
Lex Berezhny 9d97b90ff4 pylint 2019-05-26 14:01:06 -04:00
Lex Berezhny 370b34f860 canonical_url takes into account whether channel signature is valid 2019-05-26 14:01:06 -04:00
Lex Berezhny affa46e0f6 initial import of canonical url via the new sql wallet server 2019-05-26 14:01:06 -04:00
Jack Robison 6b93705599 Merge pull request #2181 from lbryio/improve-set-build
improve set_build.py, pin torba version
2019-05-24 19:00:38 -04:00
Jack Robison 925e39c5a8 improve set_build.py, pin torba version
- adds BUILD_COMMIT to lbrynet.build_type
2019-05-24 18:02:55 -04:00
Jack Robison 1efa2c9406 Merge pull request #2180 from lbryio/fix-completed-false-negative
fix false negative with file `completed` field
2019-05-24 17:18:04 -04:00
Jack Robison 4f8bcfcd18 fix false negative with file completed field 2019-05-24 16:47:25 -04:00
Jack Robison 3ed72a215e Merge pull request #2179 from lbryio/fix-range-request-start-offset
start returning bytes for a range request at the requested starting position
2019-05-24 16:46:21 -04:00
Jack Robison 85d94d4ca3 start returning range request bytes at the requested offset 2019-05-24 15:32:40 -04:00
Jack Robison b922ea8e7e Merge pull request #2178 from lbryio/lru-cached-read-blob
add a lru cache for decrypted blobs to minimize redownloading them, add `blob_lru_cache_size` to the config to set the cache size
2019-05-24 13:56:59 -04:00
Jack Robison cb6ddaf276 add blob_lru_cache_size config setting, defaults to 32 2019-05-24 12:02:58 -04:00
Lex Berezhny 1c2029cddf fix startup error when single key account present in wallet 2019-05-22 18:31:49 -04:00
Alex Grintsvayg d8d136f1d8 uniform spacing on docstring 2019-05-22 16:51:14 -04:00
Victor Shyba fedc0e5bc3 add checktrie.py script 2019-05-22 12:20:33 -04:00
Alex Grintsvayg 4fbccb9336 uniform spacing on docstring 2019-05-22 09:23:31 -04:00
Lex Berezhny ef5dd5593f v0.38.0rc2 2019-05-20 21:39:16 -04:00
Lex Berezhny baed974be0 several minor wallet server fixes 2019-05-20 21:35:20 -04:00
Lex Berezhny b2b3509b61 regenerate protobufs 2019-05-20 15:19:14 -04:00
Lex Berezhny 51720d9b73 add MAX() to claimtrie GROUP BY algo to make results consistent across SQLite versions 2019-05-20 15:19:14 -04:00
Lex Berezhny 6abb569ec8 v0.38.0rc1 2019-05-19 23:03:25 -04:00
Lex Berezhny c2fbbadc69 start trending calculation as wallet server sync nears best height 2019-05-19 21:16:19 -04:00
Lex Berezhny b8897223ec added zscore trending in wallet server 2019-05-19 21:16:19 -04:00
hackrush c2c184b4ef Updated to include the environment variables needed in mac 2019-05-18 21:44:49 -04:00
Lex Berezhny 07dbdef3fe dont run test_resolve_lru_cache_doesnt_persist_errors 2019-05-18 21:43:46 -04:00
Lex Berezhny b841fb2f0c fix docopt parser issue 2019-05-18 21:43:46 -04:00
Lex Berezhny 2b75982446 disable trending until its fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 021ca3efd2 updated doc strings and fixed wallet server claimtrie implementation 2019-05-18 21:43:46 -04:00
Lex Berezhny e56b72cf04 pylint and use torba master 2019-05-18 21:43:46 -04:00
Lex Berezhny 342f485ddf good stuff 2019-05-18 21:43:46 -04:00
Lex Berezhny 8fe16fbfc9 performance 2019-05-18 21:43:46 -04:00
Lex Berezhny 1e3035029e timer fix 2019-05-18 21:43:46 -04:00
Lex Berezhny 5f4c02f836 wallet server sync performance improvements 2019-05-18 21:43:46 -04:00
Lex Berezhny 03a1185e0f pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny 0e1705d19e instrumented most wallet server claim related functions with timer 2019-05-18 21:43:46 -04:00
Lex Berezhny 82c739daa6 abstract _apply_constraints_for_array_attributes for wallet server 2019-05-18 21:43:46 -04:00
Lex Berezhny 4702bea9f0 claim_search --not_tags feature + integration tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 75211b7e84 updated claim_search doc string 2019-05-18 21:43:46 -04:00
Lex Berezhny 9f9273bd82 working claim_search with tags 2019-05-18 21:43:46 -04:00
Lex Berezhny d2608bdf43 fixed wallet/server/test_sqldb.py tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 091bef1ead sequence/amount_order resolve 2019-05-18 21:43:46 -04:00
Lex Berezhny 998ee9feba resolve integration tests fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 587f0b2ea8 integration tests for jsonrpc_transaction_show fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 048aa07e80 test_file_commands integration tests fixed 2019-05-18 21:43:46 -04:00
Lex Berezhny 7ca01511f4 fixing integration tests 2019-05-18 21:43:46 -04:00
Lex Berezhny 8dee83aac0 disable cache for integration tests while using custom torba branch 2019-05-18 21:43:46 -04:00
Lex Berezhny 879ae5cc72 .travis.yml 2019-05-18 21:43:46 -04:00
Lex Berezhny 74a9e17178 .travis.yml 2019-05-18 21:43:46 -04:00
Lex Berezhny 93d2173fad chris45 integration test passing 2019-05-18 21:43:46 -04:00
Lex Berezhny a459ee34be test_url unit test fix 2019-05-18 21:43:46 -04:00
Lex Berezhny 2e6297f834 pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny 9accbfcf8b move exception handling into is_signed_by 2019-05-18 21:43:46 -04:00
Lex Berezhny 6946860521 added result schema types 2019-05-18 21:43:46 -04:00
Lex Berezhny 9b8be8682c massive progress 2019-05-18 21:43:46 -04:00
Lex Berezhny 03455310ae rewrote URL parser 2019-05-18 21:43:46 -04:00
Lex Berezhny e5edaed677 removed test code from wallet server daemon.py 2019-05-18 21:43:46 -04:00
Lex Berezhny 5e97215eaa integration tests should clone sqltrie torba branch 2019-05-18 21:43:46 -04:00
Lex Berezhny 266f94bc95 pylint 2019-05-18 21:43:46 -04:00
Lex Berezhny d26caded53 use sqltrie branch of torba 2019-05-18 21:43:46 -04:00
Lex Berezhny 2260608bb6 working sql claimtrie 2019-05-18 21:43:46 -04:00
Lex Berezhny bb1f42057e v0.37.1 2019-05-17 23:01:47 -04:00
Lex Berezhny 695f6658a0 v0.37.1rc2 2019-05-17 21:43:37 -04:00
Jack Robison eca677a720 Merge pull request #2148 from lbryio/lru-cache-concurrent
Fix resolve internals caching and persisting errors
2019-05-17 19:26:22 -04:00
Jack Robison 51f301785f fix test, add lru_cache_concurrent decorator 2019-05-17 18:57:07 -04:00
Jack Robison 41a6e288aa add test_resolve_lru_cache_doesnt_persist_errors 2019-05-17 18:56:28 -04:00
Jack Robison 97d8d0ef4c Merge pull request #2133 from lbryio/more-wallets
Add missing wallet server
2019-05-17 15:50:10 -04:00
Niko Storni 5baeb5f538 fix examples 2019-05-17 15:19:11 -04:00
Niko Storni cc49c8407c add missing wallet server
switch to .com domain
2019-05-17 15:19:11 -04:00
Jack Robison 3f37963f7f Merge pull request #2139 from lbryio/android-platform-name
fix system name on android
2019-05-17 14:39:20 -04:00
Jack Robison 12eb83f773 fix system name on android 2019-05-17 14:00:25 -04:00
Jack Robison ca1d135ba4 Merge pull request #2138 from lbryio/fix-file-save-doc
fix save_file doc
2019-05-17 13:44:22 -04:00
Jack Robison 107e57e3ff fix docs 2019-05-17 13:11:04 -04:00
Jack Robison 928689a10d Merge pull request #2130 from lbryio/fix-datastore
Fix updating refreshed announcements in the datastore
2019-05-17 12:58:11 -04:00
Jack Robison 6ecc22f2c7 fix refreshing peers in the datastore 2019-05-17 12:22:47 -04:00
Lex Berezhny 079c81f298 v0.37.1rc1 2019-05-17 08:59:10 -04:00
Victor Shyba 8afe178d62 remove supports from resolve 2019-05-17 08:55:39 -04:00
Jack Robison 8777dcee7c v0.37.0 2019-05-13 23:39:22 -04:00
Jack Robison 1cca4a91dd Merge pull request #2128 from lbryio/add-streaming-server-settings
add `streaming_server` and `streaming_get` settings to configure the interface/port of the media server and if it supports the `/get` endpoint
2019-05-13 22:51:53 -04:00
Jack Robison 41abfbdd9b add streaming_server and streaming_get settings
-split rpc runner from range request runner
2019-05-13 22:27:48 -04:00
Jack Robison 46ee65c8f2 Merge pull request #2127 from lbryio/add-streaming-url
add `streaming_url` field to `File` api response type
2019-05-13 21:05:52 -04:00
Jack Robison fd19daad9e add streaming_url to file api response type 2019-05-13 19:05:26 -04:00
Jack Robison 26d183cbab Merge pull request #2125 from lbryio/dht_wip
Remove search junctions, make search retry, some refactoring
2019-05-13 18:02:56 -04:00
Victor Shyba 71a4be02dd use loop to create task 2019-05-13 15:57:03 -03:00
Victor Shyba 124fdb7d31 fixes from review 2019-05-13 14:57:58 -03:00
Victor Shyba 2439743804 probe peer instead of scheduling ping for removal 2019-05-13 14:51:03 -03:00
Victor Shyba 6f79875135 fix test 2019-05-13 14:27:49 -03:00
Victor Shyba b91d2190f4 disable infinite peer search, cleanup logging, tune scores to slow connections 2019-05-13 14:24:13 -03:00
Victor Shyba d024433d1b simplify accumulate task 2019-05-13 13:34:49 -03:00
Victor Shyba 7c4eb416d6 dont clearbanned if there are working peers, dont remove node immediately during search 2019-05-13 13:14:08 -03:00
Victor Shyba fb457c820a reorganizing iterative find code 2019-05-13 04:34:39 -03:00
Victor Shyba 18af2dcd4e remove peer immediately instead of probing when its bad during search 2019-05-13 02:52:24 -03:00
Victor Shyba 0075dcc2c0 iterative find probes bad peers + test case for edge scenario Tom found 2019-05-13 02:40:04 -03:00
Victor Shyba 6e327d729b fix test 2019-05-12 04:28:58 -03:00
Victor Shyba 20022d49c1 less verbose dht tests 2019-05-12 04:13:58 -03:00
Victor Shyba 32d1563885 fix handling of invalid token 2019-05-12 04:12:12 -03:00
Victor Shyba f5cef14d8d test get token on announce 2019-05-12 03:55:07 -03:00
Victor Shyba bac7d99b8a add ability to re-join network on disconnect + tests 2019-05-12 03:39:11 -03:00
Victor Shyba 6f06026511 announcer goes idle if no peers in routing table 2019-05-12 01:10:05 -03:00
Victor Shyba f02df86709 fix announce loop when there are no peers to announce 2019-05-12 00:42:19 -03:00
Victor Shyba b7d76fd09f add integration tests, fix bug that made refresh not exclude bad peers 2019-05-11 23:30:05 -03:00
Victor Shyba e675f1387c remove peer junction and some refactoring 2019-05-11 04:58:56 -03:00
Victor Shyba 0da2827c78 fix dht deadlock 2019-05-10 19:54:59 -03:00
Jack Robison 309cfc840c v0.37.0rc7 2019-05-10 15:34:38 -04:00
Jack Robison 68ee31b62a Merge pull request #2123 from lbryio/stop-timed-out-downloads
Stop timed out downloads
2019-05-10 15:29:33 -04:00
Jack Robison ad2da24d64 improve test 2019-05-10 14:57:04 -04:00
Jack Robison 648dbcc36b add timeout to blob downloads after the head blob, stop timed out streams 2019-05-10 14:50:01 -04:00
Victor Shyba e59e96190e fix peer search using inexistent variable, fix test to cover it 2019-05-10 13:49:35 -04:00
Jack Robison 7d2f592bb5 Merge pull request #2107 from lbryio/fix-update-file-we-dont-have
fix stream_update failing on metadata only updates for files we don't have
2019-05-10 12:28:53 -04:00
Jack Robison 1a182bc9bd bytes/str 2019-05-10 11:50:20 -04:00
Jack Robison 324cbdcdb2 only save the content claim if stream_hash isn't None
-stream_hash being none means we don't have the file in the database
2019-05-10 11:50:20 -04:00
Lex Berezhny 6c0c0ee322 added logging to publish command 2019-05-10 11:43:34 -04:00
Jack Robison 1e54271d8a Merge pull request #2105 from lbryio/fix-store-content-fee
fix store content fee
2019-05-09 18:47:30 -04:00
Jack Robison 54b9ed6cb2 fix store content fee 2019-05-09 18:14:54 -04:00
Lex Berezhny 558a02d187 correctly format lat/long and fee amounts in JSON RPC response 2019-05-09 16:19:49 -04:00
Jack Robison 3567b331c0 v0.37.0rc6 2019-05-09 11:52:34 -04:00
Jack Robison 9e4cf3c995 Merge pull request #2048 from lbryio/timeout_settings
Adjust default max_connections_per_download and peer_connect_timeout settings
2019-05-09 11:14:59 -04:00
Thomas Zarebczan db1d547b19 adjust the default max_connections_per_download and peer_connect_timeout 2019-05-09 10:38:32 -04:00
Jack Robison c31efd780b docs 2019-05-09 10:32:07 -04:00
Jack Robison 7cdb7987e3 Merge pull request #2102 from lbryio/shutdown_fast
Set a shutdown timeout for TCPSite to avoid hanging on exit
2019-05-09 10:24:13 -04:00
Victor Shyba ec07037e88 set shutdown_timeout on TCPSite 2019-05-09 01:01:07 -03:00
Lex Berezhny 9bb7b0571e fix integrat tests 2019-05-09 00:00:39 -04:00
Lex Berezhny d98162a118 stream_update --replace also clears the channel 2019-05-09 00:00:39 -04:00
Jack Robison c5ae59bff7 Merge pull request #2100 from lbryio/junction_fixes
DHT junction fixes for cases that can lock up
2019-05-08 23:52:54 -04:00
Victor Shyba efbf2f49a9 fix generator junction 2019-05-08 23:02:38 -03:00
Victor Shyba 77c69f661d add locking cases scenarios 2019-05-08 23:02:38 -03:00
Victor Shyba 2e09d2adfa close search on transport close 2019-05-08 23:02:38 -03:00
Jack Robison 8620a0e424 Merge pull request #2095 from lbryio/add-content-fee-and-saved-file-to-db
add saved_file and content_fee columns to file table
2019-05-08 20:37:56 -04:00
Jack Robison e2c123456b write decrypted blob data from run_in_executor 2019-05-08 16:07:32 -04:00
Jack Robison c9be79f062 prevent re-migrating 10 to 11 2019-05-08 15:13:07 -04:00
Jack Robison a2a1ef34ab fix tests 2019-05-08 14:55:09 -04:00
Jack Robison 761a2b235c set written_bytes, download_path, file_name, and download_directory to None if the file doesn't exist 2019-05-08 14:11:44 -04:00
Jack Robison 3ca2fee592 use stream size from claim if available 2019-05-08 14:11:39 -04:00
Jack Robison a480b2d25f fix race when closing blob client protocol after getting a response 2019-05-08 12:21:11 -04:00
Jack Robison d7032b12d7 add saved_file and content_fee columns to file table
-drop not null constraints for file_name and download_directory

-add migrator
2019-05-08 10:26:06 -04:00
Jack Robison 84381ff76c Merge pull request #2093 from lbryio/fix_announce_on_storage
Refactor announcer + fixups for last db migrator code
2019-05-07 10:46:12 -04:00
Victor Shyba 007dd43861 re-raise cancelled errors 2019-05-07 11:10:49 -03:00
Victor Shyba 1bc4e4b702 typo on logging, announcements logging is info, not warning 2019-05-07 11:10:49 -03:00
Victor Shyba b91f27219d use fetchall as there is an update while iterating 2019-05-07 11:10:49 -03:00
Victor Shyba dc4560cc9a refactor announcer 2019-05-07 11:10:49 -03:00
Victor Shyba 84b471d486 limit batch size on announcer test so it covers batching logic 2019-05-07 11:10:49 -03:00
Victor Shyba c6a3c05a0c add missing migrator line 2019-05-07 11:10:49 -03:00
Jack Robison dd665a758d Merge pull request #2065 from lbryio/streaming-bug-fixes
Streaming bug fixes
2019-05-07 10:10:39 -04:00
Jack Robison 32dc1297ec fix infinite loop 2019-05-06 18:46:26 -04:00
Jack Robison bef244ba77 dont make duplicate files 2019-05-06 18:46:26 -04:00
Jack Robison 3c231d6215 handle get with file_name/download_directory/save_file arg for a stream that is already running 2019-05-06 18:46:26 -04:00
Jack Robison a212cf6ba4 set content_fee attribute 2019-05-06 18:46:26 -04:00
Jack Robison bc060f1378 pay fee after downloading the sd blob/adding to the db 2019-05-06 18:46:26 -04:00
Jack Robison 22c701fd50 fix completed field for an item in file list 2019-05-06 18:46:26 -04:00
Jack Robison 4f0e8fce6e add optional download_directory argument to get 2019-05-06 18:46:26 -04:00
Jack Robison f642cfe9dd fix resuming save tasks on startup 2019-05-06 18:46:26 -04:00
Jack Robison 24e073680b add connection id workaround 2019-05-06 18:46:26 -04:00
Jack Robison b2f63a1545 fix tests 2019-05-06 18:46:26 -04:00
Jack Robison f506b3e6d4 tests 2019-05-06 18:46:26 -04:00
Jack Robison f7412514b3 fix partial content 0 padding 2019-05-06 18:46:26 -04:00
Jack Robison bd6a609b30 fix logging 2019-05-06 18:46:26 -04:00
Jack Robison 4e32b69d1d don't set running streams as stopped on startup 2019-05-06 18:46:26 -04:00
Jack Robison 1116c7f29e logging 2019-05-06 18:46:26 -04:00
Jack Robison d80c671cf2 look victor 2019-05-06 18:46:26 -04:00
Jack Robison 14b12cbea2 block /stream and /get requests on the stream manager starting 2019-05-06 18:46:26 -04:00
Jack Robison 1f7feafb67 force close open streaming requests and api calls on shutdown 2019-05-06 18:46:26 -04:00
Jack Robison cbe689ea7f logging 2019-05-06 18:46:26 -04:00
Jack Robison 9099ee2e8e fix/refactor starting and stopping files
-move partial content handling into ManagedStream
-add delayed stop test
2019-05-06 18:46:26 -04:00
Jack Robison b134e0c9c9 fix blob_get 2019-05-06 18:46:26 -04:00
Jack Robison f8c0e80cfc add __slots__ to BlobInfo, StreamDescriptor, and KademliaPeer 2019-05-06 18:46:26 -04:00
Jack Robison 1f266ebdad add logging for streams we already have 2019-05-06 18:46:26 -04:00
Jack Robison b09e1f8825 fix written_bytes never being 0 2019-05-06 18:46:26 -04:00
Jack Robison c496278a5b Merge pull request #2092 from lbryio/fix_announce_on_storage
Set should_announce=1 to head and descriptor blobs when saving a stream to database
2019-05-06 18:45:36 -04:00
Victor Shyba cacc65d45e fix edge case from test for single blob stream 2019-05-06 18:18:12 -03:00
Victor Shyba 2247b354b3 integration test for announcing both on publish and download 2019-05-06 17:27:12 -03:00
Victor Shyba 43111f0497 bump db revision 2019-05-06 17:19:40 -03:00
Victor Shyba 6c142b4477 add migrator for previous bugs 2019-05-06 17:12:04 -03:00
Victor Shyba e8683a24bd fix should_announce being set on stream creation 2019-05-06 16:49:52 -03:00
Victor Shyba a4d17d1705 re-enable tests for announcing on stream creation 2019-05-06 16:49:03 -03:00
Baltazar Gomez a030e13548 fix incorrect media-type of .cbr archives 2019-05-05 20:57:36 -04:00
Jack Robison 32b8601c38 Merge pull request #2083 from lbryio/fix_get_params
make save_file a parameter instead of a flag so it obeys conf
2019-05-05 19:44:29 -04:00
Victor Shyba f5bd28bfdb make save_file a parameter instead of a flag so it obeys conf 2019-05-05 19:46:42 -03:00
Lex Berezhny 6cbbe23c7c account_list defaults to 0 confirmations for balance 2019-05-04 18:08:33 -04:00
Lex Berezhny d7c2e4e9b3 fixed bug where lbrynet failed to start with single-key accounts in the wallet 2019-05-04 17:56:39 -04:00
Jack Robison d61ddbb950 Merge pull request #2076 from lbryio/download_improv
make ttfb configurable, fix race condition on writers, ban on time scales, remove idle code for getting unstuck on long downloads, fix foreign key error and improve downloader loop checks
2019-05-02 17:18:16 -04:00
Victor Shyba cf7bb6a391 fix tests and stream being deleted on data timeout 2019-05-02 16:41:48 -03:00
Victor Shyba 68d7328918 always close the write handle 2019-05-02 12:20:49 -03:00
Victor Shyba ff2985cc80 close protocol if transport reuse brings responses before requests 2019-05-02 12:13:33 -03:00
Victor Shyba ac69fcbc5f fix possible division by zero 2019-05-02 11:52:09 -03:00
Victor Shyba 609cf42868 cleanup downloader code 2019-05-01 18:04:45 -03:00
Victor Shyba e43b29fcd1 if the current writer is closed, its fine to open a new one 2019-05-01 15:09:11 -03:00
Victor Shyba 6594b7c50c test concurrency through races on getting a writer 2019-05-01 15:08:47 -03:00
Victor Shyba 971f2a1cef log download speed only if downloading full streams 2019-05-01 14:31:11 -03:00
Victor Shyba fc88261cc6 add head_blob_only to ttfb 2019-05-01 13:59:45 -03:00
Victor Shyba 895b3992c9 set type for stall download timeout 2019-05-01 13:26:51 -03:00
Victor Shyba 41951d8ec2 improve ttfb script confs 2019-05-01 13:17:13 -03:00
Victor Shyba ae2121c5c4 fix tests and new test 2019-05-01 13:16:45 -03:00
Victor Shyba 771fe935f4 test get-delete-get respects existing files 2019-05-01 13:16:32 -03:00
Victor Shyba 60f7766cf7 small adjustment to new_peer_or_finished 2019-05-01 02:55:51 -03:00
Victor Shyba 63bd983012 pylint 2019-05-01 00:42:51 -03:00
Victor Shyba 7ffce1040c fix foreign key error 2019-04-30 19:51:19 -03:00
Victor Shyba ac6835fa60 dont get stuck 2019-04-30 19:51:02 -03:00
Victor Shyba 3cf5c536c0 improve ban timing and downloader loop friction 2019-04-30 17:56:29 -03:00
Lex Berezhny 85cc1a4763 v0.37.0rc5 2019-04-29 17:22:47 -04:00
Lex Berezhny 180cfcfb72 fix chris45 tests 2019-04-29 16:01:59 -04:00
Lex Berezhny 864c1dd901 fix test_claim_commands integration tests 2019-04-29 16:01:59 -04:00
Lex Berezhny aeaa7ba986 switched from sha256 for unencrypted file hash to sha384 2019-04-29 16:01:59 -04:00
Lex Berezhny b8325c1f89 regenerated docs 2019-04-29 09:49:39 -04:00
Lex Berezhny a36936b5ff v0.37.0rc4 2019-04-29 00:27:53 -04:00
Victor Shyba 7cf245a0e9 encode timestamp and confirmations into file list 2019-04-29 00:26:14 -04:00
Victor Shyba 9f83cce574 test confirmations and timestamp on file list, being updated on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 30b113d64a depth -> confirmations 2019-04-29 00:26:14 -04:00
Victor Shyba ccd2023d37 add timestamp field to resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 3127cd11cf test for timestamp field on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 6e483cba8d tests: add confirm parameters, like the other calls 2019-04-28 22:53:43 -04:00
Victor Shyba 46c03c0cb2 refactor: move generate to blockchain call 2019-04-28 22:53:43 -04:00
Victor Shyba c2ca0a9467 tests: replace sleep with the proper generate call 2019-04-28 22:53:43 -04:00
Victor Shyba b350e74aca fix stream update callback to update the main dict 2019-04-28 22:53:43 -04:00
Victor Shyba de9473f75a test resolve updates file list 2019-04-28 22:53:43 -04:00
Lex Berezhny bfc107fdb3 removed blockchain.block.get_block from wallet server API 2019-04-27 18:03:35 -04:00
Lex Berezhny 6a2efa54ab dropped block_show command from API, users should run a full node if they need access to blocks 2019-04-27 18:03:35 -04:00
Lex Berezhny 72f0e1ca14 added timestamp field to transaction output results from JSON RPC calls 2019-04-27 17:10:49 -04:00
Lex Berezhny 9a1262d557 regenerate docs 2019-04-27 13:13:20 -04:00
Lex Berezhny 706a8e52dc v0.37.0rc3 2019-04-27 13:12:27 -04:00
Lex Berezhny f3c8776878 fixed --preview mode on stream_create/update 2019-04-27 13:11:32 -04:00
Lex Berezhny 539ea04039 fixed signing streams using --channel_name argument 2019-04-27 11:52:29 -04:00
Lex Berezhny 8f96b07559 updating stream claim with new file now correctly updates file list removing old file 2019-04-27 10:55:20 -04:00
Lex Berezhny 0a1e97fe23 cli test fix 2019-04-26 23:37:10 -04:00
Lex Berezhny da637b9c0e added --replace option to stream_update and channel_update, publish defaults to --replace 2019-04-26 23:37:10 -04:00
Lex Berezhny 3618de1227 added --clear_fee ability to stream update command 2019-04-26 17:52:29 -04:00
Victor Shyba 3cb8a63bf9 add channel_name parameter on claim search 2019-04-26 17:30:21 -04:00
Victor Shyba bad99de5b3 add tests for channel_name, which should work for urls too 2019-04-26 17:30:21 -04:00
Lex Berezhny 3d1efbc9bb v0.37.0rc2 2019-04-25 20:40:07 -04:00
Victor Shyba 7bdef3146b fix ttfb 2019-04-25 20:38:33 -04:00
Victor Shyba 2145bfcabf install uvloop on wallet server docker image and default to it 2019-04-25 20:07:10 -04:00
Lex Berezhny b88832d0a9 regenerated docs/api.json 2019-04-24 23:03:30 -04:00
Lex Berezhny db08882f20 v0.37.0rc1 2019-04-24 21:35:59 -04:00
Jack Robison 6f52d36b22 Merge pull request #2020 from lbryio/download-range-requests
Support HTTP 206 partial content requests for streaming downloads
2019-04-24 14:21:41 -04:00
Jack Robison 056a629e87 remove unused property 2019-04-24 13:39:06 -04:00
Jack Robison 1ffb99f9f9 rename streaming_only setting -> save_files 2019-04-24 12:44:12 -04:00
Jack Robison a8e612773d test 2019-04-24 09:48:04 -04:00
Jack Robison 67088b2ca2 test_blob_writers_concurrency 2019-04-24 09:48:04 -04:00
Jack Robison dab49e812f set writer result or error before calling close_handle 2019-04-24 09:48:04 -04:00
Jack Robison 0f3f38d8a3 pylint 2019-04-24 09:48:04 -04:00
Jack Robison 7ba433c2e5 fix setting blob verified event and canceling blob writers
-raise OSError if multiple writers for one peer are opened for a blob
2019-04-24 09:48:04 -04:00
Jack Robison bb7649e8ad test file_save stopped before finished writing with streaming_only on/off 2019-04-24 09:48:04 -04:00
Jack Robison 832537a5cf set deleted downloads as streaming mode on startup 2019-04-24 09:48:04 -04:00
Jack Robison 3234d70270 add peer address/port to hash blob writer 2019-04-24 09:48:04 -04:00
Jack Robison 432fe444f8 fix cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 861c25716e add unit tests for cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Victor Shyba 9ba6653475 raise proper error when sd is not available 2019-04-24 09:48:04 -04:00
Jack Robison 8107e97304 remove debugging /streams endpoint 2019-04-24 09:48:04 -04:00
Jack Robison e2663f6d57 remove unnecessary except/raise 2019-04-24 09:48:04 -04:00
Jack Robison 35479dcbd2 add file_save integration tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a997277aa reduce the amount of Blob objects created 2019-04-24 09:48:04 -04:00
Jack Robison a7d94f4514 add file_save 2019-04-24 09:48:04 -04:00
Jack Robison a0e3338177 only re-reflect streams if the sd blob is readable 2019-04-24 09:48:04 -04:00
Jack Robison d5a353066c missing await 2019-04-24 09:48:04 -04:00
Jack Robison bc29fc4465 type 2019-04-24 09:48:04 -04:00
Jack Robison 987f175fcc logging 2019-04-24 09:48:04 -04:00
Jack Robison 5b1398b588 rename test_streaming.py 2019-04-24 09:48:04 -04:00
Jack Robison 873b58efc9 update example_daemon_settings.yml 2019-04-24 09:48:04 -04:00
Jack Robison 01a81fceed more streaming_only + save_blobs integration testing 2019-04-24 09:48:04 -04:00
Jack Robison a14b0aa573 tests 2019-04-24 09:48:04 -04:00
Jack Robison a1abb7c8fb test range requests 2019-04-24 09:48:04 -04:00
Jack Robison 46b166952b non async blob_completed callback, tests 2019-04-24 09:48:04 -04:00
Jack Robison 1375153fda fix publish updates not being saved to the db, don't query the db for the stream hash 2019-04-24 09:48:04 -04:00
Jack Robison 1a14157d37 fix unawaited task 2019-04-24 09:48:04 -04:00
Jack Robison 40b9fa38b0 update mock dht 2019-04-24 09:48:04 -04:00
Jack Robison 4d0e9e0428 relative http redirect from /get -> /stream 2019-04-24 09:48:04 -04:00
Jack Robison 3a53ef5690 catch TransportNotConnected 2019-04-24 09:48:04 -04:00
Jack Robison 33a68b5cef handle dht is_closing 2019-04-24 09:48:04 -04:00
Jack Robison 63bf084a6a cancel dht response future 2019-04-24 09:48:04 -04:00
Jack Robison a882f2e8c5 merge conflicts 2019-04-24 09:48:04 -04:00
Jack Robison 119e0ec408 cancelled error 2019-04-24 09:48:04 -04:00
Jack Robison 7d782d516a track and close blob readers 2019-04-24 09:48:04 -04:00
Jack Robison d3eac48035 tests, add __slots__ to AbstractBlob 2019-04-24 09:48:04 -04:00
Jack Robison 60c9ae64b4 endpoint to call get and redirect to the partial content stream 2019-04-24 09:48:04 -04:00
Jack Robison 2d314dce60 add save_file argument to get 2019-04-24 09:48:04 -04:00
Jack Robison 287b89db66 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 7a160f7335 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 04ddee7d89 blob tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a916a8e8e tests 2019-04-24 09:48:04 -04:00
Jack Robison f125468ebf fix claim not being saved to the db 2019-04-24 09:48:04 -04:00
Jack Robison f0e17cff9a support streaming downloads / range requests 2019-04-24 09:48:04 -04:00
Jack Robison 93267efe0b remove StreamAssembler, refactor ManagedStream and StreamDownloader (WIP) 2019-04-24 09:48:04 -04:00
Jack Robison c663e5a3cf add cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 676f0015aa refactor BlobFile into AbstractBlob, BlobFile, and BlobBuffer classes 2019-04-24 09:48:04 -04:00
Jack Robison d44a79ada2 cache resolve_host, add async_timed_cache decorator 2019-04-24 09:48:04 -04:00
Jack Robison d7ed8ffaa6 add streaming_only and save_blobs settings 2019-04-24 09:48:04 -04:00
Jack Robison b230d8fbcb rename BlobFileManager -> BlobManager 2019-04-24 09:48:04 -04:00
Lex Berezhny 7d2428fd17 regenerate api.json 2019-04-23 23:49:39 -04:00
Lex Berezhny 4157045a41 block_show command example 2019-04-23 23:49:39 -04:00
Lex Berezhny 58987857cf v0.36.0 2019-04-22 18:47:58 -04:00
Lex Berezhny fa8858325c add lbry:// to permanent_url response field 2019-04-22 18:41:11 -04:00
Lex Berezhny 3357a419ab ability to overide the file name, hash and size when updating a stream claim 2019-04-22 18:05:49 -04:00
Lex Berezhny a782adea75 regenerate api doc 2019-04-22 14:06:42 -04:00
Lex Berezhny e8991e6d52 improve account locking/unlocing CLI docs 2019-04-22 14:06:42 -04:00
Lex Berezhny 13bd81c2b6 v0.36.0rc4 2019-04-21 23:11:23 -04:00
Victor Shyba 21241b98a4 workaround for having new encoder fields on resolve so app can use them 2019-04-21 22:51:48 -04:00
Lex Berezhny bf467e06c5 added create stream image detection test 2019-04-21 22:12:02 -04:00
Lex Berezhny 886ed6415a added examples which include all metadata passed to create stream and create channel 2019-04-21 21:34:13 -04:00
Lex Berezhny 97db241109 v0.36.0rc3 2019-04-21 20:20:53 -04:00
Lex Berezhny 216379918b renamed txo sub_type to value_type in txo json encoder output 2019-04-21 17:40:40 -04:00
Lex Berezhny fcab4eb9bd make sure to use python3 when running set_build.py 2019-04-21 16:31:46 -04:00
Lex Berezhny 19f300cc45 added protobuf to get and file_list commands 2019-04-21 14:28:43 -04:00
Lex Berezhny 26fec098e4 fix stream unit tests 2019-04-21 14:28:43 -04:00
Lex Berezhny 7824de655e updated stream manager to pull protobuf instead of hex from resolve response 2019-04-21 14:28:43 -04:00
Lex Berezhny 6ef1b1dd59 resolve command also returns protobuf 2019-04-21 14:28:43 -04:00
Lex Berezhny 9fae8f5153 added --include_protobuf to add hexlified protobuf to RPC responses 2019-04-21 14:28:43 -04:00
Lex Berezhny f660691e78 v0.36.0rc2 2019-04-21 00:50:27 -04:00
Lex Berezhny bf60683ba6 update to protobuf change where ClaimList defaults to collection list type 2019-04-21 00:45:51 -04:00
Lex Berezhny 68215699f7 fix chri45 test 2019-04-21 00:45:51 -04:00
Lex Berezhny 1644ffbf22 regenerated docs 2019-04-21 00:45:51 -04:00
Lex Berezhny 71f5061848 exposed new protobuf fields in APIs and updated tests 2019-04-21 00:45:51 -04:00
Lex Berezhny 61cadb5cbe v0.36.0rc1 2019-04-20 02:34:27 -04:00
Lex Berezhny 2034104cfc test fixes 2019-04-20 02:33:41 -04:00
Lex Berezhny 633f725634 stream manager test fix 2019-04-20 02:33:41 -04:00
Lex Berezhny 8bf2e41c95 --contact_email -> --email and --homepage_url -> --website_url 2019-04-20 02:33:41 -04:00
Lex Berezhny fa9a4a75c6 update to an even newer protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny 90bef98bc3 use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 35c43647aa use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 449a583286 default to claimType == 1 when it is missing for old claim protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny b1f813782a regenerated protobuf python code 2019-04-20 02:33:41 -04:00
Lex Berezhny 60080cbd19 updating to new protobufs 2019-04-20 02:33:41 -04:00
Victor Shyba 665dba62c7 simplify save from resolve, deduplicate items 2019-04-20 01:19:40 -04:00
Victor Shyba 76dea9e89b batch save supports 2019-04-20 01:19:40 -04:00
Victor Shyba ebc9ed01c5 make lru_cache a power of two, following py docs 2019-04-20 01:19:40 -04:00
Victor Shyba 55116763e9 cache transactions 2019-04-20 01:19:40 -04:00
Victor Shyba e1a6b4e3d6 fix dict handling mistake 2019-04-20 01:19:40 -04:00
Victor Shyba 911ca8c37e cache and share txs fetches while validating concurrently on batches 2019-04-20 01:19:40 -04:00
jessop 7f5deaf6c8 adds api examples to docs 2019-04-16 14:18:23 -04:00
Lex Berezhny 0200756218 put generated commands in groups 2019-04-12 14:24:16 -04:00
ポール ウェッブ a63442f843 Fixes and updated gitignore 2019-04-12 11:53:37 -04:00
Oleg Silkin 74b3da79aa Adds integration tests 2019-04-07 15:27:59 -04:00
Oleg Silkin b6a4589da5 Adds comment group to lbrynet with commands list and create 2019-04-07 15:27:59 -04:00
Oleg Silkin fbbb0a9931 Adds server URL to configuration settings 2019-04-07 15:27:59 -04:00
Oleg Silkin a94d65d522 Adds utility functions for making batch jsonrpc requests to comment server 2019-04-07 15:27:59 -04:00
Oleg Silkin fa2ead002e Fixes incompatible attrs issue when running pip install -e . 2019-04-06 18:43:34 -04:00
Thomas Zarebczan 8dfee4867a Update python3 > python, add mac instructions 2019-04-06 17:10:32 -04:00
Thomas Zarebczan 6614ab8e79 updated with new venv command 2019-04-06 17:10:32 -04:00
Thomas Zarebczan d32c1dfbb3 update install for windowz
Didn't need to install any build tools
Also, venv commands are a bit different.
2019-04-06 17:10:32 -04:00
Lex Berezhny 9f42b958fa run examples fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 574a912f7d run examples as part of ci 2019-04-06 17:08:15 -04:00
Lex Berezhny 2cfcc3cd45 test fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 9af625e627 test fixes 2019-04-06 17:08:15 -04:00
Lex Berezhny 1871c94003 new examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 93220c3874 daemon json rpc return type docs 2019-04-06 17:08:15 -04:00
Lex Berezhny 362ed3f1c8 remove unused import 2019-04-06 17:08:15 -04:00
Lex Berezhny 62a182b8bb automated example generator 2019-04-06 17:08:15 -04:00
Lex Berezhny 6f76d9e24a moved integration testcase.py into lbrynet so that it can be used from scripts 2019-04-06 17:08:15 -04:00
Lex Berezhny 55e83fe05c regenerate api.json 2019-04-06 17:08:15 -04:00
Lex Berezhny 2f0c7d6a3f added title to examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 59e1df9efa generate api examples 2019-04-06 17:08:15 -04:00
Victor Shyba fd56dc68cd fix depth on resolve, add tests 2019-04-05 09:40:26 -04:00
Victor Shyba a0a810edc3 make permanent url name#claim_id on torba too 2019-04-05 09:40:26 -04:00
Victor Shyba c1d3b5984d fix old hash reverse confusion 2019-04-05 09:40:26 -04:00
Victor Shyba 1fb719dd61 fix valid_at_height not being shown for name resolutions, which fixes tests 2019-04-05 09:40:26 -04:00
Victor Shyba b87316fb02 make permanent url name#id only for now 2019-04-05 09:40:26 -04:00
Victor Shyba 65f3d78084 fix invalid signatures for missing certificates 2019-04-05 09:40:26 -04:00
Victor Shyba 47bc1526f7 test invalid signature to be False, not accepting None 2019-04-05 09:40:26 -04:00
Victor Shyba 3acabec4bb fixes from review 2019-04-03 00:51:05 -04:00
Victor Shyba 36a62d3756 port orchstr8 fix to our test case as its lbrynet specific 2019-04-03 00:51:05 -04:00
Victor Shyba 2e22473ce3 pylint and smaller fixes 2019-04-03 00:51:05 -04:00
Victor Shyba 0b0fd64642 further refactor resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba ca17d8597d move resolve from ledger to resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba d16e2716c8 handle invalid data 2019-04-03 00:51:05 -04:00
Victor Shyba 89c80cedfd test resolve for invalid data 2019-04-03 00:51:05 -04:00
Thomas Zarebczan 021e30ffde fix: extension check
disregard the first period, otherwise the `filename.` test fails.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan eb2cd8fec1 stream type mapping
Mapping extensions to stream types. I went with document / model / binary for the ones we were missing. This can be adjusted later on if not correct, but at least we'll have the main ones. Added extensions for lbry, cbz, cbr also.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan 6356818557 fix: time to first byte script
Fix with for metadata
2019-04-01 16:14:53 -04:00
YULIUS KURNIAWAN KRISTIANTO 8bf724a78b Update CONTRIBUTING.md 2019-04-01 16:12:49 -04:00
YULIUS KURNIAWAN KRISTIANTO 65d0a924c1 Update README.md 2019-04-01 16:12:49 -04:00
Lex Berezhny ef7bd00f22 round USD fees to nearest penny 2019-03-31 18:14:13 -04:00
Lex Berezhny 3c1fdb5114 v0.35.0 2019-03-29 21:51:05 -04:00
Lex Berezhny 83411acfbd split claim_abandon into stream_abandon and channel_abandon 2019-03-29 21:48:55 -04:00
Lex Berezhny 4230812f82 added integration test to make sure publishing the same stream re-signs it with the original channel 2019-03-29 21:14:20 -04:00
Lex Berezhny 7b57ba96dd fix docopt parsing and stream manager tests 2019-03-29 21:14:20 -04:00
Lex Berezhny 668840921f typo fix 2019-03-29 21:14:20 -04:00
Lex Berezhny a3bd335f7f --languages, --locations fully support all attributes, add BTC 2019-03-29 21:14:20 -04:00
Lex Berezhny 4e6bca1389 support_abandon sends new --keep support to same address as the first support being abandoned 2019-03-29 21:14:20 -04:00
Lex Berezhny c4b15920ff dont show supports to foreign address if the inputs are also foreign 2019-03-29 21:14:20 -04:00
Lex Berezhny 2a55e583e7 added to docstring that a channel_id is a channel claim id 2019-03-29 21:14:20 -04:00
Lex Berezhny 2c4fde06c5 --new-signing-key -> --new_signing_key 2019-03-29 21:14:20 -04:00
Lex Berezhny eca83d7f47 --clear-tags -> --clear_tags 2019-03-29 21:14:20 -04:00
124 changed files with 13585 additions and 6371 deletions

No files matched your search

+1 -1
View File
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
We are also available on Discord at https://chat.lbry.com
-->
+3
View File
@@ -1,3 +1,4 @@
.DS_Store
/build
/dist
/.tox
@@ -8,3 +9,5 @@
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+24 -7
View File
@@ -3,6 +3,11 @@ dist: xenial
language: python
python: "3.7"
env:
global:
# must also be updated in wine_build.sh
- TORBA=master
jobs:
include:
@@ -10,7 +15,7 @@ jobs:
name: "pylint lbrynet"
install:
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script: pylint lbrynet
@@ -18,7 +23,7 @@ jobs:
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
@@ -29,12 +34,24 @@ jobs:
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
@@ -68,8 +85,8 @@ jobs:
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- python scripts/set_build.py
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
@@ -110,7 +127,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
@@ -118,7 +135,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
cache:
+1 -1
View File
@@ -1,3 +1,3 @@
## Contributing to LBRY
https://lbry.io/faq/contributing
https://lbry.tech/contribute
+16 -13
View File
@@ -9,35 +9,32 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
## Prerequisites
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
After installing python 3, you'll need to install some additional libraries depending on your operating system.
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
### macOS
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set
1. PYTHONUNBUFFERED=1
2. EVENT_NOKQUEUE=1
Remaining dependencies can then be installed by running:
```
brew install python3 protobuf
brew install python protobuf
```
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
### Linux
On Ubuntu (we recommend 18.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
@@ -52,8 +49,14 @@ To install:
git clone https://github.com/lbryio/lbry.git
cd lbry
virtualenv lbry-venv --python=python3.7
Creating venv:
python -m venv lbry-venv
Activating lbry-venv on Linux/Mac:
source lbry-venv/bin/activate
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
+3 -3
View File
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
+2784 -1332
View File
File diff suppressed because it is too large. Load diff
+7 -5
View File
@@ -4,20 +4,22 @@
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
- lbryumx1.lbry.com:50001
- lbryumx2.lbry.com:50001
- lbryumx4.lbry.com:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
delete_blobs_on_remove: True
save_blobs: true
save_files: false
dht_node_port: 4444
peer_port: 3333
use_upnp: True
use_upnp: true
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - blob_server
# - dht
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.35.0rc2"
__version__ = "0.38.0rc6"
version = tuple(__version__.split('.'))
+270 -132
View File
@@ -4,6 +4,8 @@ import asyncio
import binascii
import logging
import typing
import contextlib
from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
@@ -21,10 +23,6 @@ log = logging.getLogger(__name__)
_hexmatch = re.compile("^[a-f,0-9]+$")
def is_valid_hashcharacter(char: str) -> bool:
return len(char) == 1 and _hexmatch.match(char)
def is_valid_blobhash(blobhash: str) -> bool:
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@@ -46,143 +44,74 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
return encrypted, digest.hexdigest()
class BlobFile:
"""
A chunk of data available on the network which is specified by a hashsum
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
if len(data) != length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
This class is used to create blobs on the local filesystem
when we already know the blob hash before hand (i.e., when downloading blobs)
Also can be used for reading from blobs on the local filesystem
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
class AbstractBlob:
"""
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
This class is non-io specific
"""
__slots__ = [
'loop',
'blob_hash',
'length',
'blob_completed_callback',
'blob_directory',
'writers',
'verified',
'writing',
'readers'
]
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self.loop = loop
self.blob_hash = blob_hash
self.length = length
self.blob_dir = blob_dir
self.file_path = os.path.join(blob_dir, self.blob_hash)
self.writers: typing.List[HashBlobWriter] = []
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=loop)
self.blob_write_lock = asyncio.Lock(loop=loop)
if self.file_exists:
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
self.length = length
self.verified.set()
self.finished_writing.set()
self.saved_verified_blob = False
self.blob_completed_callback = blob_completed_callback
self.blob_directory = blob_directory
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
self.readers: typing.List[typing.BinaryIO] = []
@property
def file_exists(self):
return os.path.isfile(self.file_path)
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
def __del__(self):
if self.writers or self.readers:
log.warning("%s not closed before being garbage collected", self.blob_hash)
self.close()
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
raise NotImplementedError()
@contextlib.contextmanager
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
with self._reader_context() as reader:
try:
error = finished.exception()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
self.writers.remove(writer)
if not error: # the blob downloaded, cancel all the other download attempts and set the result
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
log.exception("something else")
raise error
return callback
self.readers.append(reader)
yield reader
finally:
if reader in self.readers:
self.readers.remove(reader)
async def save_verified_blob(self, writer, verified_bytes: bytes):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
async with self.blob_write_lock:
if self.verified.is_set():
return
await self.loop.run_in_executor(None, _save_verified)
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers.append(writer)
fut.add_done_callback(self.writer_finished(writer))
return writer
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
"""
with open(self.file_path, 'rb') as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def close(self):
while self.writers:
self.writers.pop().finished.cancel()
def delete(self):
self.close()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
self.verified.clear()
self.finished_writing.clear()
self.length = None
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with open(self.file_path, "rb") as f:
buff = f.read()
if len(buff) != self.length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
@classmethod
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_dir, blob_hash, length)
writer = blob.open_for_writing()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def set_length(self, length):
def set_length(self, length) -> None:
if self.length is not None and length == self.length:
return
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
@@ -190,8 +119,217 @@ class BlobFile:
return
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
def get_length(self):
def get_length(self) -> typing.Optional[int]:
return self.length
def get_is_verified(self):
def get_is_verified(self) -> bool:
return self.verified.is_set()
def is_readable(self) -> bool:
return self.verified.is_set()
def is_writeable(self) -> bool:
return not self.writing.is_set()
def write_blob(self, blob_bytes: bytes):
if not self.is_writeable():
raise OSError("cannot open blob for writing")
try:
self.writing.set()
self._write_blob(blob_bytes)
finally:
self.writing.clear()
def close(self):
while self.writers:
peer, writer = self.writers.popitem()
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
writer.finished.cancel()
while self.readers:
reader = self.readers.pop()
if reader:
reader.close()
def delete(self):
self.close()
self.verified.clear()
self.length = None
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
"""
if not self.is_readable():
raise OSError('blob files cannot be read')
with self.reader_context() as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with self.reader_context() as reader:
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
writer = blob.get_blob_writer()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers[(peer_address, peer_port)] = writer
def remove_writer(_):
if (peer_address, peer_port) in self.writers:
del self.writers[(peer_address, peer_port)]
fut.add_done_callback(remove_writer)
def writer_finished_callback(finished: asyncio.Future):
try:
err = finished.exception()
if err:
raise err
verified_bytes = finished.result()
while self.writers:
_, other = self.writers.popitem()
if other is not writer:
other.close_handle()
self.save_verified_blob(verified_bytes)
except (InvalidBlobHashError, InvalidDataError) as error:
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
pass
fut.add_done_callback(writer_finished_callback)
return writer
class BlobBuffer(AbstractBlob):
"""
An in-memory only blob
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self._verified_bytes: typing.Optional[BytesIO] = None
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError("cannot open blob for reading")
try:
yield self._verified_bytes
finally:
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
return super().delete()
def __del__(self):
super().__del__()
if self._verified_bytes:
self.delete()
class BlobFile(AbstractBlob):
"""
A blob existing on the local file system
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
if not blob_directory or not os.path.isdir(blob_directory):
raise OSError(f"invalid blob directory '{blob_directory}'")
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
if self.file_exists:
file_size = int(os.stat(self.file_path).st_size)
if length and length != file_size:
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
self.delete()
else:
self.length = file_size
self.verified.set()
@property
def file_exists(self):
return os.path.isfile(self.file_path)
def is_writeable(self) -> bool:
return super().is_writeable() and not os.path.isfile(self.file_path)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
return super().get_blob_writer(peer_address, peer_port)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
handle = open(self.file_path, 'rb')
try:
yield handle
finally:
handle.close()
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def delete(self):
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> BlobInfo:
if not blob_dir or not os.path.isdir(blob_dir):
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
return await super().create_from_unencrypted(
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
)
+7
View File
@@ -2,6 +2,13 @@ import typing
class BlobInfo:
__slots__ = [
'blob_hash',
'blob_num',
'length',
'iv',
]
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
self.blob_hash = blob_hash
self.blob_num = blob_num
+67 -22
View File
@@ -2,18 +2,21 @@ import os
import typing
import asyncio
import logging
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobFileManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
class BlobManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
@@ -27,55 +30,97 @@ class BlobFileManager:
self._node_data_store = node_data_store
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, BlobFile] = {}
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
buffer = self.blobs.pop(blob_hash)
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
if buffer.is_readable():
with buffer.reader_context() as reader:
self.blobs[blob_hash].write_blob(reader.read())
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
return self.blobs[blob_hash]
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
def get_files_in_blob_dir() -> typing.Set[str]:
if not self.blob_dir:
return set()
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
self.connection_manager.start()
return True
def stop(self):
self.connection_manager.stop()
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
self.completed_blob_hashes.clear()
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
return self.blobs[blob_hash]
def get_stream_descriptor(self, sd_hash):
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
async def blob_completed(self, blob: BlobFile):
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
if blob.blob_hash is None:
raise Exception("Blob hash is None")
if not blob.length:
raise Exception("Blob has a length of 0")
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
if isinstance(blob, BlobFile):
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
else:
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
blobs = [self.get_blob(b) for b in blob_hashes]
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
if blob_hash not in self.blobs:
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
+3 -2
View File
@@ -44,16 +44,15 @@ class HashBlobWriter:
self._hashsum.update(data)
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
self.close_handle()
return
self.buffer.write(data)
if self.len_so_far == expected_length:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
@@ -62,6 +61,8 @@ class HashBlobWriter:
self.close_handle()
def close_handle(self):
if not self.finished.done():
self.finished.cancel()
if self.buffer is not None:
self.buffer.close()
self.buffer = None
+64 -23
View File
@@ -4,37 +4,52 @@ import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.connection_manager import ConnectionManager
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
connection_manager: typing.Optional['ConnectionManager'] = None):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.transport: typing.Optional[asyncio.Transport] = None
self.peer_timeout = peer_timeout
self.transport: asyncio.Transport = None
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.connection_manager = connection_manager
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._response_fut: typing.Optional[asyncio.Future] = None
self.buf = b''
# this is here to handle the race when the downloader is closed right as response_fut gets a result
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if self.connection_manager:
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
# assert self.peer_address is not None
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if not self._response_fut:
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
return self.close()
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
@@ -63,8 +78,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def _write(self, data):
def _write(self, data: bytes):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
@@ -87,14 +101,24 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
@@ -133,6 +157,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return self._blob_bytes_received, self.close()
def close(self):
self.closed.set()
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
@@ -145,18 +170,21 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
self.closed.clear()
blob_hash = blob.blob_hash
if blob.get_is_verified() or not blob.is_writeable():
return 0, self.transport
try:
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self._blob_bytes_received = 0
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
# i'm not sure how to fix this race condition - jack
log.exception(e)
return self._blob_bytes_received, self.transport
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
@@ -165,30 +193,43 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
except asyncio.CancelledError:
self.close()
raise
finally:
if self.writer and not self.writer.closed():
self.writer.close_handle()
self.writer = None
def connection_made(self, transport: asyncio.Transport):
addr = transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr[0], addr[1]
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
if self.connection_manager:
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None)\
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
protocol = BlobExchangeClientProtocol(
loop, blob_download_timeout, connection_manager
)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.connection_made(connected_transport)
protocol.transport = connected_transport
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
@@ -196,7 +237,7 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: s
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or blob.file_exists:
if blob.get_is_verified() or not blob.is_writeable():
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
+43 -57
View File
@@ -1,21 +1,22 @@
import asyncio
import typing
import logging
from lbrynet.utils import drain_tasks
from lbrynet.utils import cache_concurrent
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
log = logging.getLogger(__name__)
class BlobDownloader:
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
@@ -24,114 +25,99 @@ class BlobDownloader:
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.failures: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.time_since_last_blob = loop.time()
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'BlobFile'):
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
# the safe net side is that any failure will reset the peer score, triggering the race back
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
# for peer, task in self.active_connections.items():
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
# return False
return not (blob.get_is_verified() or blob.file_exists)
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
if blob.get_is_verified():
return
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
)
if bytes_received == blob.get_length():
self.time_since_last_blob = self.loop.time()
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = self.failures.get(peer, 0) + 1
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = 0
self.connections[peer] = transport
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
self.scores[peer] = rough_speed
elapsed = self.loop.time() - start
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
async def new_peer_or_finished(self, blob: 'BlobFile'):
async def get_and_re_add_peers():
try:
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
self.peer_queue.put_nowait(new_peers)
except asyncio.TimeoutError:
pass
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
active_tasks = list(self.active_connections.values())
try:
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
finally:
drain_tasks(tasks)
async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
def cleanup_active(self):
if not self.active_connections and not self.connections:
self.clearbanned()
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
def clearbanned(self):
now = self.loop.time()
if now - self.time_since_last_blob > 60.0:
return
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
self.peer_queue.put_nowait(forgiven)
for banned_peer in forgiven:
self.ignored.pop(banned_peer)
self.ignored = dict((
(peer, when) for (peer, when) in self.ignored.items()
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
))
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
@cache_concurrent
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
connection_id: int = 0) -> 'AbstractBlob':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
self.is_running.set()
try:
while not blob.get_is_verified():
batch: typing.List['KademliaPeer'] = []
while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set()
while not self.peer_queue.empty():
batch.extend(self.peer_queue.get_nowait())
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
batch.update(self.peer_queue.get_nowait())
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in batch:
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
await self.new_peer_or_finished(blob)
await self.new_peer_or_finished()
self.cleanup_active()
if batch:
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
else:
self.clearbanned()
blob.close()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
while self.active_connections:
self.active_connections.popitem()[1].cancel()
blob.close()
def close(self):
self.scores.clear()
self.ignored.clear()
self.is_running.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
blob_hash: str) -> 'BlobFile':
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
blob_hash: str) -> 'AbstractBlob':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
+14 -5
View File
@@ -8,13 +8,13 @@ from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPr
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
log = logging.getLogger(__name__)
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -22,15 +22,23 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None
def connection_made(self, transport):
self.transport = transport
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
self.transport.write(BlobResponse(to_send).serialize())
serialized = BlobResponse(to_send).serialize()
self.transport.write(serialized)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
@@ -72,6 +80,7 @@ class BlobServerProtocol(asyncio.Protocol):
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
@@ -94,10 +103,10 @@ class BlobServerProtocol(asyncio.Protocol):
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
+1
View File
@@ -1,2 +1,3 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
+51 -21
View File
@@ -121,7 +121,7 @@ class Toggle(Setting[bool]):
)
parser.add_argument(
self.no_cli_name,
help=f"Opposite of --{self.cli_name}",
help=f"Opposite of {self.cli_name}",
dest=self.name,
action="store_false",
default=NOT_SET
@@ -192,7 +192,21 @@ class MaxKeyFee(Setting[dict]):
)
class Servers(Setting[list]):
class ListSetting(Setting[list]):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
f"Setting '{self.name}' must be a tuple or list."
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
action='append'
)
class Servers(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
@@ -225,16 +239,8 @@ class Servers(Setting[list]):
return [f"{host}:{port}" for host, port in value]
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
nargs="*",
help=self.doc,
default=NOT_SET
)
class Strings(Setting[list]):
class Strings(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
@@ -269,7 +275,7 @@ class ArgumentAccess:
def load(self, args):
for setting in self.configuration.get_settings():
value = getattr(args, setting.name, NOT_SET)
if value != NOT_SET:
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
self.args[setting.name] = setting.deserialize(value)
def __contains__(self, item: str):
@@ -480,10 +486,15 @@ class Config(CLIConfig):
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
)
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
@@ -493,7 +504,7 @@ class Config(CLIConfig):
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 8,
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
@@ -515,31 +526,50 @@ class Config(CLIConfig):
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
('lbryumx1.lbry.com', 50001), # US EAST
('lbryumx2.lbry.com', 50001), # US EAST
('lbryumx4.lbry.com', 50001) #EU
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.io', 4444), # US EAST
('lbrynet2.lbry.io', 4444), # US WEST
('lbrynet3.lbry.io', 4444), # EU
('lbrynet4.lbry.io', 4444) # ASIA
('lbrynet1.lbry.com', 4444), # US EAST
('lbrynet2.lbry.com', 4444), # US WEST
('lbrynet3.lbry.com', 4444), # EU
('lbrynet4.lbry.com', 4444) # ASIA
])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
# media server
streaming_server = String('Host name and port to serve streaming media over range requests',
'localhost:5280', metavar='HOST:PORT')
streaming_get = Toggle("Enable the /get endpoint for the streaming media server. "
"Disable to prevent new streams from being added.", True)
@property
def streaming_host(self):
return self.streaming_server.split(':')[0]
@property
def streaming_port(self):
return int(self.streaming_server.split(':')[1])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.set_default_paths()
+90
View File
@@ -0,0 +1,90 @@
import asyncio
import typing
import collections
import logging
log = logging.getLogger(__name__)
CONNECTED_EVENT = "connected"
DISCONNECTED_EVENT = "disconnected"
TRANSFERRED_EVENT = "transferred"
class ConnectionManager:
def __init__(self, loop: asyncio.AbstractEventLoop):
self.loop = loop
self.incoming_connected: typing.Set[str] = set()
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._task: typing.Optional[asyncio.Task] = None
@property
def status(self):
return self._status
def sent_data(self, host_and_port: str, size: int):
self.outgoing[host_and_port] += size
def received_data(self, host_and_port: str, size: int):
self.incoming[host_and_port] += size
def connection_made(self, host_and_port: str):
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
def outgoing_connection_lost(self, host_and_port: str):
if host_and_port in self.outgoing_connected:
self.outgoing_connected.remove(host_and_port)
def incoming_connection_lost(self, host_and_port: str):
if host_and_port in self.incoming_connected:
self.incoming_connected.remove(host_and_port)
async def _update(self):
self._status = {
'incoming_bps': {},
'outgoing_bps': {},
'total_incoming_mbs': 0.0,
'total_outgoing_mbs': 0.0,
'time': self.loop.time()
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
) / (now - last)) / 1000000.0
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
) / (now - last)) / 1000000.0
self._status['time'] = now
def stop(self):
if self._task:
self._task.cancel()
self._task = None
self.outgoing.clear()
self.outgoing_connected.clear()
self.incoming.clear()
self.incoming_connected.clear()
self._status.clear()
def start(self):
self.stop()
self._task = self.loop.create_task(self._update())
+36 -44
View File
@@ -13,55 +13,47 @@ class BlobAnnouncer:
self.loop = loop
self.node = node
self.storage = storage
self.pending_call: asyncio.Handle = None
self.announce_task: asyncio.Task = None
self.running = False
self.announce_queue: typing.List[str] = []
async def _announce(self, batch_size: typing.Optional[int] = 10):
if not batch_size:
return
if not self.node.joined.is_set():
await self.node.joined.wait()
blob_hashes = await self.storage.get_blobs_to_announce()
if blob_hashes:
self.announce_queue.extend(blob_hashes)
log.info("%i blobs to announce", len(blob_hashes))
batch = []
while len(self.announce_queue):
cnt = 0
announced = []
while self.announce_queue and cnt < batch_size:
blob_hash = self.announce_queue.pop()
announced.append(blob_hash)
batch.append(self.node.announce_blob(blob_hash))
cnt += 1
to_await = []
while batch:
to_await.append(batch.pop())
if to_await:
await asyncio.gather(*tuple(to_await), loop=self.loop)
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
if self.running:
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
async def _submit_announcement(self, blob_hash):
try:
peers = len(await self.node.announce_blob(blob_hash))
if peers > 4:
return blob_hash
else:
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
def announce(self, batch_size: typing.Optional[int] = 10):
self.announce_task = self.loop.create_task(self._announce(batch_size))
async def _announce(self, batch_size: typing.Optional[int] = 10):
while batch_size:
if not self.node.joined.is_set():
await self.node.joined.wait()
await asyncio.sleep(60, loop=self.loop)
if not self.node.protocol.routing_table.get_peers():
log.warning("No peers in DHT, announce round skipped")
continue
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
while len(self.announce_queue):
log.info("%i blobs to announce", len(self.announce_queue))
announced = await asyncio.gather(*[
self._submit_announcement(
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
], loop=self.loop)
announced = list(filter(None, announced))
if announced:
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
def start(self, batch_size: typing.Optional[int] = 10):
if self.running:
raise Exception("already running")
self.running = True
self.announce(batch_size)
assert not self.announce_task or self.announce_task.done(), "already running"
self.announce_task = self.loop.create_task(self._announce(batch_size))
def stop(self):
self.running = False
if self.pending_call:
if not self.pending_call.cancelled():
self.pending_call.cancel()
self.pending_call = None
if self.announce_task:
if not (self.announce_task.done() or self.announce_task.cancelled()):
self.announce_task.cancel()
self.announce_task = None
if self.announce_task and not self.announce_task.done():
self.announce_task.cancel()
+69 -98
View File
@@ -2,11 +2,8 @@ import logging
import asyncio
import typing
import binascii
import contextlib
from lbrynet.utils import resolve_host
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
from lbrynet.dht.protocol.distance import Distance
from lbrynet.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
from lbrynet.dht.protocol.protocol import KademliaProtocol
@@ -31,7 +28,7 @@ class Node:
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
async def refresh_node(self):
async def refresh_node(self, force_once=False):
while True:
# remove peers with expired blob announcements from the datastore
self.protocol.data_store.removed_expired_peers()
@@ -58,6 +55,8 @@ class Node:
peers = await self.peer_search(node_ids.pop())
total_peers.extend(peers)
else:
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
await fut
@@ -67,30 +66,33 @@ class Node:
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
await fut
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
announced_to_node_ids = []
while not announced_to_node_ids:
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
peers = await self.peer_search(hash_value)
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
peers = await self.peer_search(hash_value)
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
announced_to_node_ids.extend([node_id for node_id, contacted in stored_to_tup if contacted])
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
if stored_to:
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(announced_to_node_ids), len(peers))
return announced_to_node_ids
len(stored_to), len(peers))
else:
log.warning("Failed announcing %s, stored to 0 peers", blob_hash[:8])
return stored_to
def stop(self) -> None:
if self.joined.is_set():
@@ -101,6 +103,7 @@ class Node:
self._refresh_task.cancel()
if self.protocol and self.protocol.ping_queue.running:
self.protocol.ping_queue.stop()
self.protocol.stop()
if self.listening_port is not None:
self.listening_port.close()
self._join_task = None
@@ -113,6 +116,7 @@ class Node:
lambda: self.protocol, (interface, self.internal_udp_port)
)
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
self.protocol.start()
else:
log.warning("Already bound to port %s", self.listening_port)
@@ -130,33 +134,31 @@ class Node:
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
if (address, port) not in known_node_addresses and\
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
known_node_addresses.append((address, port))
url_to_addr[address] = host
if known_node_addresses:
while not self.protocol.routing_table.get_peers():
success = False
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
try:
await peer.ping()
success = True
except asyncio.TimeoutError:
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
round(self.protocol.rpc_timeout, 2))
if success:
break
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
# in the routing table with good peers who are near us
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
peers = [
KademliaPeer(self.loop, address, udp_port=port)
for (address, port) in known_node_addresses
]
while True:
if not self.protocol.routing_table.get_peers():
if self.joined.is_set():
self.joined.clear()
self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
if self.protocol.routing_table.get_peers():
self.joined.set()
log.info(
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
else:
continue
await asyncio.sleep(1, loop=self.loop)
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
@@ -169,79 +171,48 @@ class Node:
)
)
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.bottom_out_limit,
max_results: int = constants.k) -> IterativeNodeFinder:
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = 40,
max_results: int = -1) -> IterativeValueFinder:
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
@contextlib.asynccontextmanager
async def stream_peer_search_junction(self, hash_queue: asyncio.Queue, bottom_out_limit=20,
max_results=-1) -> AsyncGeneratorJunction:
peer_generator = AsyncGeneratorJunction(self.loop)
async def _add_hashes_from_queue():
while True:
blob_hash = await hash_queue.get()
peer_generator.add_generator(
self.get_iterative_value_finder(
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
max_results=max_results
)
)
add_hashes_task = self.loop.create_task(_add_hashes_from_queue())
try:
async with peer_generator as junction:
yield junction
await peer_generator.finished.wait()
finally:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
def peer_search_junction(self, node_id: bytes, max_results=constants.k*2,
bottom_out_limit=20) -> AsyncGeneratorJunction:
peer_generator = AsyncGeneratorJunction(self.loop)
peer_generator.add_generator(
self.get_iterative_node_finder(
node_id, bottom_out_limit=bottom_out_limit, max_results=max_results
)
)
return peer_generator
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20) -> typing.List['KademliaPeer']:
accumulated: typing.List['KademliaPeer'] = []
async with self.peer_search_junction(node_id, max_results=max_results,
bottom_out_limit=bottom_out_limit) as junction:
async for peers in junction:
accumulated.extend(peers)
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
) -> typing.List['KademliaPeer']:
peers = []
async for iteration_peers in self.get_iterative_node_finder(
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
peers.extend(iteration_peers)
distance = Distance(node_id)
accumulated.sort(key=lambda peer: distance(peer.node_id))
return accumulated[:count]
peers.sort(key=lambda peer: distance(peer.node_id))
return peers[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
async for peers in search_junction:
if peers:
result_queue.put_nowait([
peer for peer in peers
if not (
peer.address == self.protocol.external_ip
and peer.tcp_port == self.protocol.peer_port
)
])
tasks = []
try:
while True:
blob_hash = await search_queue.get()
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
finally:
for task in tasks:
task.cancel()
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
result_queue.put_nowait(results)
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue()
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
q = peer_queue or asyncio.Queue(loop=self.loop)
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
+19 -6
View File
@@ -31,7 +31,10 @@ class PeerManager:
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
def reset(self):
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
statistic.clear()
def report_failure(self, address: str, udp_port: int):
now = self._loop.time()
@@ -104,11 +107,12 @@ class PeerManager:
delay = self._loop.time() - constants.check_refresh_interval
if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
return
addr_tup = (address, udp_port)
if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
return
# fixme: find a way to re-enable that without breaking other parts
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
# return
#addr_tup = (address, udp_port)
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
# return
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
last_requested = self._last_requested.get((address, udp_port))
last_replied = self._last_replied.get((address, udp_port))
@@ -135,6 +139,15 @@ class PeerManager:
class KademliaPeer:
__slots__ = [
'loop',
'_node_id',
'address',
'udp_port',
'tcp_port',
'protocol_version',
]
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
if node_id is not None:
@@ -1,100 +0,0 @@
import asyncio
import typing
import logging
import traceback
if typing.TYPE_CHECKING:
from types import AsyncGeneratorType
log = logging.getLogger(__name__)
def cancel_task(task: typing.Optional[asyncio.Task]):
if task and not (task.done() or task.cancelled()):
task.cancel()
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
class AsyncGeneratorJunction:
"""
A helper to interleave the results from multiple async generators into one
async generator.
"""
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
self.loop = loop
self.result_queue = queue or asyncio.Queue(loop=loop)
self.tasks: typing.List[asyncio.Task] = []
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
self.can_iterate = asyncio.Event(loop=self.loop)
self.finished = asyncio.Event(loop=self.loop)
@property
def running(self):
return any(self.running_iterators.values())
async def wait_for_generators(self):
async def iterate(iterator: typing.AsyncGenerator):
try:
async for item in iterator:
self.result_queue.put_nowait(item)
finally:
self.running_iterators[iterator] = False
while True:
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
self.running_iterators[async_gen] = True
self.tasks.append(self.loop.create_task(iterate(async_gen)))
if not self.can_iterate.is_set():
self.can_iterate.set()
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
"""
Add an async generator. This can be called during an iteration of the generator junction.
"""
self.generator_queue.put_nowait(async_gen)
def __aiter__(self):
return self
async def __anext__(self):
if not self.can_iterate.is_set():
await self.can_iterate.wait()
if not self.running:
raise StopAsyncIteration()
try:
return await self.result_queue.get()
finally:
self.awaiting = None
def aclose(self):
async def _aclose():
for iterator in list(self.running_iterators.keys()):
result = iterator.aclose()
if asyncio.iscoroutine(result):
await result
self.running_iterators[iterator] = False
drain_tasks(self.tasks)
raise StopAsyncIteration()
if not self.finished.is_set():
self.finished.set()
return self.loop.create_task(_aclose())
async def __aenter__(self):
self.tasks.append(self.loop.create_task(self.wait_for_generators()))
return self
async def __aexit__(self, exc_type, exc, tb):
try:
await self.aclose()
except StopAsyncIteration:
pass
finally:
if exc_type:
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
err = traceback.format_exception(exc_type, exc, tb)
log.error(err)
+37 -43
View File
@@ -9,68 +9,62 @@ if typing.TYPE_CHECKING:
class DictDataStore:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
# Dictionary format:
# { <key>: [<contact>, <value>, <lastPublished>, <originallyPublished> <original_publisher_id>] }
self._data_store: typing.Dict[bytes,
typing.List[typing.Tuple['KademliaPeer', bytes, float, float, bytes]]] = {}
self._get_time = loop.time
# { <key>: [(<contact>, <age>), ...] }
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
self.loop = loop
self._peer_manager = peer_manager
self.completed_blobs: typing.Set[str] = set()
def filter_bad_and_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
def removed_expired_peers(self):
now = self.loop.time()
keys = list(self._data_store.keys())
for key in keys:
to_remove = []
for (peer, ts) in self._data_store[key]:
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
to_remove.append((peer, ts))
for item in to_remove:
self._data_store[key].remove(item)
if not self._data_store[key]:
del self._data_store[key]
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired and unknown/good peers
"""
peers = []
for peer in map(lambda p: p[0],
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration,
self._data_store[key])):
for peer in self.filter_expired_peers(key):
if self._peer_manager.peer_is_good(peer) is not False:
peers.append(peer)
return peers
yield peer
def filter_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired peers
"""
return list(
map(
lambda p: p[0],
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration, self._data_store[key])
)
)
def removed_expired_peers(self):
expired_keys = []
for key in self._data_store.keys():
unexpired_peers = self.filter_expired_peers(key)
if not unexpired_peers:
expired_keys.append(key)
else:
self._data_store[key] = [x for x in self._data_store[key] if x[0] in unexpired_peers]
for key in expired_keys:
del self._data_store[key]
now = self.loop.time()
for (peer, ts) in self._data_store.get(key, []):
if ts + constants.data_expiration > now:
yield peer
def has_peers_for_blob(self, key: bytes) -> bool:
return key in self._data_store and len(self.filter_bad_and_expired_peers(key)) > 0
return key in self._data_store
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes, compact_address: bytes, last_published: int,
originally_published: int, original_publisher_id: bytes) -> None:
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
now = self.loop.time()
if key in self._data_store:
if compact_address not in map(lambda store_tuple: store_tuple[1], self._data_store[key]):
self._data_store[key].append(
(contact, compact_address, last_published, originally_published, original_publisher_id)
)
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
if len(current):
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
else:
self._data_store[key].append((contact, now))
else:
self._data_store[key] = [(contact, compact_address, last_published, originally_published,
original_publisher_id)]
self._data_store[key] = [(contact, now)]
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
return [] if key not in self._data_store else [peer for peer in self.filter_bad_and_expired_peers(key)]
return list(self.filter_bad_and_expired_peers(key))
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
peers = set()
for key in self._data_store:
for values in self._data_store[key]:
if values[0] not in peers:
peers.add(values[0])
for key, stored in self._data_store.items():
peers.update(set(map(lambda tup: tup[0], stored)))
return list(peers)
+55 -54
View File
@@ -5,7 +5,7 @@ from itertools import chain
import typing
import logging
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.error import RemoteException, TransportNotConnected
from lbrynet.dht.protocol.distance import Distance
from typing import TYPE_CHECKING
@@ -66,11 +66,7 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
"""
if len(key) != constants.hash_length:
raise ValueError("invalid key length: %i" % len(key))
if not shortlist:
shortlist = routing_table.find_close_peers(key)
distance = Distance(key)
shortlist.sort(key=lambda peer: distance(peer.node_id), reverse=True)
return shortlist
return shortlist or routing_table.find_close_peers(key)
class IterativeFinder:
@@ -91,12 +87,11 @@ class IterativeFinder:
self.max_results = max_results
self.exclude = exclude or []
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
self.contacted: typing.Set['KademliaPeer'] = set()
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
self.closest_peer: typing.Optional['KademliaPeer'] = None
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
self.iteration_queue = asyncio.Queue(loop=self.loop)
@@ -107,6 +102,12 @@ class IterativeFinder:
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
for peer in get_shortlist(routing_table, key, shortlist):
if peer.node_id:
self._add_active(peer)
else:
# seed nodes
self._schedule_probe(peer)
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
"""
@@ -138,25 +139,18 @@ class IterativeFinder:
def _is_closer(self, peer: 'KademliaPeer') -> bool:
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
def _update_closest(self):
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
if self.closest_peer and self.closest_peer is not self.shortlist[-1]:
if self._is_closer(self.shortlist[-1]):
def _add_active(self, peer):
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
self.active.add(peer)
if self._is_closer(peer):
self.prev_closest_peer = self.closest_peer
self.closest_peer = self.shortlist[-1]
self.closest_peer = peer
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.add(peer)
self._add_active(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
@@ -169,53 +163,59 @@ class IterativeFinder:
log.warning(str(err))
self.active.discard(peer)
return
except TransportNotConnected:
return self.aclose()
except RemoteException:
return
return await self._handle_probe_result(peer, response)
async def _search_round(self):
"""
Send up to constants.alpha (5) probes to the closest peers in the shortlist
Send up to constants.alpha (5) probes to closest active peers
"""
added = 0
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
to_probe = list(self.active - self.contacted)
to_probe.sort(key=lambda peer: self.distance(self.key))
for peer in to_probe:
if added >= constants.alpha:
break
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
if origin_address in self.exclude:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.add((peer.address, peer.udp_port))
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.add(t)
added += 1
self._schedule_probe(peer)
added += 1
log.debug("running %d probes", len(self.running_probes))
if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8])
self.search_exhausted()
def _schedule_probe(self, peer: 'KademliaPeer'):
self.contacted.add(peer)
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.add(t)
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
try:
if self.running:
await self._search_round()
if self.running:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration):
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
if self.running:
self.loop.call_soon(self.aclose)
@@ -264,6 +264,7 @@ class IterativeNodeFinder(IterativeFinder):
self.yielded_peers: typing.Set['KademliaPeer'] = set()
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response)
@@ -271,12 +272,16 @@ class IterativeNodeFinder(IterativeFinder):
self.put_result(self.active, finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
not_yet_yielded = [
peer for peer in from_iter
if peer not in self.yielded_peers
and peer.node_id != self.protocol.node_id
and self.peer_manager.peer_is_good(peer) is not False
]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
if to_yield:
for peer in to_yield:
self.yielded_peers.add(peer)
self.yielded_peers.update(to_yield)
self.iteration_queue.put_nowait(to_yield)
if finish:
self.iteration_queue.put_nowait(None)
@@ -286,21 +291,17 @@ class IterativeNodeFinder(IterativeFinder):
if found:
log.debug("found")
return self.put_result(self.shortlist, finish=True)
return self.put_result(self.active, finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
self.bottom_out_count = 0
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
log.info("bottom out %i %i %i %i", len(self.active), len(self.contacted), len(self.shortlist),
self.bottom_out_count)
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active, True)
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.debug("max results")
self.put_result(self.active, True)
class IterativeValueFinder(IterativeFinder):
+77 -39
View File
@@ -45,19 +45,19 @@ class KademliaRPC:
def ping():
return b'pong'
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int,
original_publisher_id: bytes, age: int) -> bytes:
if original_publisher_id is None:
original_publisher_id = rpc_contact.node_id
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
if len(blob_hash) != constants.hash_bits // 8:
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
if not 0 < port < 65535:
raise ValueError(f"invalid tcp port: {port}")
rpc_contact.update_tcp_port(port)
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
pass
elif not self.verify_token(token, rpc_contact.compact_ip()):
raise ValueError("Invalid token")
now = int(self.loop.time())
originally_published = now - age
if not self.verify_token(token, rpc_contact.compact_ip()):
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
pass
else:
raise ValueError("Invalid token")
self.protocol.data_store.add_peer_to_blob(
rpc_contact, blob_hash, rpc_contact.compact_address_tcp(), now, originally_published, original_publisher_id
rpc_contact, blob_hash
)
return b'OK'
@@ -191,12 +191,14 @@ class PingQueue:
self._process_task: asyncio.Task = None
self._running = False
self._running_pings: typing.Set[asyncio.Task] = set()
self._default_delay = constants.maybe_ping_delay
@property
def running(self):
return self._running
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
delay = delay if delay is not None else self._default_delay
now = self._loop.time()
for peer in peers:
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
@@ -207,7 +209,7 @@ class PingQueue:
try:
if self._protocol.peer_manager.peer_is_good(peer):
if peer not in self._protocol.routing_table.get_peers():
await self._protocol.add_peer(peer)
self._protocol.add_peer(peer)
return
await self._protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
@@ -268,11 +270,21 @@ class KademliaProtocol(DatagramProtocol):
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
self.rpc_timeout = rpc_timeout
self._split_lock = asyncio.Lock(loop=self.loop)
self._to_remove: typing.Set['KademliaPeer'] = set()
self._to_add: typing.Set['KademliaPeer'] = set()
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
@functools.lru_cache(128)
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
def start(self):
self.maintaing_routing_task = self.loop.create_task(self.routing_table_task())
def stop(self):
if self.maintaing_routing_task:
self.maintaing_routing_task.cancel()
if self.transport:
self.disconnect()
@@ -363,13 +375,30 @@ class KademliaProtocol(DatagramProtocol):
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
return await self._add_peer(peer)
async def add_peer(self, peer: 'KademliaPeer') -> bool:
def add_peer(self, peer: 'KademliaPeer'):
if peer.node_id == self.node_id:
return False
async with self._split_lock:
return await self._add_peer(peer)
self._to_add.add(peer)
self._wakeup_routing_task.set()
async def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
def remove_peer(self, peer: 'KademliaPeer'):
self._to_remove.add(peer)
self._wakeup_routing_task.set()
async def routing_table_task(self):
while True:
while self._to_remove:
async with self._split_lock:
peer = self._to_remove.pop()
self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
while self._to_add:
async with self._split_lock:
await self._add_peer(self._to_add.pop())
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
self._wakeup_routing_task.clear()
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
binascii.hexlify(self.node_id)[:8].decode())
method = message.method
@@ -387,7 +416,7 @@ class KademliaProtocol(DatagramProtocol):
result = self.node_rpc.ping()
elif method == b'store':
blob_hash, token, port, original_publisher_id, age = a
result = self.node_rpc.store(sender_contact, blob_hash, token, port, original_publisher_id, age)
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
elif method == b'findNode':
key, = a
result = self.node_rpc.find_node(sender_contact, key)
@@ -396,11 +425,11 @@ class KademliaProtocol(DatagramProtocol):
key, = a
result = self.node_rpc.find_value(sender_contact, key)
await self.send_response(
self.send_response(
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
)
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
# This is an RPC method request
self.peer_manager.report_last_requested(address[0], address[1])
try:
@@ -408,7 +437,7 @@ class KademliaProtocol(DatagramProtocol):
except IndexError:
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
await self._handle_rpc(peer, request_datagram)
self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
# will be added to our routing table if successful
is_good = self.peer_manager.peer_is_good(peer)
@@ -416,12 +445,12 @@ class KademliaProtocol(DatagramProtocol):
self.ping_queue.enqueue_maybe_ping(peer)
# only add a requesting contact to the routing table if it has replied to one of our requests
elif is_good is True:
await self.add_peer(peer)
self.add_peer(peer)
except ValueError as err:
log.debug("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
await self.send_error(
self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
@@ -430,13 +459,13 @@ class KademliaProtocol(DatagramProtocol):
log.warning("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
await self.send_error(
self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
)
async def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
# Find the message that triggered this response
if response_datagram.rpc_id in self.sent_messages:
peer, df, request = self.sent_messages[response_datagram.rpc_id]
@@ -459,7 +488,7 @@ class KademliaProtocol(DatagramProtocol):
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
df.set_result(response_datagram)
await self.add_peer(peer)
self.add_peer(peer)
else:
log.warning("%s:%i replied, but after we cancelled the request attempt",
peer.address, peer.udp_port)
@@ -510,37 +539,39 @@ class KademliaProtocol(DatagramProtocol):
return
if isinstance(message, RequestDatagram):
self.loop.create_task(self.handle_request_datagram(address, message))
self.handle_request_datagram(address, message)
elif isinstance(message, ErrorDatagram):
self.handle_error_datagram(address, message)
else:
assert isinstance(message, ResponseDatagram), "sanity"
self.loop.create_task(self.handle_response_datagram(address, message))
self.handle_response_datagram(address, message)
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
await self._send(peer, request)
self._send(peer, request)
response_fut = self.sent_messages[request.rpc_id][1]
try:
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except asyncio.CancelledError:
if not response_fut.done():
response_fut.cancel()
raise
except (asyncio.TimeoutError, RemoteException):
self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
async with self._split_lock:
self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
self.remove_peer(peer)
raise
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
await self._send(peer, response)
def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
self._send(peer, response)
async def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
await self._send(peer, error)
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
self._send(peer, error)
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
ErrorDatagram]):
if not self.transport:
if not self.transport or self.transport.is_closing():
raise TransportNotConnected()
data = message.bencode()
@@ -598,12 +629,15 @@ class KademliaProtocol(DatagramProtocol):
return True
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
try:
async def __store():
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
raise ValueError(res)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, True
try:
return await __store()
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
except ValueError as err:
@@ -611,6 +645,10 @@ class KademliaProtocol(DatagramProtocol):
except Exception as err:
if 'Invalid token' in str(err):
self.peer_manager.clear_token(peer.node_id)
try:
return await __store()
except:
return peer.node_id, False
else:
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
+6
View File
@@ -32,6 +32,12 @@ class DownloadDataTimeout(Exception):
self.download = download
class ResolveTimeout(Exception):
def __init__(self, uri):
super().__init__(f'Failed to resolve "{uri}" within the timeout')
self.uri = uri
class RequestCanceledError(Exception):
pass
+7 -8
View File
@@ -25,7 +25,7 @@ def display(data):
print(json.dumps(data, indent=2))
async def execute_command(conf, method, params):
async def execute_command(conf, method, params, callback=display):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
@@ -33,12 +33,9 @@ async def execute_command(conf, method, params):
try:
data = await resp.json()
if 'result' in data:
display(data['result'])
return callback(data['result'])
elif 'error' in data:
if 'message' in data['error']:
display(data['error']['message'])
else:
display(data['error'])
return callback(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
@@ -172,7 +169,7 @@ def add_command_parser(parent, command):
def get_argument_parser():
main = ArgumentParser(
'lbrynet', description='An interface to the LBRY Network.'
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
)
main.add_argument(
'-v', '--version', dest='cli_version', action="store_true",
@@ -195,8 +192,8 @@ def get_argument_parser():
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
start.set_defaults(command='start', start_parser=start)
Config.contribute_to_argparse(start)
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
api = Daemon.get_api_definitions()
groups = {}
@@ -242,6 +239,7 @@ def main(argv=None):
if args.help:
args.start_parser.print_help()
return 0
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
file_handler = logging.handlers.RotatingFileHandler(
conf.log_file_path, maxBytes=2097152, backupCount=5
@@ -267,6 +265,7 @@ def main(argv=None):
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
+22 -15
View File
@@ -16,7 +16,7 @@ from lbrynet import utils
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.dht.node import Node
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.extras.daemon.Component import Component
@@ -63,7 +63,7 @@ class DatabaseComponent(Component):
@staticmethod
def get_current_db_revision():
return 9
return 11
@property
def revision_filename(self):
@@ -244,13 +244,14 @@ class WalletComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.wallet_manager = None
self._started = False
@property
def component(self):
return self.wallet_manager
async def get_status(self):
if self.wallet_manager and self.running:
if self.wallet_manager and self._started:
local_height = self.wallet_manager.ledger.headers.height
remote_height = await self.wallet_manager.ledger.network.get_server_height()
best_hash = self.wallet_manager.get_best_blockhash()
@@ -265,6 +266,7 @@ class WalletComponent(Component):
async def start(self):
log.info("Starting torba wallet")
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
self._started = True
await self.wallet_manager.start()
async def stop(self):
@@ -278,10 +280,10 @@ class BlobComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_manager: BlobFileManager = None
self.blob_manager: typing.Optional[BlobManager] = None
@property
def component(self) -> typing.Optional[BlobFileManager]:
def component(self) -> typing.Optional[BlobManager]:
return self.blob_manager
async def start(self):
@@ -291,8 +293,10 @@ class BlobComponent(Component):
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
data_store = dht_node.protocol.data_store
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
storage, data_store)
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
if not os.path.isdir(blob_dir):
os.mkdir(blob_dir)
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
return await self.blob_manager.setup()
async def stop(self):
@@ -302,7 +306,10 @@ class BlobComponent(Component):
count = 0
if self.blob_manager:
count = len(self.blob_manager.completed_blob_hashes)
return {'finished_blobs': count}
return {
'finished_blobs': count,
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
}
class DHTComponent(Component):
@@ -403,7 +410,7 @@ class StreamManagerComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.stream_manager: StreamManager = None
self.stream_manager: typing.Optional[StreamManager] = None
@property
def component(self) -> typing.Optional[StreamManager]:
@@ -413,7 +420,7 @@ class StreamManagerComponent(Component):
if not self.stream_manager:
return
return {
'managed_files': len(self.stream_manager.streams)
'managed_files': len(self.stream_manager.streams),
}
async def start(self):
@@ -442,7 +449,7 @@ class PeerProtocolServerComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_server: BlobServer = None
self.blob_server: typing.Optional[BlobServer] = None
@property
def component(self) -> typing.Optional[BlobServer]:
@@ -451,7 +458,7 @@ class PeerProtocolServerComponent(Component):
async def start(self):
log.info("start blob server")
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = self.conf.tcp_port
address = await wallet.get_unused_address()
@@ -485,7 +492,7 @@ class UPnPComponent(Component):
while True:
if now:
await self._maintain_redirects()
await asyncio.sleep(360)
await asyncio.sleep(360, loop=self.component_manager.loop)
async def _maintain_redirects(self):
# setup the gateway if necessary
@@ -508,7 +515,7 @@ class UPnPComponent(Component):
pass
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
external_ip = await utils.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
@@ -528,7 +535,7 @@ class UPnPComponent(Component):
self.upnp_redirects.update(upnp_redirects)
except (asyncio.TimeoutError, UPnPError):
self.upnp = None
return self._maintain_redirects()
return
elif self.upnp: # check existing redirects are still active
found = set()
mappings = await self.upnp.get_redirects()
File diff suppressed because it is too large. Load diff
+3 -24
View File
@@ -1,27 +1,6 @@
import aiohttp
import logging
from lbrynet.conf import Config
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
from lbrynet.extras.cli import execute_command
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
async def daemon_rpc(conf: Config, method: str, *args, **kwargs):
async with aiohttp.ClientSession() as session:
message = {'method': method, 'params': [args, kwargs]}
async with session.get(conf.api_connection_url, json=message) as resp:
data = await resp.json()
if 'result' in data:
return data['result']
elif 'error' in data:
raise JSONRPCException(data['error'])
def daemon_rpc(conf: Config, method: str, **kwargs):
return execute_command(conf, method, kwargs, callback=lambda data: data)
+51
View File
@@ -0,0 +1,51 @@
import logging
import aiohttp
import hashlib
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives.asymmetric import utils
from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key
log = logging.getLogger(__name__)
def sign_comment(**kwargs):
private_key = generate_private_key(
public_exponent=65537,
key_size=4096,
backend=default_backend()
)
chosen_hash = hashes.SHA256()
hasher = hashes.Hash(chosen_hash, default_backend())
value_to_hash = b':'.join(bytes(v, 'utf-8') for v in kwargs.values() if type(v) is str)
hasher.update(value_to_hash)
digest = hasher.finalize()
signature = private_key.sign(
digest,
padding.PSS(
mgf=padding.MGF1(hashes.SHA256()),
salt_length=padding.PSS.MAX_LENGTH
),
utils.Prehashed(chosen_hash)
)
m = hashlib.sha3_256()
m.update(signature)
return m.hexdigest()
def rpc_body(method: str, rpc_id: any, **params) -> dict:
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
async def jsonrpc_post(url: str, method: str, **params) -> any:
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
headers = {'Content-Type': 'application/json'}
async with aiohttp.request('POST', url, json=json_body, headers=headers) as response:
try:
result = await response.json()
return result['result'] if 'result' in result else result
except aiohttp.client.ContentTypeError as cte:
log.exception('Unable to decode respose from server: %s', cte)
return await response.text()
@@ -123,8 +123,8 @@ class LBRYioFeed(MarketFeed):
def __init__(self):
super().__init__(
"BTCLBC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
@@ -140,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
def __init__(self):
super().__init__(
"USDBTC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
+166 -41
View File
@@ -1,38 +1,123 @@
import logging
from decimal import Decimal
from binascii import hexlify
from binascii import hexlify, unhexlify
from datetime import datetime
from json import JSONEncoder
from ecdsa import BadSignatureError
from google.protobuf.message import DecodeError
from lbrynet.schema.claim import Claim
from lbrynet.wallet.ledger import MainNetLedger
from lbrynet.wallet.ledger import MainNetLedger, Account
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.stream.managed_stream import ManagedStream
log = logging.getLogger(__name__)
def encode_txo_doc():
return {
'txid': "hash of transaction in hex",
'nout': "position in the transaction",
'height': "block where transaction was recorded",
'amount': "value of the txo as a decimal",
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'payment'",
'name': "when type is 'claim' or 'support', this is the claim name",
'claim_id': "when type is 'claim' or 'support', this is the claim id",
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
'protobuf': "hex encoded raw protobuf version of 'value' field",
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
'signing_channel': "for signed claims only, metadata of signing channel",
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
}
def encode_tx_doc():
return {
'txid': "hash of transaction in hex",
'height': "block where transaction was recorded",
'inputs': [encode_txo_doc()],
'outputs': [encode_txo_doc()],
'total_input': "sum of inputs as a decimal",
'total_output': "sum of outputs, sans fee, as a decimal",
'total_fee': "fee amount",
'hex': "entire transaction encoded in hex",
}
def encode_account_doc():
return {
'id': 'account_id',
'is_default': 'this account is used by default',
'ledger': 'name of crypto currency and network',
'name': 'optional account name',
'seed': 'human friendly words from which account can be recreated',
'encrypted': 'if account is encrypted',
'private_key': 'extended private key',
'public_key': 'extended public key',
'address_generator': 'settings for generating addresses',
'modified_on': 'date of last modification to account settings'
}
def encode_file_doc():
return {
'streaming_url': '(str) url to stream the file using range requests',
'completed': '(bool) true if download is completed',
'file_name': '(str) name of file',
'download_directory': '(str) download directory',
'points_paid': '(float) credit paid to download file',
'stopped': '(bool) true if download is stopped',
'stream_hash': '(str) stream hash of file',
'stream_name': '(str) stream name',
'suggested_file_name': '(str) suggested file name',
'sd_hash': '(str) sd hash of file',
'download_path': '(str) download path of file',
'mime_type': '(str) mime type of file',
'key': '(str) key attached to file',
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
'total_bytes': '(int) file upper bound size in bytes',
'written_bytes': '(int) written size in bytes',
'blobs_completed': '(int) number of fully downloaded blobs',
'blobs_in_stream': '(int) total blobs on stream',
'blobs_remaining': '(int) total blobs remaining to download',
'status': '(str) downloader status',
'claim_id': '(str) None if claim is not found else the claim id',
'txid': '(str) None if claim is not found else the transaction id',
'nout': '(int) None if claim is not found else the transaction output index',
'outpoint': '(str) None if claim is not found else the tx and output',
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
}
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
self.include_protobuf = include_protobuf
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Account):
return self.encode_account(obj)
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, Claim):
claim_dict = obj.to_dict()
if obj.is_stream:
claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
if 'fee' in claim_dict['stream'] and 'address' in claim_dict['stream']['fee']:
claim_dict['stream']['fee']['address'] = obj.stream.fee.address
elif obj.is_channel:
claim_dict['channel']['public_key'] = obj.channel.public_key
return claim_dict
return self.encode_claim(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -59,54 +144,94 @@ class JSONResponseEncoder(JSONEncoder):
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'height': tx_height,
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_name:
output['type'] = 'claim'
output['claim_op'] = 'create'
elif txo.script.is_update_claim:
output['type'] = 'claim'
output['claim_op'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'payment'
if txo.script.is_claim_involved:
output.update({
'name': txo.claim_name,
'normalized_name': txo.normalized_name,
'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url,
'meta': self.encode_claim_meta(txo.meta)
})
if 'short_url' in output['meta']:
output['short_url'] = output['meta'].pop('short_url')
if 'canonical_url' in output['meta']:
output['canonical_url'] = output['meta'].pop('canonical_url')
if txo.script.is_claim_name or txo.script.is_update_claim:
claim = txo.claim
output['value'] = claim
if claim.is_signed:
output['valid_signature'] = None
if check_signature and txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
try:
output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
except BadSignatureError:
output['valid_signature'] = False
except ValueError:
log.exception(
'txo.id: %s, txo.channel.id:%s, output: %s',
txo.id, txo.channel.id, output
)
output['valid_signature'] = False
if txo.script.is_claim_name:
output['type'] = 'claim'
elif txo.script.is_update_claim:
output['type'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'basic'
try:
output['value'] = txo.claim
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if check_signature and txo.claim.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
output['is_channel_signature_valid'] = False
except DecodeError:
pass
return output
def encode_claim_meta(self, meta):
for key, value in meta.items():
if key.endswith('_amount'):
if isinstance(value, int):
meta[key] = dewies_to_lbc(value)
if meta.get('creation_height', 0) > 0:
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
return meta
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
def encode_account(self, account):
result = account.to_dict()
result['id'] = account.id
result.pop('certificates', None)
result['is_default'] = self.ledger.accounts[0] == account
return result
def encode_file(self, managed_stream):
file = managed_stream.as_dict()
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
file.update({
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
})
return file
def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict()
if 'public_key' in encoded:
encoded['public_key_id'] = self.ledger.public_key_to_address(
unhexlify(encoded['public_key'])
)
return encoded
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
from .migrate7to8 import do_migration
elif current == 8:
from .migrate8to9 import do_migration
elif current == 9:
from .migrate9to10 import do_migration
elif current == 10:
from .migrate10to11 import do_migration
else:
raise Exception("DB migration of version {} to {} is not available".format(current,
current+1))
@@ -0,0 +1,54 @@
import sqlite3
import os
import binascii
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
current_columns = []
for col_info in cursor.execute("pragma table_info('file');").fetchall():
current_columns.append(col_info[1])
if 'content_fee' in current_columns or 'saved_file' in current_columns:
connection.close()
print("already migrated")
return
cursor.execute(
"pragma foreign_keys=off;"
)
cursor.execute("""
create table if not exists new_file (
stream_hash text primary key not null references stream,
file_name text,
download_directory text,
blob_data_rate real not null,
status text not null,
saved_file integer not null,
content_fee text
);
""")
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
saved_file = 0
if download_dir != '{stream}' and file_name != '{stream}':
try:
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
binascii.unhexlify(file_name).decode())):
saved_file = 1
else:
download_dir, file_name = None, None
except Exception:
download_dir, file_name = None, None
else:
download_dir, file_name = None, None
cursor.execute(
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
)
cursor.execute("drop table file")
cursor.execute("alter table new_file rename to file")
connection.commit()
connection.close()
@@ -0,0 +1,20 @@
import sqlite3
import os
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
query = "select stream_hash, sd_hash from main.stream"
for stream_hash, sd_hash in cursor.execute(query).fetchall():
head_blob_hash = cursor.execute(
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
(stream_hash,)
).fetchone()
if not head_blob_hash:
continue
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
connection.commit()
connection.close()
+160 -79
View File
@@ -8,6 +8,7 @@ import time
from torba.client.basedatabase import SQLiteMixin
from lbrynet.conf import Config
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet.wallet.transaction import Transaction
from lbrynet.schema.claim import Claim
from lbrynet.dht.constants import data_expiration
from lbrynet.blob.blob_info import BlobInfo
@@ -54,10 +55,6 @@ class StoredStreamClaim:
def nout(self) -> typing.Optional[int]:
return None if not self.outpoint else int(self.outpoint.split(":")[1])
@property
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.claim else self.claim.claim_dict['stream']['metadata']
def as_dict(self) -> typing.Dict:
return {
"name": self.claim_name,
@@ -118,8 +115,8 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name, *claim_args) in _batched_select(
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
@@ -145,7 +142,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
"key": stream_key,
"stream_name": stream_name, # hex
"suggested_file_name": suggested_file_name, # hex
"claim": claim
"claim": claim,
"saved_file": bool(saved_file),
"content_fee": None if not raw_content_fee else Transaction(
binascii.unhexlify(raw_content_fee)
)
}
)
for claim_name, claim_id in _batched_select(
@@ -157,21 +158,12 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
# add all blobs, except the last one, which is empty
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
@@ -183,6 +175,11 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
# ensure should_announce is set regardless if insert was ignored
transaction.execute(
"update blob set should_announce=1 where blob_hash in (?, ?)",
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
)
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
@@ -195,13 +192,21 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
data_payment_rate: float, status: str) -> int:
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
content_fee: typing.Optional[Transaction]) -> int:
if not file_name and not download_directory:
encoded_file_name, encoded_download_dir = None, None
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
transaction.execute(
"insert or replace into file values (?, ?, ?, ?, ?)",
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?)",
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
)
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
@@ -250,10 +255,12 @@ class SQLiteStorage(SQLiteMixin):
create table if not exists file (
stream_hash text primary key not null references stream,
file_name text not null,
download_directory text not null,
file_name text,
download_directory text,
blob_data_rate real not null,
status text not null
status text not null,
saved_file integer not null,
content_fee text
);
create table if not exists content_claim (
@@ -296,27 +303,28 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # blob functions # # # # # # # # #
def add_completed_blob(self, blob_hash: str, length: int):
def _add_blob(transaction: sqlite3.Connection):
transaction.execute(
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
def _add_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
(blob_hash, length, 0, 0, "pending", 0, 0)
[
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
for blob_hash, length in blob_hashes_and_lengths
]
)
transaction.execute(
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
)
return self.db.run(_add_blob)
if finished:
transaction.executemany(
"update blob set status='finished' where blob.blob_hash=?", [
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
]
)
return await self.db.run(_add_blobs)
def get_blob_status(self, blob_hash: str):
return self.run_and_return_one_or_none(
"select status from blob where blob_hash=?", blob_hash
)
def add_known_blob(self, blob_hash: str, length: int):
return self.db.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
)
def should_announce(self, blob_hash: str):
return self.run_and_return_one_or_none(
"select should_announce from blob where blob_hash=?", blob_hash
@@ -419,6 +427,26 @@ class SQLiteStorage(SQLiteMixin):
}
return self.db.run(_sync_blobs)
def sync_files_to_blobs(self):
def _sync_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
transaction.execute(
"select distinct sb.stream_hash from stream_blob sb "
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
).fetchall()
)
return self.db.run(_sync_blobs)
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
def _set_streaming(transaction: sqlite3.Connection):
transaction.executemany(
"update file set file_name=null, download_directory=null where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
return self.db.run(_set_streaming)
# # # # # # # # # stream functions # # # # # # # # #
async def stream_exists(self, sd_hash: str) -> bool:
@@ -481,20 +509,53 @@ class SQLiteStorage(SQLiteMixin):
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
)
def get_stream_info_for_sd_hash(self, sd_blob_hash):
return self.run_and_return_one_or_none(
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
sd_blob_hash
)
def delete_stream(self, descriptor: 'StreamDescriptor'):
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
# # # # # # # # # file stuff # # # # # # # # #
def save_downloaded_file(self, stream_hash, file_name, download_directory,
data_payment_rate) -> typing.Awaitable[int]:
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float,
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
return self.save_published_file(
stream_hash, file_name, download_directory, data_payment_rate, status="running"
stream_hash, file_name, download_directory, data_payment_rate, status="running",
content_fee=content_fee
)
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
status="finished") -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float,
status: str = "finished",
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
content_fee)
async def update_manually_removed_files_since_last_run(self):
"""
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1"
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
else:
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
)
return await self.db.run(update_manually_removed_files)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
@@ -503,24 +564,45 @@ class SQLiteStorage(SQLiteMixin):
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
stream_hash
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
file_name: typing.Optional[str]):
if not file_name or not download_dir:
encoded_file_name, encoded_download_dir = None, None
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
encoded_download_dir, encoded_file_name, stream_hash,
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
binascii.hexlify(content_fee.raw), stream_hash,
))
async def set_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
stream_hash,
))
async def clear_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
stream_hash,
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
typing.Optional[Transaction]]],
download_directory: str):
def _recover(transaction: sqlite3.Connection):
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
for descriptor, sd_blob in descriptors_and_sds:
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
for descriptor, sd_blob, content_fee in descriptors_and_sds:
content_claim = transaction.execute(
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
).fetchone()
delete_stream(transaction, descriptor) # this will also delete the content claim
store_stream(transaction, sd_blob, descriptor)
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
download_directory, 0.0, 'stopped')
download_directory, 0.0, 'stopped', content_fee=content_fee)
if content_claim:
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
@@ -539,16 +621,18 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # support functions # # # # # # # # #
def save_supports(self, claim_id, supports):
def save_supports(self, claim_id_to_supports: dict):
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
def _save_support(transaction):
transaction.execute("delete from support where claim_id=?", (claim_id,))
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
)
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
for claim_id, supports in claim_id_to_supports.items():
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
)
return self.db.run(_save_support)
def get_supports(self, *claim_ids):
@@ -576,7 +660,7 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # claim functions # # # # # # # # #
async def save_claims(self, claim_infos):
support_callbacks = []
claim_id_to_supports = {}
update_file_callbacks = []
def _save_claims(transaction):
@@ -591,7 +675,7 @@ class SQLiteStorage(SQLiteMixin):
sequence = claim_info['claim_sequence']
certificate_id = claim_info['value'].signing_channel_id
try:
source_hash = claim_info['value'].stream.sd_hash
source_hash = claim_info['value'].stream.source.sd_hash
except (AttributeError, ValueError):
source_hash = None
serialized = binascii.hexlify(claim_info['value'].to_bytes())
@@ -602,7 +686,7 @@ class SQLiteStorage(SQLiteMixin):
# if this response doesn't have support info don't overwrite the existing
# support info
if 'supports' in claim_info:
support_callbacks.append((claim_id, claim_info['supports']))
claim_id_to_supports[claim_id] = claim_info['supports']
if not source_hash:
continue
stream_hash = transaction.execute(
@@ -632,23 +716,20 @@ class SQLiteStorage(SQLiteMixin):
await self.db.run(_save_claims)
if update_file_callbacks:
await asyncio.wait(update_file_callbacks)
if support_callbacks:
await asyncio.wait([
self.save_supports(*args) for args in support_callbacks
])
if claim_id_to_supports:
await self.save_supports(claim_id_to_supports)
def save_claims_for_resolve(self, claim_infos):
to_save = []
to_save = {}
for info in claim_infos:
if 'value' in info:
if info['value']:
to_save.append(info)
to_save[info['claim_id']] = info
else:
if 'certificate' in info and info['certificate']['value']:
to_save.append(info['certificate'])
if 'claim' in info and info['claim']['value']:
to_save.append(info['claim'])
return self.save_claims(to_save)
for key in ('certificate', 'claim'):
if info.get(key, {}).get('value'):
to_save[info[key]['claim_id']] = info[key]
return self.save_claims(to_save.values())
@staticmethod
def _save_content_claim(transaction, claim_outpoint, stream_hash):
@@ -671,7 +752,7 @@ class SQLiteStorage(SQLiteMixin):
if not known_sd_hash:
raise Exception("stream not found")
# check the claim contains the same sd hash
if known_sd_hash[0] != claim.stream.sd_hash:
if known_sd_hash[0] != claim.stream.source.sd_hash:
raise Exception("stream mismatch")
# if there is a current claim associated to the file, check that the new claim is an update to it
+6 -1
View File
@@ -3,18 +3,23 @@ import os
import logging.handlers
from lbrynet import build_type, __version__ as lbrynet_version
from torba import __version__ as torba_version
log = logging.getLogger(__name__)
def get_platform() -> dict:
os_system = platform.system()
if os.environ and 'ANDROID_ARGUMENT' in os.environ:
os_system = 'android'
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"os_system": os_system,
"lbrynet_version": lbrynet_version,
"torba_version": torba_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
+521
View File
@@ -0,0 +1,521 @@
import json
import logging
import os.path
import hashlib
from typing import Tuple, List
from string import ascii_letters
from decimal import Decimal, ROUND_UP
from binascii import hexlify, unhexlify
from google.protobuf.json_format import MessageToDict
from torba.client.hash import Base58
from torba.client.constants import COIN
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.schema.base import Metadata, BaseMessageList
from lbrynet.schema.types.v2.claim_pb2 import (
Fee as FeeMessage,
Location as LocationMessage,
Language as LanguageMessage
)
log = logging.getLogger(__name__)
def calculate_sha384_file_hash(file_path):
sha384 = hashlib.sha384()
with open(file_path, 'rb') as f:
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
sha384.update(chunk)
return sha384.digest()
class Dimmensional(Metadata):
__slots__ = ()
@property
def width(self) -> int:
return self.message.width
@width.setter
def width(self, width: int):
self.message.width = width
@property
def height(self) -> int:
return self.message.height
@height.setter
def height(self, height: int):
self.message.height = height
@property
def dimensions(self) -> Tuple[int, int]:
return self.width, self.height
@dimensions.setter
def dimensions(self, dimensions: Tuple[int, int]):
self.message.width, self.message.height = dimensions
def _extract(self, file_metadata, field):
try:
setattr(self, field, file_metadata.getValues(field)[0])
except:
log.exception(f'Could not extract {field} from file metadata.')
def update(self, file_metadata=None, height=None, width=None):
if height is not None:
self.height = height
elif file_metadata:
self._extract(file_metadata, 'height')
if width is not None:
self.width = width
elif file_metadata:
self._extract(file_metadata, 'width')
class Playable(Metadata):
__slots__ = ()
@property
def duration(self) -> int:
return self.message.duration
@duration.setter
def duration(self, duration: int):
self.message.duration = duration
def update(self, file_metadata=None, duration=None):
if duration is not None:
self.duration = duration
elif file_metadata:
try:
self.duration = file_metadata.getValues('duration')[0].seconds
except:
log.exception('Could not extract duration from file metadata.')
class Image(Dimmensional):
__slots__ = ()
class Audio(Playable):
__slots__ = ()
class Video(Dimmensional, Playable):
__slots__ = ()
def update(self, file_metadata=None, height=None, width=None, duration=None):
Dimmensional.update(self, file_metadata, height, width)
Playable.update(self, file_metadata, duration)
class Source(Metadata):
__slots__ = ()
def update(self, file_path=None):
if file_path is not None:
self.name = os.path.basename(file_path)
self.media_type, stream_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.size = os.path.getsize(file_path)
if self.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
return stream_type
@property
def name(self) -> str:
return self.message.name
@name.setter
def name(self, name: str):
self.message.name = name
@property
def size(self) -> int:
return self.message.size
@size.setter
def size(self, size: int):
self.message.size = size
@property
def media_type(self) -> str:
return self.message.media_type
@media_type.setter
def media_type(self, media_type: str):
self.message.media_type = media_type
@property
def file_hash(self) -> str:
return hexlify(self.message.hash).decode()
@file_hash.setter
def file_hash(self, file_hash: str):
self.message.hash = unhexlify(file_hash.encode())
@property
def file_hash_bytes(self) -> bytes:
return self.message.hash
@file_hash_bytes.setter
def file_hash_bytes(self, file_hash_bytes: bytes):
self.message.hash = file_hash_bytes
@property
def sd_hash(self) -> str:
return hexlify(self.message.sd_hash).decode()
@sd_hash.setter
def sd_hash(self, sd_hash: str):
self.message.sd_hash = unhexlify(sd_hash.encode())
@property
def sd_hash_bytes(self) -> bytes:
return self.message.sd_hash
@sd_hash_bytes.setter
def sd_hash_bytes(self, sd_hash: bytes):
self.message.sd_hash = sd_hash
@property
def url(self) -> str:
return self.message.url
@url.setter
def url(self, url: str):
self.message.url = url
class Fee(Metadata):
__slots__ = ()
def update(self, address: str = None, currency: str = None, amount=None):
if address is not None:
self.address = address
if currency is not None and amount is not None:
currency = currency.lower()
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
setattr(self, currency, Decimal(amount))
@property
def currency(self) -> str:
return FeeMessage.Currency.Name(self.message.currency)
@property
def address(self) -> str:
return Base58.encode(self.message.address)
@address.setter
def address(self, address: str):
self.message.address = Base58.decode(address)
@property
def address_bytes(self) -> bytes:
return self.message.address
@address_bytes.setter
def address_bytes(self, address: bytes):
self.message.address = address
@property
def amount(self) -> Decimal:
if self.currency == 'LBC':
return self.lbc
if self.currency == 'BTC':
return self.btc
if self.currency == 'USD':
return self.usd
DEWIES = Decimal(COIN)
@property
def lbc(self) -> Decimal:
if self.message.currency != FeeMessage.LBC:
raise ValueError('LBC can only be returned for LBC fees.')
return Decimal(self.message.amount / self.DEWIES)
@lbc.setter
def lbc(self, amount: Decimal):
self.dewies = int(amount * self.DEWIES)
@property
def dewies(self) -> int:
if self.message.currency != FeeMessage.LBC:
raise ValueError('Dewies can only be returned for LBC fees.')
return self.message.amount
@dewies.setter
def dewies(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.LBC
SATOSHIES = Decimal(COIN)
@property
def btc(self) -> Decimal:
if self.message.currency != FeeMessage.BTC:
raise ValueError('BTC can only be returned for BTC fees.')
return Decimal(self.message.amount / self.SATOSHIES)
@btc.setter
def btc(self, amount: Decimal):
self.satoshis = int(amount * self.SATOSHIES)
@property
def satoshis(self) -> int:
if self.message.currency != FeeMessage.BTC:
raise ValueError('Satoshies can only be returned for BTC fees.')
return self.message.amount
@satoshis.setter
def satoshis(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.BTC
PENNIES = Decimal('100.0')
PENNY = Decimal('0.01')
@property
def usd(self) -> Decimal:
if self.message.currency != FeeMessage.USD:
raise ValueError('USD can only be returned for USD fees.')
return Decimal(self.message.amount / self.PENNIES)
@usd.setter
def usd(self, amount: Decimal):
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
@property
def pennies(self) -> int:
if self.message.currency != FeeMessage.USD:
raise ValueError('Pennies can only be returned for USD fees.')
return self.message.amount
@pennies.setter
def pennies(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.USD
class ClaimReference(Metadata):
__slots__ = ()
@property
def claim_id(self) -> str:
return hexlify(self.claim_hash[::-1]).decode()
@claim_id.setter
def claim_id(self, claim_id: str):
self.claim_hash = unhexlify(claim_id)[::-1]
@property
def claim_hash(self) -> bytes:
return self.message.claim_hash
@claim_hash.setter
def claim_hash(self, claim_hash: bytes):
self.message.claim_hash = claim_hash
class ClaimList(BaseMessageList[ClaimReference]):
__slots__ = ()
item_class = ClaimReference
@property
def _message(self):
return self.message.claim_references
def append(self, value):
self.add().claim_id = value
@property
def ids(self) -> List[str]:
return [c.claim_id for c in self]
class Language(Metadata):
__slots__ = ()
@property
def langtag(self) -> str:
langtag = []
if self.language:
langtag.append(self.language)
if self.script:
langtag.append(self.script)
if self.region:
langtag.append(self.region)
return '-'.join(langtag)
@langtag.setter
def langtag(self, langtag: str):
parts = langtag.split('-')
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2:
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@property
def language(self) -> str:
if self.message.language:
return LanguageMessage.Language.Name(self.message.language)
@language.setter
def language(self, language: str):
self.message.language = LanguageMessage.Language.Value(language)
@property
def script(self) -> str:
if self.message.script:
return LanguageMessage.Script.Name(self.message.script)
@script.setter
def script(self, script: str):
self.message.script = LanguageMessage.Script.Value(script)
@property
def region(self) -> str:
if self.message.region:
return LocationMessage.Country.Name(self.message.region)
@region.setter
def region(self, region: str):
self.message.region = LocationMessage.Country.Value(region)
class LanguageList(BaseMessageList[Language]):
__slots__ = ()
item_class = Language
def append(self, value: str):
self.add().langtag = value
class Location(Metadata):
__slots__ = ()
def from_value(self, value):
if isinstance(value, str) and value.startswith('{'):
value = json.loads(value)
if isinstance(value, dict):
for key, val in value.items():
setattr(self, key, val)
elif isinstance(value, str):
parts = value.split(':')
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
country = parts and parts.pop(0)
if country:
self.country = country
state = parts and parts.pop(0)
if state:
self.state = state
city = parts and parts.pop(0)
if city:
self.city = city
code = parts and parts.pop(0)
if code:
self.code = code
latitude = parts and parts.pop(0)
if latitude:
self.latitude = latitude
longitude = parts and parts.pop(0)
if longitude:
self.longitude = longitude
else:
raise ValueError(f'Could not parse country value: {value}')
def to_dict(self):
d = MessageToDict(self.message)
if self.message.longitude:
d['longitude'] = self.longitude
if self.message.latitude:
d['latitude'] = self.latitude
return d
@property
def country(self) -> str:
if self.message.country:
return LocationMessage.Country.Name(self.message.country)
@country.setter
def country(self, country: str):
self.message.country = LocationMessage.Country.Value(country)
@property
def state(self) -> str:
return self.message.state
@state.setter
def state(self, state: str):
self.message.state = state
@property
def city(self) -> str:
return self.message.city
@city.setter
def city(self, city: str):
self.message.city = city
@property
def code(self) -> str:
return self.message.code
@code.setter
def code(self, code: str):
self.message.code = code
GPS_PRECISION = Decimal('10000000')
@property
def latitude(self) -> str:
if self.message.latitude:
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
@latitude.setter
def latitude(self, latitude: str):
latitude = Decimal(latitude)
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
self.message.latitude = int(latitude * self.GPS_PRECISION)
@property
def longitude(self) -> str:
if self.message.longitude:
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
@longitude.setter
def longitude(self, longitude: str):
longitude = Decimal(longitude)
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
self.message.longitude = int(longitude * self.GPS_PRECISION)
class LocationList(BaseMessageList[Location]):
__slots__ = ()
item_class = Location
def append(self, value):
self.add().from_value(value)
+50 -3
View File
@@ -1,4 +1,5 @@
from binascii import hexlify, unhexlify
from typing import List, Iterator, TypeVar, Generic
from google.protobuf.message import DecodeError
from google.protobuf.json_format import MessageToDict
@@ -21,9 +22,10 @@ class Signable:
self.unsigned_payload = None
self.signing_channel_hash = None
@property
def is_undetermined(self):
return self.message.WhichOneof('type') is None
def clear_signature(self):
self.signature = None
self.unsigned_payload = None
self.signing_channel_hash = None
@property
def signing_channel_id(self):
@@ -72,3 +74,48 @@ class Signable:
def __bytes__(self):
return self.to_bytes()
class Metadata:
__slots__ = 'message',
def __init__(self, message):
self.message = message
I = TypeVar('I')
class BaseMessageList(Metadata, Generic[I]):
__slots__ = ()
item_class = None
@property
def _message(self):
return self.message
def add(self) -> I:
return self.item_class(self._message.add())
def extend(self, values: List[str]):
for value in values:
self.append(value)
def append(self, value: str):
raise NotImplemented
def __len__(self):
return len(self._message)
def __iter__(self) -> Iterator[I]:
for item in self._message:
yield self.item_class(item)
def __getitem__(self, item) -> I:
return self.item_class(self._message[item])
def __delitem__(self, key):
del self._message[key]
+265 -408
View File
@@ -1,74 +1,85 @@
import os.path
from typing import List, Tuple
from decimal import Decimal
import logging
from typing import List
from binascii import hexlify, unhexlify
from google.protobuf.json_format import MessageToDict
from google.protobuf.message import DecodeError
from hachoir.core.log import log as hachoir_log
from hachoir.parser import createParser as binary_file_parser
from hachoir.metadata import extractMetadata as binary_file_metadata
from hachoir.core.log import log as hachoir_log
from torba.client.hash import Base58
from torba.client.constants import COIN
from lbrynet.schema import compat
from lbrynet.schema.base import Signable
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.schema.types.v2.claim_pb2 import (
Claim as ClaimMessage,
Fee as FeeMessage,
Location as LocationMessage,
Language as LanguageMessage
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
from lbrynet.schema.attrs import (
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
LanguageList, LocationList, ClaimList, ClaimReference
)
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
hachoir_log.use_print = False
log = logging.getLogger(__name__)
class Claim(Signable):
STREAM = 'stream'
CHANNEL = 'channel'
COLLECTION = 'collection'
REPOST = 'repost'
__slots__ = 'version',
message_class = ClaimMessage
def __init__(self, claim_message=None):
super().__init__(claim_message)
def __init__(self, message=None):
super().__init__(message)
self.version = 2
@property
def claim_type(self) -> str:
return self.message.WhichOneof('type')
def get_message(self, type_name):
message = getattr(self.message, type_name)
if self.claim_type is None:
message.SetInParent()
if self.claim_type != type_name:
raise ValueError(f'Claim is not a {type_name}.')
return message
@property
def is_stream(self):
return self.message.WhichOneof('type') == 'stream'
@property
def is_channel(self):
return self.message.WhichOneof('type') == 'channel'
@property
def stream_message(self):
if self.is_undetermined:
self.message.stream.SetInParent()
if not self.is_stream:
raise ValueError('Claim is not a stream.')
return self.message.stream
return self.claim_type == self.STREAM
@property
def stream(self) -> 'Stream':
return Stream(self)
@property
def channel_message(self):
if self.is_undetermined:
self.message.channel.SetInParent()
if not self.is_channel:
raise ValueError('Claim is not a channel.')
return self.message.channel
def is_channel(self):
return self.claim_type == self.CHANNEL
@property
def channel(self) -> 'Channel':
return Channel(self)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
@property
def is_repost(self):
return self.claim_type == self.REPOST
@property
def repost(self) -> 'Repost':
return Repost(self)
@property
def is_collection(self):
return self.claim_type == self.COLLECTION
@property
def collection(self) -> 'Collection':
return Collection(self)
@classmethod
def from_bytes(cls, data: bytes) -> 'Claim':
@@ -87,394 +98,157 @@ class Claim(Signable):
return claim
class Dimmensional:
__slots__ = ()
@property
def width(self) -> int:
return self.message.width
@width.setter
def width(self, width: int):
self.message.width = width
@property
def height(self) -> int:
return self.message.height
@height.setter
def height(self, height: int):
self.message.height = height
@property
def dimensions(self) -> Tuple[int, int]:
return self.width, self.height
@dimensions.setter
def dimensions(self, dimensions: Tuple[int, int]):
self.message.width, self.message.height = dimensions
class Playable:
__slots__ = ()
@property
def duration(self) -> int:
return self.message.duration
@duration.setter
def duration(self, duration: int):
self.message.duration = duration
def set_duration_from_path(self, file_path):
try:
file_metadata = binary_file_metadata(binary_file_parser(file_path))
self.duration = file_metadata.getValues('duration')[0].seconds
except:
pass
class Image(Dimmensional):
__slots__ = 'message',
def __init__(self, image_message):
self.message = image_message
class Video(Dimmensional, Playable):
__slots__ = 'message',
def __init__(self, video_message):
self.message = video_message
class Audio(Playable):
__slots__ = 'message',
def __init__(self, audio_message):
self.message = audio_message
class File:
__slots__ = '_file',
def __init__(self, file_message):
self._file = file_message
@property
def name(self) -> str:
return self._file.name
@name.setter
def name(self, name: str):
self._file.name = name
@property
def size(self) -> int:
return self._file.size
@size.setter
def size(self, size: int):
self._file.size = size
class Fee:
__slots__ = '_fee',
def __init__(self, fee_message):
self._fee = fee_message
@property
def currency(self) -> str:
return FeeMessage.Currency.Name(self._fee.currency)
@property
def address(self) -> str:
return Base58.encode(self._fee.address)
@address.setter
def address(self, address: str):
self._fee.address = Base58.decode(address)
@property
def address_bytes(self) -> bytes:
return self._fee.address
@address_bytes.setter
def address_bytes(self, address: bytes):
self._fee.address = address
@property
def amount(self) -> Decimal:
if self.currency == 'LBC':
return self.lbc
if self.currency == 'USD':
return self.usd
DEWIES = Decimal(COIN)
@property
def lbc(self) -> Decimal:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('LBC can only be returned for LBC fees.')
return Decimal(self._fee.amount / self.DEWIES)
@lbc.setter
def lbc(self, amount: Decimal):
self.dewies = int(amount * self.DEWIES)
@property
def dewies(self) -> int:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('Dewies can only be returned for LBC fees.')
return self._fee.amount
@dewies.setter
def dewies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.LBC
PENNIES = Decimal(100.0)
@property
def usd(self) -> Decimal:
if self._fee.currency != FeeMessage.USD:
raise ValueError('USD can only be returned for USD fees.')
return Decimal(self._fee.amount / self.PENNIES)
@usd.setter
def usd(self, amount: Decimal):
self.pennies = int(amount * self.PENNIES)
@property
def pennies(self) -> int:
if self._fee.currency != FeeMessage.USD:
raise ValueError('Pennies can only be returned for USD fees.')
return self._fee.amount
@pennies.setter
def pennies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.USD
class BaseClaimSubType:
class BaseClaim:
__slots__ = 'claim', 'message'
def __init__(self, claim: Claim):
claim_type = None
object_fields = 'thumbnail',
repeat_fields = 'tags', 'languages', 'locations'
def __init__(self, claim: Claim = None):
self.claim = claim or Claim()
@property
def tags(self) -> List:
return self.message.tags
@property
def title(self) -> str:
return self.message.title
@title.setter
def title(self, title: str):
self.message.title = title
@property
def description(self) -> str:
return self.message.description
@description.setter
def description(self, description: str):
self.message.description = description
@property
def thumbnail_url(self) -> str:
return self.message.thumbnail_url
@thumbnail_url.setter
def thumbnail_url(self, thumbnail_url: str):
self.message.thumbnail_url = thumbnail_url
@property
def language(self) -> str:
if len(self.languages) > 0:
return LanguageMessage.Language.Name(self.languages[0].language)
@language.setter
def language(self, language: str):
value = LanguageMessage.Language.Value(language)
if len(self.languages) > 0:
self.languages[0].language = value
else:
self.languages.add().language = value
@property
def languages(self):
return self.message.languages
@property
def location_country(self) -> str:
if len(self.locations) > 0:
return LocationMessage.Country.Name(self.locations[0].country)
@location_country.setter
def location_country(self, country: str):
value = LocationMessage.Country.Value(country)
if len(self.locations) > 0:
self.locations[0].location = value
else:
self.locations.add().location = value
@property
def locations(self):
return self.message.locations
self.message = self.claim.get_message(self.claim_type)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
claim.update(claim.pop(self.claim_type))
if 'languages' in claim:
claim['languages'] = self.langtags
if 'locations' in claim:
claim['locations'] = [l.to_dict() for l in self.locations]
return claim
def update(self, tags=None, clear_tags=False, **kwargs):
def update(self, **kwargs):
for key in list(kwargs):
for field in self.object_fields:
if key.startswith(f'{field}_'):
attr = getattr(self, field)
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
continue
if clear_tags:
self.message.ClearField('tags')
if tags is not None:
if isinstance(tags, str):
self.tags.append(tags)
elif isinstance(tags, list):
self.tags.extend(tags)
else:
raise ValueError(f"Unknown tag type: {tags}")
for l in self.repeat_fields:
field = getattr(self, l)
if kwargs.pop(f'clear_{l}', False):
del field[:]
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
field.append(items)
elif isinstance(items, list):
field.extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
for key, value in kwargs.items():
setattr(self, key, value)
@property
def title(self) -> str:
return self.claim.message.title
class Channel(BaseClaimSubType):
@title.setter
def title(self, title: str):
self.claim.message.title = title
@property
def description(self) -> str:
return self.claim.message.description
@description.setter
def description(self, description: str):
self.claim.message.description = description
@property
def thumbnail(self) -> Source:
return Source(self.claim.message.thumbnail)
@property
def tags(self) -> List:
return self.claim.message.tags
@property
def languages(self) -> LanguageList:
return LanguageList(self.claim.message.languages)
@property
def langtags(self) -> List[str]:
return [l.langtag for l in self.languages]
@property
def locations(self) -> LocationList:
return LocationList(self.claim.message.locations)
class Stream(BaseClaim):
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.channel_message
claim_type = Claim.STREAM
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
object_fields = BaseClaim.object_fields + ('source',)
@public_key.setter
def public_key(self, sd_public_key: str):
self.message.public_key = unhexlify(sd_public_key.encode())
def to_dict(self):
claim = super().to_dict()
if 'source' in claim:
if 'hash' in claim['source']:
claim['source']['hash'] = self.source.file_hash
if 'sd_hash' in claim['source']:
claim['source']['sd_hash'] = self.source.sd_hash
if 'media_type' in claim['source']:
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
fee = claim.get('fee', {})
if 'address' in fee:
fee['address'] = self.fee.address
if 'amount' in fee:
fee['amount'] = str(self.fee.amount)
return claim
@property
def public_key_bytes(self) -> bytes:
return self.message.public_key
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
if kwargs.pop('clear_fee', False):
self.message.ClearField('fee')
else:
self.fee.update(
kwargs.pop('fee_address', None),
kwargs.pop('fee_currency', None),
kwargs.pop('fee_amount', None)
)
@public_key_bytes.setter
def public_key_bytes(self, public_key: bytes):
self.message.public_key = public_key
if 'sd_hash' in kwargs:
self.source.sd_hash = kwargs.pop('sd_hash')
if 'file_name' in kwargs:
self.source.name = kwargs.pop('file_name')
if 'file_hash' in kwargs:
self.source.file_hash = kwargs.pop('file_hash')
@property
def contact_email(self) -> str:
return self.message.contact_email
stream_type = None
if file_path is not None:
stream_type = self.source.update(file_path=file_path)
elif self.source.name:
self.source.media_type, stream_type = guess_media_type(self.source.name)
elif self.source.media_type:
stream_type = guess_stream_type(self.source.media_type)
@contact_email.setter
def contact_email(self, contact_email: str):
self.message.contact_email = contact_email
if 'file_size' in kwargs:
self.source.size = kwargs.pop('file_size')
@property
def homepage_url(self) -> str:
return self.message.homepage_url
@homepage_url.setter
def homepage_url(self, homepage_url: str):
self.message.homepage_url = homepage_url
@property
def cover_url(self) -> str:
return self.message.cover_url
@cover_url.setter
def cover_url(self, cover_url: str):
self.message.cover_url = cover_url
class Stream(BaseClaimSubType):
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.stream_message
def update(
self, file_path=None, stream_type=None,
fee_currency=None, fee_amount=None, fee_address=None,
**kwargs):
duration_was_not_set = True
sub_types = ('image', 'video', 'audio')
for key in list(kwargs.keys()):
for sub_type in sub_types:
if key.startswith(f'{sub_type}_'):
stream_type = sub_type
sub_obj = getattr(self, sub_type)
sub_obj_attr = key[len(f'{sub_type}_'):]
setattr(sub_obj, sub_obj_attr, kwargs.pop(key))
if sub_obj_attr == 'duration':
duration_was_not_set = False
break
if stream_type is not None:
if stream_type not in sub_types:
raise Exception(
f"stream_type of '{stream_type}' is not valid, must be one of: {sub_types}"
)
sub_obj = getattr(self, stream_type)
if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
sub_obj.set_duration_from_path(file_path)
if stream_type in ('image', 'video', 'audio'):
media = getattr(self, stream_type)
media_args = {'file_metadata': None}
if file_path is not None:
try:
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
except:
log.exception('Could not read file metadata.')
if isinstance(media, Playable):
media_args['duration'] = duration
if isinstance(media, Dimmensional):
media_args['height'] = height
media_args['width'] = width
media.update(**media_args)
super().update(**kwargs)
if file_path is not None:
self.media_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.file.size = os.path.getsize(file_path)
if self.file.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
if fee_amount and fee_currency:
if fee_address:
self.fee.address = fee_address
if fee_currency.lower() == 'lbc':
self.fee.lbc = Decimal(fee_amount)
elif fee_currency.lower() == 'usd':
self.fee.usd = Decimal(fee_amount)
else:
raise Exception(f'Unknown currency type: {fee_currency}')
@property
def sd_hash(self) -> str:
return hexlify(self.message.sd_hash).decode()
@sd_hash.setter
def sd_hash(self, sd_hash: str):
self.message.sd_hash = unhexlify(sd_hash.encode())
@property
def sd_hash_bytes(self) -> bytes:
return self.message.sd_hash
@sd_hash_bytes.setter
def sd_hash_bytes(self, sd_hash: bytes):
self.message.sd_hash = sd_hash
@property
def author(self) -> str:
return self.message.author
@@ -507,14 +281,6 @@ class Stream(BaseClaimSubType):
def release_time(self, release_time: int):
self.message.release_time = release_time
@property
def media_type(self) -> str:
return self.message.media_type
@media_type.setter
def media_type(self, media_type: str):
self.message.media_type = media_type
@property
def fee(self) -> Fee:
return Fee(self.message.fee)
@@ -524,8 +290,12 @@ class Stream(BaseClaimSubType):
return self.message.HasField('fee')
@property
def file(self) -> File:
return File(self.message.file)
def source(self) -> Source:
return Source(self.message.source)
@property
def stream_type(self) -> str:
return self.message.WhichOneof('type')
@property
def image(self) -> Image:
@@ -538,3 +308,90 @@ class Stream(BaseClaimSubType):
@property
def audio(self) -> Audio:
return Audio(self.message.audio)
class Channel(BaseClaim):
__slots__ = ()
claim_type = Claim.CHANNEL
object_fields = BaseClaim.object_fields + ('cover',)
repeat_fields = BaseClaim.repeat_fields + ('featured',)
def to_dict(self):
claim = super().to_dict()
claim['public_key'] = self.public_key
if 'featured' in claim:
claim['featured'] = self.featured.ids
return claim
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
@public_key.setter
def public_key(self, sd_public_key: str):
self.message.public_key = unhexlify(sd_public_key.encode())
@property
def public_key_bytes(self) -> bytes:
return self.message.public_key
@public_key_bytes.setter
def public_key_bytes(self, public_key: bytes):
self.message.public_key = public_key
@property
def email(self) -> str:
return self.message.email
@email.setter
def email(self, email: str):
self.message.email = email
@property
def website_url(self) -> str:
return self.message.website_url
@website_url.setter
def website_url(self, website_url: str):
self.message.website_url = website_url
@property
def cover(self) -> Source:
return Source(self.message.cover)
@property
def featured(self) -> ClaimList:
return ClaimList(self.message.featured)
class Repost(BaseClaim):
__slots__ = ()
claim_type = Claim.REPOST
@property
def reference(self) -> ClaimReference:
return ClaimReference(self.message)
class Collection(BaseClaim):
__slots__ = ()
claim_type = Claim.COLLECTION
repeat_fields = BaseClaim.repeat_fields + ('claims',)
def to_dict(self):
claim = super().to_dict()
if 'claim_references' in claim:
claim['claim_references'] = self.claims.ids
return claim
@property
def claims(self) -> ClaimList:
return ClaimList(self.message)
+29 -19
View File
@@ -1,19 +1,28 @@
import json
from decimal import Decimal
from google.protobuf.message import DecodeError
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
from lbrynet.schema.types.v1.metadata_pb2 import Metadata as MetadataMessage
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
def from_old_json_schema(claim, payload: bytes):
value = json.loads(payload)
try:
value = json.loads(payload)
except:
raise DecodeError('Could not parse JSON.')
stream = claim.stream
stream.media_type = value.get('content_type', value.get('content-type', 'application/octet-stream'))
stream.source.sd_hash = value['sources']['lbry_sd_hash']
stream.source.media_type = (
value.get('content_type', value.get('content-type')) or
'application/octet-stream'
)
stream.title = value.get('title', '')
stream.description = value.get('description', '')
stream.thumbnail_url = value.get('thumbnail', '')
if value.get('thumbnail', ''):
stream.thumbnail.url = value.get('thumbnail', '')
stream.author = value.get('author', '')
stream.license = value.get('license', '')
stream.license_url = value.get('license_url', '')
@@ -22,10 +31,9 @@ def from_old_json_schema(claim, payload: bytes):
if language.lower() == 'english':
language = 'en'
try:
stream.language = language
stream.languages.append(language)
except:
pass
stream.sd_hash = value['sources']['lbry_sd_hash']
if value.get('nsfw', False):
stream.tags.append('mature')
if "fee" in value:
@@ -35,8 +43,10 @@ def from_old_json_schema(claim, payload: bytes):
stream.fee.lbc = Decimal(fee[currency]['amount'])
elif currency == 'USD':
stream.fee.usd = Decimal(fee[currency]['amount'])
elif currency == 'BTC':
stream.fee.btc = Decimal(fee[currency]['amount'])
else:
raise ValueError(f'Unknown currency: {currency}')
raise DecodeError(f'Unknown currency: {currency}')
stream.fee.address = fee[currency]['address']
return claim
@@ -44,18 +54,21 @@ def from_old_json_schema(claim, payload: bytes):
def from_types_v1(claim, payload: bytes):
old = OldClaimMessage()
old.ParseFromString(payload)
if old.claimType == 1:
if old.claimType == 2:
channel = claim.channel
channel.public_key_bytes = old.certificate.publicKey
else:
stream = claim.stream
stream.title = old.stream.metadata.title
stream.description = old.stream.metadata.description
stream.author = old.stream.metadata.author
stream.license = old.stream.metadata.license
stream.license_url = old.stream.metadata.licenseUrl
stream.thumbnail_url = old.stream.metadata.thumbnail
stream.thumbnail.url = old.stream.metadata.thumbnail
if old.stream.metadata.HasField('language'):
stream.languages.add().language = old.stream.metadata.language
stream.media_type = old.stream.source.contentType
stream.sd_hash_bytes = old.stream.source.source
stream.languages.add().message.language = old.stream.metadata.language
stream.source.media_type = old.stream.source.contentType
stream.source.sd_hash_bytes = old.stream.source.source
if old.stream.metadata.nsfw:
stream.tags.append('mature')
if old.stream.metadata.HasField('fee'):
@@ -66,18 +79,15 @@ def from_types_v1(claim, payload: bytes):
stream.fee.lbc = Decimal(fee.amount)
elif currency == 'USD':
stream.fee.usd = Decimal(fee.amount)
elif currency == 'BTC':
stream.fee.btc = Decimal(fee.amount)
else:
raise ValueError(f'Unsupported currency: {currency}')
raise DecodeError(f'Unsupported currency: {currency}')
if old.HasField('publisherSignature'):
sig = old.publisherSignature
claim.signature = sig.signature
claim.signature_type = KeyType.Name(sig.signatureType)
claim.signing_channel_hash = sig.certificateId
claim.signing_channel_hash = sig.certificateId[::-1]
old.ClearField("publisherSignature")
claim.unsigned_payload = old.SerializeToString()
elif old.claimType == 2:
channel = claim.channel
channel.public_key_bytes = old.certificate.publicKey
else:
raise ValueError('claimType must be 1 for Streams and 2 for Channel')
return claim
+162 -147
View File
@@ -1,160 +1,175 @@
import os
types_map = {
# http://www.iana.org/assignments/media-types
'.a': 'application/octet-stream',
'.ai': 'application/postscript',
'.aif': 'audio/x-aiff',
'.aifc': 'audio/x-aiff',
'.aiff': 'audio/x-aiff',
'.au': 'audio/basic',
'.avi': 'video/x-msvideo',
'.bat': 'text/plain',
'.bcpio': 'application/x-bcpio',
'.bin': 'application/octet-stream',
'.bmp': 'image/bmp',
'.c': 'text/plain',
'.cdf': 'application/x-netcdf',
'.cpio': 'application/x-cpio',
'.csh': 'application/x-csh',
'.css': 'text/css',
'.csv': 'text/csv',
'.dll': 'application/octet-stream',
'.doc': 'application/msword',
'.dot': 'application/msword',
'.dvi': 'application/x-dvi',
'.eml': 'message/rfc822',
'.eps': 'application/postscript',
'.epub': 'application/epub+zip',
'.etx': 'text/x-setext',
'.exe': 'application/octet-stream',
'.gif': 'image/gif',
'.gtar': 'application/x-gtar',
'.h': 'text/plain',
'.hdf': 'application/x-hdf',
'.htm': 'text/html',
'.html': 'text/html',
'.ico': 'image/vnd.microsoft.icon',
'.ief': 'image/ief',
'.iges': 'model/iges',
'.jpe': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.jpg': 'image/jpeg',
'.js': 'application/javascript',
'.json': 'application/json',
'.ksh': 'text/plain',
'.latex': 'application/x-latex',
'.m1v': 'video/mpeg',
'.m3u': 'application/vnd.apple.mpegurl',
'.m3u8': 'application/vnd.apple.mpegurl',
'.man': 'application/x-troff-man',
'.markdown': 'text/markdown',
'.md': 'text/markdown',
'.me': 'application/x-troff-me',
'.mht': 'message/rfc822',
'.mhtml': 'message/rfc822',
'.mif': 'application/x-mif',
'.mov': 'video/quicktime',
'.movie': 'video/x-sgi-movie',
'.mp2': 'audio/mpeg',
'.mp3': 'audio/mpeg',
'.mp4': 'video/mp4',
'.mpa': 'video/mpeg',
'.mpe': 'video/mpeg',
'.mpeg': 'video/mpeg',
'.mpg': 'video/mpeg',
'.ms': 'application/x-troff-ms',
'.nc': 'application/x-netcdf',
'.nws': 'message/rfc822',
'.o': 'application/octet-stream',
'.obj': 'application/octet-stream',
'.oda': 'application/oda',
'.p12': 'application/x-pkcs12',
'.p7c': 'application/pkcs7-mime',
'.pbm': 'image/x-portable-bitmap',
'.pdf': 'application/pdf',
'.pfx': 'application/x-pkcs12',
'.pgm': 'image/x-portable-graymap',
'.pl': 'text/plain',
'.png': 'image/png',
'.pnm': 'image/x-portable-anymap',
'.pot': 'application/vnd.ms-powerpoint',
'.ppa': 'application/vnd.ms-powerpoint',
'.ppm': 'image/x-portable-pixmap',
'.pps': 'application/vnd.ms-powerpoint',
'.ppt': 'application/vnd.ms-powerpoint',
'.ps': 'application/postscript',
'.pwz': 'application/vnd.ms-powerpoint',
'.py': 'text/x-python',
'.pyc': 'application/x-python-code',
'.pyo': 'application/x-python-code',
'.qt': 'video/quicktime',
'.ra': 'audio/x-pn-realaudio',
'.ram': 'application/x-pn-realaudio',
'.ras': 'image/x-cmu-raster',
'.rdf': 'application/xml',
'.rgb': 'image/x-rgb',
'.roff': 'application/x-troff',
'.rtx': 'text/richtext',
'.sgm': 'text/x-sgml',
'.sgml': 'text/x-sgml',
'.sh': 'application/x-sh',
'.shar': 'application/x-shar',
'.snd': 'audio/basic',
'.so': 'application/octet-stream',
'.src': 'application/x-wais-source',
'.stl': 'model/stl',
'.sv4cpio': 'application/x-sv4cpio',
'.sv4crc': 'application/x-sv4crc',
'.svg': 'image/svg+xml',
'.swf': 'application/x-shockwave-flash',
'.t': 'application/x-troff',
'.tar': 'application/x-tar',
'.tcl': 'application/x-tcl',
'.tex': 'application/x-tex',
'.texi': 'application/x-texinfo',
'.texinfo': 'application/x-texinfo',
'.tif': 'image/tiff',
'.tiff': 'image/tiff',
'.tr': 'application/x-troff',
'.tsv': 'text/tab-separated-values',
'.txt': 'text/plain',
'.ustar': 'application/x-ustar',
'.vcf': 'text/x-vcard',
'.wav': 'audio/x-wav',
'.webm': 'video/webm',
'.wiz': 'application/msword',
'.wsdl': 'application/xml',
'.xbm': 'image/x-xbitmap',
'.xlb': 'application/vnd.ms-excel',
'.xls': 'application/vnd.ms-excel',
'.xml': 'text/xml',
'.xpdl': 'application/xml',
'.xpm': 'image/x-xpixmap',
'.xsl': 'application/xml',
'.xwd': 'image/x-xwindowdump',
'.zip': 'application/zip',
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
'.a': ('application/octet-stream', 'binary'),
'.ai': ('application/postscript', 'image'),
'.aif': ('audio/x-aiff', 'audio'),
'.aifc': ('audio/x-aiff', 'audio'),
'.aiff': ('audio/x-aiff', 'audio'),
'.au': ('audio/basic', 'audio'),
'.avi': ('video/x-msvideo', 'video'),
'.bat': ('text/plain', 'document'),
'.bcpio': ('application/x-bcpio', 'binary'),
'.bin': ('application/octet-stream', 'binary'),
'.bmp': ('image/bmp', 'image'),
'.c': ('text/plain', 'document'),
'.cdf': ('application/x-netcdf', 'binary'),
'.cpio': ('application/x-cpio', 'binary'),
'.csh': ('application/x-csh', 'binary'),
'.css': ('text/css', 'document'),
'.csv': ('text/csv', 'document'),
'.dll': ('application/octet-stream', 'binary'),
'.doc': ('application/msword', 'document'),
'.dot': ('application/msword', 'document'),
'.dvi': ('application/x-dvi', 'binary'),
'.eml': ('message/rfc822', 'document'),
'.eps': ('application/postscript', 'document'),
'.epub': ('application/epub+zip', 'document'),
'.etx': ('text/x-setext', 'document'),
'.exe': ('application/octet-stream', 'binary'),
'.gif': ('image/gif', 'image'),
'.gtar': ('application/x-gtar', 'binary'),
'.h': ('text/plain', 'document'),
'.hdf': ('application/x-hdf', 'binary'),
'.htm': ('text/html', 'document'),
'.html': ('text/html', 'document'),
'.ico': ('image/vnd.microsoft.icon', 'image'),
'.ief': ('image/ief', 'image'),
'.iges': ('model/iges', 'model'),
'.jpe': ('image/jpeg', 'image'),
'.jpeg': ('image/jpeg', 'image'),
'.jpg': ('image/jpeg', 'image'),
'.js': ('application/javascript', 'document'),
'.json': ('application/json', 'document'),
'.ksh': ('text/plain', 'document'),
'.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
'.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'),
'.me': ('application/x-troff-me', 'binary'),
'.mht': ('message/rfc822', 'document'),
'.mhtml': ('message/rfc822', 'document'),
'.mif': ('application/x-mif', 'binary'),
'.mov': ('video/quicktime', 'video'),
'.movie': ('video/x-sgi-movie', 'video'),
'.mp2': ('audio/mpeg', 'audio'),
'.mp3': ('audio/mpeg', 'audio'),
'.mp4': ('video/mp4', 'video'),
'.mpa': ('video/mpeg', 'video'),
'.mpe': ('video/mpeg', 'video'),
'.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'),
'.ms': ('application/x-troff-ms', 'binary'),
'.nc': ('application/x-netcdf', 'binary'),
'.nws': ('message/rfc822', 'document'),
'.o': ('application/octet-stream', 'binary'),
'.obj': ('application/octet-stream', 'model'),
'.oda': ('application/oda', 'binary'),
'.p12': ('application/x-pkcs12', 'binary'),
'.p7c': ('application/pkcs7-mime', 'binary'),
'.pbm': ('image/x-portable-bitmap', 'image'),
'.pdf': ('application/pdf', 'document'),
'.pfx': ('application/x-pkcs12', 'binary'),
'.pgm': ('image/x-portable-graymap', 'image'),
'.pl': ('text/plain', 'document'),
'.png': ('image/png', 'image'),
'.pnm': ('image/x-portable-anymap', 'image'),
'.pot': ('application/vnd.ms-powerpoint', 'document'),
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
'.ppm': ('image/x-portable-pixmap', 'image'),
'.pps': ('application/vnd.ms-powerpoint', 'document'),
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
'.ps': ('application/postscript', 'document'),
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
'.py': ('text/x-python', 'document'),
'.pyc': ('application/x-python-code', 'binary'),
'.pyo': ('application/x-python-code', 'binary'),
'.qt': ('video/quicktime', 'video'),
'.ra': ('audio/x-pn-realaudio', 'audio'),
'.ram': ('application/x-pn-realaudio', 'audio'),
'.ras': ('image/x-cmu-raster', 'image'),
'.rdf': ('application/xml', 'binary'),
'.rgb': ('image/x-rgb', 'image'),
'.roff': ('application/x-troff', 'binary'),
'.rtx': ('text/richtext', 'document'),
'.sgm': ('text/x-sgml', 'document'),
'.sgml': ('text/x-sgml', 'document'),
'.sh': ('application/x-sh', 'document'),
'.shar': ('application/x-shar', 'binary'),
'.snd': ('audio/basic', 'audio'),
'.so': ('application/octet-stream', 'binary'),
'.src': ('application/x-wais-source', 'binary'),
'.stl': ('model/stl', 'model'),
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
'.sv4crc': ('application/x-sv4crc', 'binary'),
'.svg': ('image/svg+xml', 'image'),
'.swf': ('application/x-shockwave-flash', 'binary'),
'.t': ('application/x-troff', 'binary'),
'.tar': ('application/x-tar', 'binary'),
'.tcl': ('application/x-tcl', 'binary'),
'.tex': ('application/x-tex', 'binary'),
'.texi': ('application/x-texinfo', 'binary'),
'.texinfo': ('application/x-texinfo', 'binary'),
'.tif': ('image/tiff', 'image'),
'.tiff': ('image/tiff', 'image'),
'.tr': ('application/x-troff', 'binary'),
'.tsv': ('text/tab-separated-values', 'document'),
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
'.wsdl': ('application/xml', 'document'),
'.xbm': ('image/x-xbitmap', 'image'),
'.xlb': ('application/vnd.ms-excel', 'document'),
'.xls': ('application/vnd.ms-excel', 'document'),
'.xml': ('text/xml', 'document'),
'.xpdl': ('application/xml', 'document'),
'.xpm': ('image/x-xpixmap', 'image'),
'.xsl': ('application/xml', 'document'),
'.xwd': ('image/x-xwindowdump', 'image'),
'.zip': ('application/zip', 'binary'),
# These are non-standard types, commonly found in the wild.
'.mid': 'audio/midi',
'.midi': 'audio/midi',
'.pct': 'image/pict',
'.pic': 'image/pict',
'.pict': 'image/pict',
'.rtf': 'application/rtf',
'.xul': 'text/xul',
'.m4v': 'video/m4v',
'.cbr': ('application/vnd.comicbook-rar', 'document'),
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
'.mkv': ('video/x-matroska', 'video'),
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
'.prc': ('application/x-mobipocket-ebook', 'document'),
'.rtf': ('application/rtf', 'document'),
'.xul': ('text/xul', 'document'),
# microsoft is special and has its own 'standard'
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
'.wmv': 'video/x-ms-wmv'
'.wmv': ('video/x-ms-wmv', 'video')
}
def guess_media_type(path):
_, ext = os.path.splitext(path)
return types_map.get(
ext.strip().lower(),
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
)
extension = ext.strip().lower()
if extension[1:]:
if extension in types_map:
return types_map[extension]
return f'application/x-ext-{extension[1:]}', 'binary'
return 'application/octet-stream', 'binary'
def guess_stream_type(media_type):
for media, stream in types_map.values():
if media == media_type:
return stream
return 'binary'
+117
View File
@@ -0,0 +1,117 @@
import base64
import struct
from typing import List
from binascii import hexlify
from itertools import chain
from lbrynet.schema.types.v2.result_pb2 import Outputs as OutputsMessage
class Outputs:
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total'
def __init__(self, txos: List, extra_txos: List, txs: set, offset: int, total: int):
self.txos = txos
self.txs = txs
self.extra_txos = extra_txos
self.offset = offset
self.total = total
def inflate(self, txs):
tx_map = {tx.hash: tx for tx in txs}
for txo_message in self.extra_txos:
self.message_to_txo(txo_message, tx_map)
return [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
def message_to_txo(self, txo_message, tx_map):
if txo_message.WhichOneof('meta') == 'error':
return None
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
if txo_message.WhichOneof('meta') == 'claim':
claim = txo_message.claim
txo.meta = {
'short_url': f'lbry://{claim.short_url}',
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
'is_controlling': claim.is_controlling,
'creation_height': claim.creation_height,
'activation_height': claim.activation_height,
'expiration_height': claim.expiration_height,
'effective_amount': claim.effective_amount,
'support_amount': claim.support_amount,
'trending_group': claim.trending_group,
'trending_mixed': claim.trending_mixed,
'trending_local': claim.trending_local,
'trending_global': claim.trending_global,
}
if claim.HasField('channel'):
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
try:
if txo.claim.is_channel:
txo.meta['claims_in_channel'] = claim.claims_in_channel
except:
pass
return txo
@classmethod
def from_base64(cls, data: str) -> 'Outputs':
return cls.from_bytes(base64.b64decode(data))
@classmethod
def from_bytes(cls, data: bytes) -> 'Outputs':
outputs = OutputsMessage()
outputs.ParseFromString(data)
txs = set()
for txo_message in chain(outputs.txos, outputs.extra_txos):
if txo_message.WhichOneof('meta') == 'error':
continue
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
return cls(outputs.txos, outputs.extra_txos, txs, outputs.offset, outputs.total)
@classmethod
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> str:
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total)).decode()
@classmethod
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> bytes:
page = OutputsMessage()
page.offset = offset
page.total = total or len(txo_rows)
for row in txo_rows:
cls.row_to_message(row, page.txos.add())
for row in extra_txo_rows:
cls.row_to_message(row, page.extra_txos.add())
return page.SerializeToString()
@classmethod
def row_to_message(cls, txo, txo_message):
if isinstance(txo, Exception):
txo_message.error.text = txo.args[0]
if isinstance(txo, ValueError):
txo_message.error.code = txo_message.error.INVALID
elif isinstance(txo, LookupError):
txo_message.error.code = txo_message.error.NOT_FOUND
return
txo_message.tx_hash = txo['txo_hash'][:32]
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
txo_message.height = txo['height']
txo_message.claim.short_url = txo['short_url']
if txo['canonical_url'] is not None:
txo_message.claim.canonical_url = txo['canonical_url']
txo_message.claim.is_controlling = bool(txo['is_controlling'])
txo_message.claim.creation_height = txo['creation_height']
txo_message.claim.activation_height = txo['activation_height']
txo_message.claim.expiration_height = txo['expiration_height']
if txo['claims_in_channel'] is not None:
txo_message.claim.claims_in_channel = txo['claims_in_channel']
txo_message.claim.effective_amount = txo['effective_amount']
txo_message.claim.support_amount = txo['support_amount']
txo_message.claim.trending_group = txo['trending_group']
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
if txo['channel_txo_hash']:
channel = txo_message.claim.channel
channel.tx_hash = txo['channel_txo_hash'][:32]
channel.nout, = struct.unpack('<I', txo['channel_txo_hash'][32:])
channel.height = txo['channel_height']
+474 -258
View File
@@ -19,11 +19,33 @@ DESCRIPTOR = _descriptor.FileDescriptor(
package='pb',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"M\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x42\x06\n\x04type\"\xe8\x01\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x15\n\rcontact_email\x18\x08 \x01(\t\x12\x14\n\x0chomepage_url\x18\t \x01(\t\x12\x11\n\tcover_url\x18\n \x01(\t\"\x8e\x03\n\x06Stream\x12\x0f\n\x07sd_hash\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x0e\n\x06\x61uthor\x18\x08 \x01(\t\x12\x0f\n\x07license\x18\t \x01(\t\x12\x13\n\x0blicense_url\x18\n \x01(\t\x12\x14\n\x0crelease_time\x18\x0b \x01(\x03\x12\x12\n\nmedia_type\x18\r \x01(\t\x12\x16\n\x04\x66ile\x18\x0e \x01(\x0b\x32\x08.pb.File\x12\x14\n\x03\x66\x65\x65\x18\x0f \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\x10 \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x11 \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x12 \x01(\x0b\x32\t.pb.AudioH\x00\x42\x06\n\x04type\"h\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\"\x1c\n\x08\x43urrency\x12\x07\n\x03LBC\x10\x00\x12\x07\n\x03USD\x10\x02\"0\n\x04\x46ile\x12\x0c\n\x04hash\x18\x03 \x01(\x0c\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"8\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\Line truncated
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"\xab\x02\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x12#\n\ncollection\x18\x03 \x01(\x0b\x32\r.pb.ClaimListH\x00\x12$\n\x06repost\x18\x04 \x01(\x0b\x32\x12.pb.ClaimReferenceH\x00\x12\r\n\x05title\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x1d\n\tthumbnail\x18\n \x01(\x0b\x32\n.pb.Source\x12\x0c\n\x04tags\x18\x0b \x03(\t\x12\x1f\n\tlanguages\x18\x0c \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\r \x03(\x0b\x32\x0c.pb.LocationB\x06\n\x04type\"\x84\x02\n\x06Stream\x12\x1a\n\x06source\x18\x01 \x01(\x0b\x32\n.pb.Source\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\x0f\n\x07license\x18\x03 \x01(\t\x12\x13\n\x0blicense_url\x18\x04 \x01(\t\x12\x14\n\x0crelease_time\x18\x05 \x01(\x03\x12\x14\n\x03\x66\x65\x65\x18\x06 \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\n \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x0b \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x0c \x01(\x0b\x32\t.pb.AudioH\x00\x12 \n\x08software\x18\r \x01(\x0b\x32\x0c.pb.SoftwareH\x00\x42\x06\n\x04type\"}\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x13\n\x0bwebsite_url\x18\x03 \x01(\t\x12\x19\n\x05\x63over\x18\x04 \x01(\x0b\x32\n.pb.Source\x12\x1f\n\x08\x66\x65\x61tured\x18\x05 \x01(\x0b\x32\r.pb.ClaimList\"$\n\x0e\x43laimReference\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\"\x90\x01\n\tClaimList\x12)\n\tlist_type\x18\x01 \x01(\x0e\x32\x16.pb.ClaimList.ListType\x12,\n\x10\x63laim_references\x18\x02 \x03(\x0b\x32\x12.pb.ClaimReference\"*\n\x08ListType\x12\x0e\n\nCOLLECTION\x10\x00\x12\x0e\n\nDERIVATION\x10\x02\"d\n\x06Source\x12\x0c\n\x04hash\x18\x01 \x01(\x0c\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x12\n\nmedia_type\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07sd_hash\x18\x06 \x01(\x0c\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"R\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\x12\x18\n\x05\x61udio\x18\x0f \x01(\x0b\x32\t.pb.Audio\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"l\n\x08Software\x12\n\n\x02os\x18\x01 \x01(\t\"T\n\x02OS\x12\x0e\n\nUNKNOWN_OS\x10\x00\x12\x07\n\x03\x41NY\x10\x01\x12\t\n\x05LINUX\x10\x02\x12\x0b\n\x07WINDOWS\x10\x03\x12\x07\n\x03MAC\x10\x04\x12\x0b\n\x07\x41NDROID\x10\x05\x12\x07\n\x03IOS\x10\x06\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x0Line truncated
)
_CLAIMLIST_LISTTYPE = _descriptor.EnumDescriptor(
name='ListType',
full_name='pb.ClaimList.ListType',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='COLLECTION', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DERIVATION', index=1, number=2,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=852,
serialized_end=894,
)
_sym_db.RegisterEnumDescriptor(_CLAIMLIST_LISTTYPE)
_FEE_CURRENCY = _descriptor.EnumDescriptor(
name='Currency',
full_name='pb.Fee.Currency',
@@ -31,21 +53,71 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='LBC', index=0, number=0,
name='UNKNOWN_CURRENCY', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='USD', index=1, number=2,
name='LBC', index=1, number=1,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='BTC', index=2, number=2,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='USD', index=3, number=3,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=810,
serialized_end=838,
serialized_start=1075,
serialized_end=1134,
)
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
_SOFTWARE_OS = _descriptor.EnumDescriptor(
name='OS',
full_name='pb.Software.OS',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='UNKNOWN_OS', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='ANY', index=1, number=1,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='LINUX', index=2, number=2,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='WINDOWS', index=3, number=3,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='MAC', index=4, number=4,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='ANDROID', index=5, number=5,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='IOS', index=6, number=6,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=1311,
serialized_end=1395,
)
_sym_db.RegisterEnumDescriptor(_SOFTWARE_OS)
_LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
name='Language',
full_name='pb.Language.Language',
@@ -795,8 +867,8 @@ _LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=1145,
serialized_end=2706,
serialized_start=1527,
serialized_end=3088,
)
_sym_db.RegisterEnumDescriptor(_LANGUAGE_LANGUAGE)
@@ -1609,8 +1681,8 @@ _LANGUAGE_SCRIPT = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=2709,
serialized_end=4799,
serialized_start=3091,
serialized_end=5181,
)
_sym_db.RegisterEnumDescriptor(_LANGUAGE_SCRIPT)
@@ -2623,8 +2695,8 @@ _LOCATION_COUNTRY = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=4934,
serialized_end=7078,
serialized_start=5316,
serialized_end=7460,
)
_sym_db.RegisterEnumDescriptor(_LOCATION_COUNTRY)
@@ -2650,6 +2722,62 @@ _CLAIM = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='collection', full_name='pb.Claim.collection', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='repost', full_name='pb.Claim.repost', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Claim.title', index=4,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Claim.description', index=5,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail', full_name='pb.Claim.thumbnail', index=6,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Claim.tags', index=7,
number=11, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Claim.languages', index=8,
number=12, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Claim.locations', index=9,
number=13, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -2665,102 +2793,8 @@ _CLAIM = _descriptor.Descriptor(
name='type', full_name='pb.Claim.type',
index=0, containing_type=None, fields=[]),
],
serialized_start=19,
serialized_end=96,
)
_CHANNEL = _descriptor.Descriptor(
name='Channel',
full_name='pb.Channel',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='public_key', full_name='pb.Channel.public_key', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Channel.title', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Channel.description', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail_url', full_name='pb.Channel.thumbnail_url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Channel.tags', index=4,
number=5, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Channel.languages', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Channel.locations', index=6,
number=7, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='contact_email', full_name='pb.Channel.contact_email', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='homepage_url', full_name='pb.Channel.homepage_url', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cover_url', full_name='pb.Channel.cover_url', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=99,
serialized_end=331,
serialized_start=20,
serialized_end=319,
)
@@ -2772,120 +2806,71 @@ _STREAM = _descriptor.Descriptor(
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='sd_hash', full_name='pb.Stream.sd_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
name='source', full_name='pb.Stream.source', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Stream.title', index=1,
name='author', full_name='pb.Stream.author', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Stream.description', index=2,
name='license', full_name='pb.Stream.license', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail_url', full_name='pb.Stream.thumbnail_url', index=3,
name='license_url', full_name='pb.Stream.license_url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Stream.tags', index=4,
number=5, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Stream.languages', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Stream.locations', index=6,
number=7, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='author', full_name='pb.Stream.author', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license', full_name='pb.Stream.license', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license_url', full_name='pb.Stream.license_url', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='release_time', full_name='pb.Stream.release_time', index=10,
number=11, type=3, cpp_type=2, label=1,
name='release_time', full_name='pb.Stream.release_time', index=4,
number=5, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='media_type', full_name='pb.Stream.media_type', index=11,
number=13, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file', full_name='pb.Stream.file', index=12,
number=14, type=11, cpp_type=10, label=1,
name='fee', full_name='pb.Stream.fee', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='fee', full_name='pb.Stream.fee', index=13,
number=15, type=11, cpp_type=10, label=1,
name='image', full_name='pb.Stream.image', index=6,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='image', full_name='pb.Stream.image', index=14,
number=16, type=11, cpp_type=10, label=1,
name='video', full_name='pb.Stream.video', index=7,
number=11, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='video', full_name='pb.Stream.video', index=15,
number=17, type=11, cpp_type=10, label=1,
name='audio', full_name='pb.Stream.audio', index=8,
number=12, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='audio', full_name='pb.Stream.audio', index=16,
number=18, type=11, cpp_type=10, label=1,
name='software', full_name='pb.Stream.software', index=9,
number=13, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@@ -2905,8 +2890,203 @@ _STREAM = _descriptor.Descriptor(
name='type', full_name='pb.Stream.type',
index=0, containing_type=None, fields=[]),
],
serialized_start=334,
serialized_end=732,
serialized_start=322,
serialized_end=582,
)
_CHANNEL = _descriptor.Descriptor(
name='Channel',
full_name='pb.Channel',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='public_key', full_name='pb.Channel.public_key', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='email', full_name='pb.Channel.email', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='website_url', full_name='pb.Channel.website_url', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cover', full_name='pb.Channel.cover', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='featured', full_name='pb.Channel.featured', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=584,
serialized_end=709,
)
_CLAIMREFERENCE = _descriptor.Descriptor(
name='ClaimReference',
full_name='pb.ClaimReference',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='claim_hash', full_name='pb.ClaimReference.claim_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=711,
serialized_end=747,
)
_CLAIMLIST = _descriptor.Descriptor(
name='ClaimList',
full_name='pb.ClaimList',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='list_type', full_name='pb.ClaimList.list_type', index=0,
number=1, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claim_references', full_name='pb.ClaimList.claim_references', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_CLAIMLIST_LISTTYPE,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=750,
serialized_end=894,
)
_SOURCE = _descriptor.Descriptor(
name='Source',
full_name='pb.Source',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hash', full_name='pb.Source.hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='pb.Source.name', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='size', full_name='pb.Source.size', index=2,
number=3, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='media_type', full_name='pb.Source.media_type', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='url', full_name='pb.Source.url', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sd_hash', full_name='pb.Source.sd_hash', index=5,
number=6, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=896,
serialized_end=996,
)
@@ -2951,53 +3131,8 @@ _FEE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=734,
serialized_end=838,
)
_FILE = _descriptor.Descriptor(
name='File',
full_name='pb.File',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hash', full_name='pb.File.hash', index=0,
number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='pb.File.name', index=1,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='size', full_name='pb.File.size', index=2,
number=2, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=840,
serialized_end=888,
serialized_start=999,
serialized_end=1134,
)
@@ -3034,8 +3169,8 @@ _IMAGE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=890,
serialized_end=928,
serialized_start=1136,
serialized_end=1174,
)
@@ -3067,6 +3202,13 @@ _VIDEO = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='audio', full_name='pb.Video.audio', index=3,
number=15, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -3079,8 +3221,8 @@ _VIDEO = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=930,
serialized_end=986,
serialized_start=1176,
serialized_end=1258,
)
@@ -3110,8 +3252,40 @@ _AUDIO = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=988,
serialized_end=1013,
serialized_start=1260,
serialized_end=1285,
)
_SOFTWARE = _descriptor.Descriptor(
name='Software',
full_name='pb.Software',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='os', full_name='pb.Software.os', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_SOFTWARE_OS,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=1287,
serialized_end=1395,
)
@@ -3157,8 +3331,8 @@ _LANGUAGE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1016,
serialized_end=4799,
serialized_start=1398,
serialized_end=5181,
)
@@ -3199,15 +3373,15 @@ _LOCATION = _descriptor.Descriptor(
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='latitude', full_name='pb.Location.latitude', index=4,
number=5, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
number=5, type=17, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='longitude', full_name='pb.Location.longitude', index=5,
number=6, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
number=6, type=17, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
@@ -3224,27 +3398,35 @@ _LOCATION = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=4802,
serialized_end=7078,
serialized_start=5184,
serialized_end=7460,
)
_CLAIM.fields_by_name['stream'].message_type = _STREAM
_CLAIM.fields_by_name['channel'].message_type = _CHANNEL
_CLAIM.fields_by_name['collection'].message_type = _CLAIMLIST
_CLAIM.fields_by_name['repost'].message_type = _CLAIMREFERENCE
_CLAIM.fields_by_name['thumbnail'].message_type = _SOURCE
_CLAIM.fields_by_name['languages'].message_type = _LANGUAGE
_CLAIM.fields_by_name['locations'].message_type = _LOCATION
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['stream'])
_CLAIM.fields_by_name['stream'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['channel'])
_CLAIM.fields_by_name['channel'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CHANNEL.fields_by_name['languages'].message_type = _LANGUAGE
_CHANNEL.fields_by_name['locations'].message_type = _LOCATION
_STREAM.fields_by_name['languages'].message_type = _LANGUAGE
_STREAM.fields_by_name['locations'].message_type = _LOCATION
_STREAM.fields_by_name['file'].message_type = _FILE
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['collection'])
_CLAIM.fields_by_name['collection'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['repost'])
_CLAIM.fields_by_name['repost'].containing_oneof = _CLAIM.oneofs_by_name['type']
_STREAM.fields_by_name['source'].message_type = _SOURCE
_STREAM.fields_by_name['fee'].message_type = _FEE
_STREAM.fields_by_name['image'].message_type = _IMAGE
_STREAM.fields_by_name['video'].message_type = _VIDEO
_STREAM.fields_by_name['audio'].message_type = _AUDIO
_STREAM.fields_by_name['software'].message_type = _SOFTWARE
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['image'])
_STREAM.fields_by_name['image'].containing_oneof = _STREAM.oneofs_by_name['type']
@@ -3254,8 +3436,18 @@ _STREAM.fields_by_name['video'].containing_oneof = _STREAM.oneofs_by_name['type'
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['audio'])
_STREAM.fields_by_name['audio'].containing_oneof = _STREAM.oneofs_by_name['type']
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['software'])
_STREAM.fields_by_name['software'].containing_oneof = _STREAM.oneofs_by_name['type']
_CHANNEL.fields_by_name['cover'].message_type = _SOURCE
_CHANNEL.fields_by_name['featured'].message_type = _CLAIMLIST
_CLAIMLIST.fields_by_name['list_type'].enum_type = _CLAIMLIST_LISTTYPE
_CLAIMLIST.fields_by_name['claim_references'].message_type = _CLAIMREFERENCE
_CLAIMLIST_LISTTYPE.containing_type = _CLAIMLIST
_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
_FEE_CURRENCY.containing_type = _FEE
_VIDEO.fields_by_name['audio'].message_type = _AUDIO
_SOFTWARE_OS.containing_type = _SOFTWARE
_LANGUAGE.fields_by_name['language'].enum_type = _LANGUAGE_LANGUAGE
_LANGUAGE.fields_by_name['script'].enum_type = _LANGUAGE_SCRIPT
_LANGUAGE.fields_by_name['region'].enum_type = _LOCATION_COUNTRY
@@ -3264,13 +3456,16 @@ _LANGUAGE_SCRIPT.containing_type = _LANGUAGE
_LOCATION.fields_by_name['country'].enum_type = _LOCATION_COUNTRY
_LOCATION_COUNTRY.containing_type = _LOCATION
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
DESCRIPTOR.message_types_by_name['ClaimReference'] = _CLAIMREFERENCE
DESCRIPTOR.message_types_by_name['ClaimList'] = _CLAIMLIST
DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
DESCRIPTOR.message_types_by_name['Fee'] = _FEE
DESCRIPTOR.message_types_by_name['File'] = _FILE
DESCRIPTOR.message_types_by_name['Image'] = _IMAGE
DESCRIPTOR.message_types_by_name['Video'] = _VIDEO
DESCRIPTOR.message_types_by_name['Audio'] = _AUDIO
DESCRIPTOR.message_types_by_name['Software'] = _SOFTWARE
DESCRIPTOR.message_types_by_name['Language'] = _LANGUAGE
DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -3282,13 +3477,6 @@ Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), d
))
_sym_db.RegisterMessage(Claim)
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
DESCRIPTOR = _CHANNEL,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Channel)
))
_sym_db.RegisterMessage(Channel)
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
DESCRIPTOR = _STREAM,
__module__ = 'claim_pb2'
@@ -3296,6 +3484,34 @@ Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,),
))
_sym_db.RegisterMessage(Stream)
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
DESCRIPTOR = _CHANNEL,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Channel)
))
_sym_db.RegisterMessage(Channel)
ClaimReference = _reflection.GeneratedProtocolMessageType('ClaimReference', (_message.Message,), dict(
DESCRIPTOR = _CLAIMREFERENCE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimReference)
))
_sym_db.RegisterMessage(ClaimReference)
ClaimList = _reflection.GeneratedProtocolMessageType('ClaimList', (_message.Message,), dict(
DESCRIPTOR = _CLAIMLIST,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimList)
))
_sym_db.RegisterMessage(ClaimList)
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
DESCRIPTOR = _SOURCE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Source)
))
_sym_db.RegisterMessage(Source)
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
DESCRIPTOR = _FEE,
__module__ = 'claim_pb2'
@@ -3303,13 +3519,6 @@ Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
))
_sym_db.RegisterMessage(Fee)
File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dict(
DESCRIPTOR = _FILE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.File)
))
_sym_db.RegisterMessage(File)
Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(
DESCRIPTOR = _IMAGE,
__module__ = 'claim_pb2'
@@ -3331,6 +3540,13 @@ Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), d
))
_sym_db.RegisterMessage(Audio)
Software = _reflection.GeneratedProtocolMessageType('Software', (_message.Message,), dict(
DESCRIPTOR = _SOFTWARE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Software)
))
_sym_db.RegisterMessage(Software)
Language = _reflection.GeneratedProtocolMessageType('Language', (_message.Message,), dict(
DESCRIPTOR = _LANGUAGE,
__module__ = 'claim_pb2'
+376
View File
@@ -0,0 +1,376 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: result.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='result.proto',
package='pb',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe7\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x02 \x01(\t\x12\x15\n\rcanonical_url\x18\x03 \x01(\t\x12\x16\n\x0eis_controlling\x18\x04 \x01(\x08\x12\x17\n\x0f\x63reation_height\x18\x05 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x06 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\x07 \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\x08 \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\n \x01(\x04\x12\x16\n\x0esupport_amount\x18\x0b \x01(\x04\x12\x16\n\x0etrending_group\x18\x0c \x01(\r\x12\x16\n\x0etrending_mixed\x18\r \x01(\x02\x12\x16\n\x0etrending_local\x18\x0e \x01(\x02\x12\x17\n\x0ftrending_global\x18\x0f \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
)
_ERROR_CODE = _descriptor.EnumDescriptor(
name='Code',
full_name='pb.Error.Code',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='UNKNOWN_CODE', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='NOT_FOUND', index=1, number=1,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='INVALID', index=2, number=2,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=660,
serialized_end=712,
)
_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
_OUTPUTS = _descriptor.Descriptor(
name='Outputs',
full_name='pb.Outputs',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='txos', full_name='pb.Outputs.txos', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='total', full_name='pb.Outputs.total', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='offset', full_name='pb.Outputs.offset', index=3,
number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=20,
serialized_end=118,
)
_OUTPUT = _descriptor.Descriptor(
name='Output',
full_name='pb.Output',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='tx_hash', full_name='pb.Output.tx_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='nout', full_name='pb.Output.nout', index=1,
number=2, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='height', full_name='pb.Output.height', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claim', full_name='pb.Output.claim', index=3,
number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='error', full_name='pb.Output.error', index=4,
number=15, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='meta', full_name='pb.Output.meta',
index=0, containing_type=None, fields=[]),
],
serialized_start=120,
serialized_end=243,
)
_CLAIMMETA = _descriptor.Descriptor(
name='ClaimMeta',
full_name='pb.ClaimMeta',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='channel', full_name='pb.ClaimMeta.channel', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='short_url', full_name='pb.ClaimMeta.short_url', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=4,
number=5, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=5,
number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=6,
number=7, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=7,
number=8, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=8,
number=10, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=9,
number=11, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=10,
number=12, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=11,
number=13, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=12,
number=14, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=13,
number=15, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=246,
serialized_end=605,
)
_ERROR = _descriptor.Descriptor(
name='Error',
full_name='pb.Error',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='code', full_name='pb.Error.code', index=0,
number=1, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='text', full_name='pb.Error.text', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_ERROR_CODE,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=607,
serialized_end=712,
)
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT
_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA
_OUTPUT.fields_by_name['error'].message_type = _ERROR
_OUTPUT.oneofs_by_name['meta'].fields.append(
_OUTPUT.fields_by_name['claim'])
_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
_OUTPUT.oneofs_by_name['meta'].fields.append(
_OUTPUT.fields_by_name['error'])
_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
_ERROR_CODE.containing_type = _ERROR
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
DESCRIPTOR.message_types_by_name['Error'] = _ERROR
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), dict(
DESCRIPTOR = _OUTPUTS,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Outputs)
))
_sym_db.RegisterMessage(Outputs)
Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict(
DESCRIPTOR = _OUTPUT,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Output)
))
_sym_db.RegisterMessage(Output)
ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), dict(
DESCRIPTOR = _CLAIMMETA,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimMeta)
))
_sym_db.RegisterMessage(ClaimMeta)
Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict(
DESCRIPTOR = _ERROR,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Error)
))
_sym_db.RegisterMessage(Error)
# @@protoc_insertion_point(module_scope)
-185
View File
@@ -1,185 +0,0 @@
import re
PROTOCOL = 'lbry://'
CHANNEL_CHAR = '@'
CLAIM_ID_CHAR = '#'
CLAIM_SEQUENCE_CHAR = ':'
BID_POSITION_CHAR = '$'
PATH_CHAR = '/'
QUERY_CHAR = '?'
CLAIM_ID_MAX_LENGTH = 40
CHANNEL_NAME_MIN_LENGTH = 1
class URIParseError(Exception):
pass
class URI(object):
__slots__ = ['name', 'claim_sequence', 'bid_position', 'claim_id', 'path']
def __init__(self, name, claim_sequence=None, bid_position=None, claim_id=None, path=None):
if len([v for v in [claim_sequence, bid_position, claim_id] if v is not None]) > 1:
raise ValueError(
"Only one of these may be present at a time: claim_sequence, bid_position, claim_id"
)
self.name = name
self.claim_sequence = claim_sequence
self.bid_position = bid_position
self.claim_id = claim_id
self.path = path
if self.path is not None and not self.contains_channel:
raise ValueError("Content claims cannot have paths")
def __str__(self):
return self.to_uri_string()
def __eq__(self, other):
for prop in self.__slots__:
if not hasattr(other, prop) or getattr(self, prop) != getattr(other, prop):
return False
return self.__class__ == other.__class__
@property
def channel_name(self):
return self.name if self.contains_channel else None
@property
def claim_name(self):
return self.name if not self.contains_channel else self.path
@property
def contains_channel(self):
return self.name.startswith(CHANNEL_CHAR)
@property
def is_channel(self):
return self.contains_channel and not self.path
def to_uri_string(self):
uri_string = PROTOCOL + "%s" % self.name
if self.claim_sequence is not None:
uri_string += CLAIM_SEQUENCE_CHAR + "%i" % self.claim_sequence
elif self.bid_position is not None:
uri_string += BID_POSITION_CHAR + "%i" % self.bid_position
elif self.claim_id is not None:
uri_string += CLAIM_ID_CHAR + "%s" % self.claim_id
if self.path is not None:
uri_string += PATH_CHAR + "%s" % self.path
return uri_string
def to_dict(self):
return {
"name": self.name,
'claim_sequence': self.claim_sequence,
'bid_position': self.bid_position,
'claim_id': self.claim_id,
'path': self.path,
}
@classmethod
def from_uri_string(cls, uri_string):
"""
Parses LBRY uri into its components
:param uri_string: format - lbry://name:n$rank#id/path
optional modifiers:
claim_sequence (int): the nth claim to the name
bid_position (int): the bid queue position of the claim for the name
claim_id (str): the claim id for the claim
path (str): claim within a channel
:return: URI
"""
match = re.match(get_schema_regex(), uri_string)
if match is None:
raise URIParseError('Invalid URI')
if match.group('content_name') and match.group('path'):
raise URIParseError('Only channels may have paths')
return cls(
name=match.group("content_or_channel_name"),
claim_sequence=int(match.group("claim_sequence")) if match.group(
"claim_sequence") is not None else None,
bid_position=int(match.group("bid_position")) if match.group(
"bid_position") is not None else None,
claim_id=match.group("claim_id"),
path=match.group("path")
)
@classmethod
def from_dict(cls, uri_dict):
"""
Creates URI from dict
:return: URI
"""
return cls(**uri_dict)
def get_schema_regex():
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
def _group(regex):
return "(?:" + regex + ")"
# TODO: regex should include the fact that content names cannot have paths
# right now this is only enforced in code, not in the regex
# Escape constants
claim_id_char = re.escape(CLAIM_ID_CHAR)
claim_sequence_char = re.escape(CLAIM_SEQUENCE_CHAR)
bid_position_char = re.escape(BID_POSITION_CHAR)
channel_char = re.escape(CHANNEL_CHAR)
path_char = re.escape(PATH_CHAR)
protocol = _named("protocol", re.escape(PROTOCOL))
# Define basic building blocks
valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/
name_content = valid_name_char + '+'
name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
positive_number = "[1-9][0-9]*"
number = '\-?' + positive_number
# Define URI components
content_name = _named("content_name", name_content)
channel_name = _named("channel_name", channel_char + name_min_channel_length)
content_or_channel_name = _named("content_or_channel_name", content_name + "|" + channel_name)
claim_id_piece = _named("claim_id", "[0-9a-f]{1," + str(CLAIM_ID_MAX_LENGTH) + "}")
claim_id = _group(claim_id_char + claim_id_piece)
bid_position_piece = _named("bid_position", number)
bid_position = _group(bid_position_char + bid_position_piece)
claim_sequence_piece = _named("claim_sequence", number)
claim_sequence = _group(claim_sequence_char + claim_sequence_piece)
modifier = _named("modifier", claim_id + "|" + bid_position + "|" + claim_sequence)
path_piece = _named("path", name_content)
path = _group(path_char + path_piece)
# Combine components
uri = _named("uri", (
'^' +
protocol + '?' +
content_or_channel_name +
modifier + '?' +
path + '?' +
'$'
))
return uri
def parse_lbry_uri(lbry_uri):
return URI.from_uri_string(lbry_uri)
+126
View File
@@ -0,0 +1,126 @@
import re
import unicodedata
from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.com/
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
def _group(regex):
return "(?:" + regex + ")"
def _oneof(*choices):
return _group('|'.join(choices))
def _claim(name, prefix=""):
return _group(
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
return (
'^' +
_named("scheme", "lbry://") + '?' +
_oneof(
_group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")),
_claim("channel", "@"),
_claim("stream")
) +
'$'
)
URL_REGEX = _create_url_regex()
def normalize_name(name):
return unicodedata.normalize('NFD', name).casefold()
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
def normalized(self):
return normalize_name(self.name)
def to_dict(self):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
def __str__(self):
if self.claim_id is not None:
return f"{self.name}#{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
class URL(NamedTuple):
stream: PathSegment
channel: PathSegment
@property
def has_channel(self):
return self.channel is not None
@property
def has_stream(self):
return self.stream is not None
@property
def has_stream_in_channel(self):
return self.has_channel and self.has_stream
@property
def parts(self) -> Tuple:
if self.has_stream_in_channel:
return self.channel, self.stream
if self.has_channel:
return self.channel,
return self.stream,
def __str__(self):
return f"lbry://{'/'.join(str(p) for p in self.parts)}"
@classmethod
def parse(cls, url):
match = re.match(URL_REGEX, url)
if match is None:
raise ValueError('Invalid LBRY URL')
segments = {}
parts = match.groupdict()
for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'):
if parts[f'{segment}_name'] is not None:
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
if 'channel_with_stream' in segments:
segments['channel'] = segments['channel_with_stream']
segments['stream'] = segments['stream_in_channel']
return cls(segments.get('stream', None), segments.get('channel', None))
-136
View File
@@ -1,136 +0,0 @@
import os
import binascii
import logging
import typing
import asyncio
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.blob_file import BlobFile
log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class StreamAssembler:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
output_file_name: typing.Optional[str] = None):
self.output_file_name = output_file_name
self.loop = loop
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.sd_blob: 'BlobFile' = None
self.descriptor: StreamDescriptor = None
self.got_descriptor = asyncio.Event(loop=self.loop)
self.wrote_bytes_event = asyncio.Event(loop=self.loop)
self.stream_finished_event = asyncio.Event(loop=self.loop)
self.output_path = ''
self.stream_handle = None
self.written_bytes: int = 0
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
if not blob or not self.stream_handle or self.stream_handle.closed:
return False
def _decrypt_and_write():
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
self.stream_handle.seek(offset)
_decrypted = blob.decrypt(
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
)
self.stream_handle.write(_decrypted)
self.stream_handle.flush()
self.written_bytes += len(_decrypted)
log.debug("decrypted %s", blob.blob_hash[:8])
await self.loop.run_in_executor(None, _decrypt_and_write)
return True
async def setup(self):
pass
async def after_got_descriptor(self):
pass
async def after_finished(self):
pass
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
if not os.path.isdir(output_dir):
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
await self.setup()
self.sd_blob = await self.get_blob(self.sd_hash)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
self.sd_blob)
await self.after_got_descriptor()
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
self.output_path = os.path.join(output_dir, self.output_file_name)
if not self.got_descriptor.is_set():
self.got_descriptor.set()
await self.blob_manager.storage.store_stream(
self.sd_blob, self.descriptor
)
await self.blob_manager.blob_completed(self.sd_blob)
written_blobs = None
save_tasks = []
try:
with open(self.output_path, 'wb') as stream_handle:
self.stream_handle = stream_handle
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
if blob_info.blob_num != i:
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
return
while self.stream_handle and not self.stream_handle.closed:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
if blob and blob.length != blob_info.length:
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
await self.blob_manager.delete_blobs([blob_info.blob_hash])
continue
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
written_blobs = i
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
break
except FileNotFoundError:
log.debug("stream assembler stopped")
return
except (ValueError, IOError, OSError):
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
self.descriptor.sd_hash)
continue
finally:
if written_blobs == len(self.descriptor.blobs) - 2:
log.debug("finished decrypting and assembling stream")
if save_tasks:
await asyncio.wait(save_tasks)
await self.after_finished()
self.stream_finished_event.set()
else:
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
len(self.descriptor.blobs) - 2)
if self.output_path and os.path.isfile(self.output_path):
log.debug("erasing incomplete file assembly: %s", self.output_path)
os.unlink(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return self.blob_manager.get_blob(blob_hash, length)
+34 -19
View File
@@ -8,7 +8,7 @@ from collections import OrderedDict
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import AbstractBlob, BlobFile
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.error import InvalidStreamDescriptorError
@@ -47,6 +47,17 @@ def file_reader(file_path: str):
class StreamDescriptor:
__slots__ = [
'loop',
'blob_dir',
'stream_name',
'key',
'suggested_file_name',
'blobs',
'stream_hash',
'sd_hash'
]
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, stream_name: str, key: str,
suggested_file_name: str, blobs: typing.List[BlobInfo], stream_hash: typing.Optional[str] = None,
sd_hash: typing.Optional[str] = None):
@@ -108,29 +119,30 @@ class StreamDescriptor:
h.update(self.old_sort_json())
return h.hexdigest()
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
old_sort: typing.Optional[bool] = False):
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
old_sort: typing.Optional[bool] = False,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
if not old_sort:
sd_data = self.as_json()
else:
sd_data = self.old_sort_json()
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
if blob_file_obj:
blob_file_obj.set_length(len(sd_data))
if not sd_blob.get_is_verified():
writer = sd_blob.open_for_writing()
writer = sd_blob.get_blob_writer()
writer.write(sd_data)
await sd_blob.verified.wait()
sd_blob.close()
return sd_blob
@classmethod
def _from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
blob: BlobFile) -> 'StreamDescriptor':
assert os.path.isfile(blob.file_path)
with open(blob.file_path, 'rb') as f:
json_bytes = f.read()
blob: AbstractBlob) -> 'StreamDescriptor':
with blob.reader_context() as blob_reader:
json_bytes = blob_reader.read()
try:
decoded = json.loads(json_bytes.decode())
except json.JSONDecodeError:
@@ -160,8 +172,10 @@ class StreamDescriptor:
@classmethod
async def from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
blob: BlobFile) -> 'StreamDescriptor':
return await loop.run_in_executor(None, lambda: cls._from_stream_descriptor_blob(loop, blob_dir, blob))
blob: AbstractBlob) -> 'StreamDescriptor':
if not blob.is_readable():
raise InvalidStreamDescriptorError(f"unreadable/missing blob: {blob.blob_hash}")
return await loop.run_in_executor(None, cls._from_stream_descriptor_blob, loop, blob_dir, blob)
@staticmethod
def get_blob_hashsum(b: typing.Dict):
@@ -194,11 +208,12 @@ class StreamDescriptor:
return h.hexdigest()
@classmethod
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False) -> 'StreamDescriptor':
async def create_stream(
cls, loop: asyncio.BaseEventLoop, blob_dir: str, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> 'StreamDescriptor':
blobs: typing.List[BlobInfo] = []
iv_generator = iv_generator or random_iv_generator()
@@ -207,7 +222,7 @@ class StreamDescriptor:
for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
)
blobs.append(blob_info)
blobs.append(
@@ -216,7 +231,7 @@ class StreamDescriptor:
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
blobs
)
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
descriptor.sd_hash = sd_blob.blob_hash
return descriptor
@@ -228,7 +243,7 @@ class StreamDescriptor:
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
@classmethod
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
async def recover(cls, blob_dir: str, sd_blob: 'AbstractBlob', stream_hash: str, stream_name: str,
suggested_file_name: str, key: str,
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
+116 -77
View File
@@ -1,108 +1,147 @@
import os
import asyncio
import typing
import logging
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
import binascii
from lbrynet.error import DownloadSDTimeout
from lbrynet.utils import resolve_host, lru_cache_concurrent
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.blob_info import BlobInfo
log = logging.getLogger(__name__)
def drain_into(a: list, b: list):
while a:
b.append(a.pop())
class StreamDownloader(StreamAssembler):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
super().__init__(loop, blob_manager, sd_hash, output_file_name)
class StreamDownloader:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager', sd_hash: str,
descriptor: typing.Optional[StreamDescriptor] = None):
self.loop = loop
self.config = config
self.output_dir = output_dir or self.config.download_dir
self.output_file_name = output_file_name
self.blob_downloader: typing.Optional[BlobDownloader] = None
self.search_queue = asyncio.Queue(loop=loop)
self.peer_queue = asyncio.Queue(loop=loop)
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.descriptor: typing.Optional[StreamDescriptor]
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.search_queue = asyncio.Queue(loop=loop) # blob hashes to feed into the iterative finder
self.peer_queue = asyncio.Queue(loop=loop) # new peers to try
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.descriptor: typing.Optional[StreamDescriptor] = descriptor
self.node: typing.Optional['Node'] = None
self.assemble_task: typing.Optional[asyncio.Task] = None
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
self.fixed_peers_delay: typing.Optional[float] = None
self.added_fixed_peers = False
self.time_to_descriptor: typing.Optional[float] = None
self.time_to_first_bytes: typing.Optional[float] = None
async def setup(self): # start the peer accumulator and initialize the downloader
if self.blob_downloader:
raise Exception("downloader is already set up")
if self.node:
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.search_queue.put_nowait(self.sd_hash)
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2)
async def after_got_descriptor(self):
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to search")
if self.blob_manager.decrypted_blob_lru_cache:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob
)
async def after_finished(self):
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
self.blob_downloader.close()
self.cached_read_blob = cached_read_blob
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.assemble_task:
self.assemble_task.cancel()
self.assemble_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader = None
if self.stream_handle:
if not self.stream_handle.closed:
self.stream_handle.close()
self.stream_handle = None
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
self.output_path and os.path.isfile(self.output_path):
os.remove(self.output_path)
async def add_fixed_peers(self):
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return await self.blob_downloader.download_blob(blob_hash, length)
def add_fixed_peers(self):
async def _add_fixed_peers():
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
if not self.config.reflector_servers:
return
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()):
self.fixed_peers_delay = 0.0
else:
self.fixed_peers_delay = self.config.fixed_peer_delay
self.loop.create_task(_add_fixed_peers())
def download(self, node: typing.Optional['Node'] = None):
self.node = node
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
self.add_fixed_peers()
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
async def load_descriptor(self, connection_id: int = 0):
# download or get the sd blob
sd_blob = self.blob_manager.get_blob(self.sd_hash)
if not sd_blob.get_is_verified():
try:
now = self.loop.time()
sd_blob = await asyncio.wait_for(
self.blob_downloader.download_blob(self.sd_hash, connection_id),
self.config.blob_download_timeout, loop=self.loop
)
log.info("downloaded sd blob %s", self.sd_hash)
self.time_to_descriptor = self.loop.time() - now
except asyncio.TimeoutError:
raise DownloadSDTimeout(self.sd_hash)
# parse the descriptor
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, sd_blob
)
log.info("loaded stream manifest %s", self.sd_hash)
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
# set up peer accumulation
if node:
self.node = node
if self.accumulate_task and not self.accumulate_task.done():
self.accumulate_task.cancel()
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
await self.add_fixed_peers()
# start searching for peers for the sd hash
self.search_queue.put_nowait(self.sd_hash)
log.info("searching for peers for stream %s", self.sd_hash)
if not self.descriptor:
await self.load_descriptor(connection_id)
# add the head blob to the peer search
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to peer search for stream %s", self.sd_hash)
if not await self.blob_manager.storage.stream_exists(self.sd_hash):
await self.blob_manager.storage.store_stream(
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
)
async def download_stream_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> 'AbstractBlob':
if not filter(lambda blob: blob.blob_hash == blob_info.blob_hash, self.descriptor.blobs[:-1]):
raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
blob = await asyncio.wait_for(
self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id),
self.config.blob_download_timeout * 10, loop=self.loop
)
return blob
def decrypt_blob(self, blob_info: 'BlobInfo', blob: 'AbstractBlob') -> bytes:
return blob.decrypt(
binascii.unhexlify(self.descriptor.key.encode()), binascii.unhexlify(blob_info.iv.encode())
)
async def read_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> bytes:
start = None
if self.time_to_first_bytes is None:
start = self.loop.time()
blob = await self.download_stream_blob(blob_info, connection_id)
decrypted = self.decrypt_blob(blob_info, blob)
if start:
self.time_to_first_bytes = self.loop.time() - start
return decrypted
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader.close()
+358 -50
View File
@@ -3,56 +3,128 @@ import asyncio
import typing
import logging
import binascii
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbrynet.utils import generate_id
from lbrynet.error import DownloadSDTimeout
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
from lbrynet.extras.daemon.storage import StoredStreamClaim
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.schema.claim import Claim
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.wallet.transaction import Transaction
log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class ManagedStream:
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
downloader: typing.Optional[StreamDownloader] = None,
SAVING_ID = 1
STREAMING_ID = 2
__slots__ = [
'loop',
'config',
'blob_manager',
'sd_hash',
'download_directory',
'_file_name',
'_status',
'stream_claim_info',
'download_id',
'rowid',
'written_bytes',
'content_fee',
'downloader',
'analytics_manager',
'fully_reflected',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
'streaming',
'_running',
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt'
]
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None):
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
descriptor: typing.Optional[StreamDescriptor] = None,
content_fee: typing.Optional['Transaction'] = None,
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.rowid = rowid
self.sd_hash = sd_hash
self.download_directory = download_directory
self._file_name = file_name
self.descriptor = descriptor
self.downloader = downloader
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.written_bytes = 0
self.content_fee = content_fee
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.tx = None
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
self.streaming = asyncio.Event(loop=self.loop)
self._running = asyncio.Event(loop=self.loop)
self.saving = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop)
self.finished_write_attempt = asyncio.Event(loop=self.loop)
@property
def descriptor(self) -> StreamDescriptor:
return self.downloader.descriptor
@property
def stream_hash(self) -> str:
return self.descriptor.stream_hash
@property
def file_name(self) -> typing.Optional[str]:
return self.downloader.output_file_name if self.downloader else self._file_name
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
@property
def status(self) -> str:
return self._status
def update_status(self, status: str):
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
@property
def finished(self) -> bool:
@@ -95,57 +167,69 @@ class ManagedStream:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) ->typing.Optional[typing.Dict]:
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property
def blobs_completed(self) -> int:
return sum([1 if self.blob_manager.get_blob(b.blob_hash).get_is_verified() else 0
return sum([1 if self.blob_manager.is_blob_verified(b.blob_hash) else 0
for b in self.descriptor.blobs[:-1]])
@property
def blobs_in_stream(self) -> int:
return len(self.descriptor.blobs) - 1
@property
def sd_hash(self):
return self.descriptor.sd_hash
@property
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
def as_dict(self) -> typing.Dict:
full_path = self.full_path if self.output_file_exists else None
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
@property
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
if self.downloader and self.downloader.written_bytes:
written_bytes = self.downloader.written_bytes
elif full_path:
written_bytes = os.stat(full_path).st_size
def as_dict(self) -> typing.Dict:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if self.full_path and self.output_file_exists:
if self.written_bytes:
written_bytes = self.written_bytes
else:
written_bytes = os.stat(self.full_path).st_size
else:
full_path = None
file_name = None
download_directory = None
written_bytes = None
return {
'completed': self.finished,
'file_name': self.file_name,
'download_directory': self.download_directory,
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
'tx': self.tx,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
'suggested_file_name': self.descriptor.suggested_file_name,
'sd_hash': self.descriptor.sd_hash,
'download_path': full_path,
'mime_type': mime_type,
'mime_type': self.mime_type,
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
@@ -159,38 +243,202 @@ class ManagedStream:
'nout': self.nout,
'outpoint': self.outpoint,
'metadata': self.metadata,
'protobuf': self.metadata_protobuf,
'channel_claim_id': self.channel_claim_id,
'channel_name': self.channel_name,
'claim_name': self.claim_name
'claim_name': self.claim_name,
'content_fee': self.content_fee
}
@classmethod
async def create(cls, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager',
async def create(cls, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=blob_manager.blob_completed
)
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
await blob_manager.blob_completed(sd_blob)
for blob in descriptor.blobs[:-1]:
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
status=cls.STATUS_FINISHED)
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
def start_download(self, node: typing.Optional['Node']):
self.downloader.download(node)
self.update_status(self.STATUS_RUNNING)
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
save_now: bool = False):
timeout = timeout or self.config.download_timeout
if self._running.is_set():
return
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
self._running.set()
try:
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
except asyncio.TimeoutError:
self._running.clear()
raise DownloadSDTimeout(self.sd_hash)
def stop_download(self):
if self.downloader:
self.downloader.stop()
self.downloader = None
if self.delayed_stop_task and not self.delayed_stop_task.done():
self.delayed_stop_task.cancel()
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
if not await self.blob_manager.storage.file_exists(self.sd_hash):
if save_now:
file_name, download_dir = self._file_name, self.download_directory
else:
file_name, download_dir = None, None
self.rowid = await self.blob_manager.storage.save_downloaded_file(
self.stream_hash, file_name, download_dir, 0.0
)
if self.status != self.STATUS_RUNNING:
await self.update_status(self.STATUS_RUNNING)
async def stop(self, finished: bool = False):
"""
Stop any running save/stream tasks as well as the downloader and update the status in the database
"""
self.stop_tasks()
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
if start_blob_num >= len(self.descriptor.blobs[:-1]):
raise IndexError(start_blob_num)
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
assert i + start_blob_num == blob_info.blob_num
if connection_id == self.STREAMING_ID:
decrypted = await self.downloader.cached_read_blob(blob_info)
else:
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
await self.start(node)
response = StreamResponse(
status=206,
headers=headers
)
await response.prepare(request)
self.streaming_responses.append((request, response))
self.streaming.set()
try:
wrote = 0
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
if not wrote:
decrypted = decrypted[first_blob_start_offset:]
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
len(self.descriptor.blobs) - 1)
await response.write_eof(decrypted)
else:
log.debug("sending browser blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await response.write(decrypted)
wrote += len(decrypted)
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
if response._eof_sent:
break
return response
finally:
response.force_close()
if (request, response) in self.streaming_responses:
self.streaming_responses.remove((request, response))
if not self.streaming_responses:
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
output_path)
self.saving.set()
self.finished_write_attempt.clear()
self.finished_writing.clear()
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
self.download_id, self.claim_name, self.sd_hash
))
self.finished_writing.set()
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
self.sd_hash[:6], self.full_path)
await self.blob_manager.storage.set_saved_file(self.stream_hash)
except Exception as err:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
self.written_bytes = 0
if isinstance(err, asyncio.TimeoutError):
self.downloader.stop()
await self.blob_manager.storage.change_file_download_dir_and_file_name(
self.stream_hash, None, None
)
self._file_name, self.download_directory = None, None
await self.blob_manager.storage.clear_saved_file(self.stream_hash)
await self.update_status(self.STATUS_STOPPED)
return
elif not isinstance(err, asyncio.CancelledError):
log.exception("unexpected error encountered writing file for stream %s", self.sd_hash)
raise err
finally:
self.saving.clear()
self.finished_write_attempt.set()
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
node: typing.Optional['Node'] = None):
await self.start(node)
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
self.file_output_task.cancel()
self.download_directory = download_directory or self.download_directory or self.config.download_dir
if not self.download_directory:
raise ValueError("no directory to download to")
if not (file_name or self._file_name or self.descriptor.suggested_file_name):
raise ValueError("no file name to download to")
if not os.path.isdir(self.download_directory):
log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
os.mkdir(self.download_directory)
self._file_name = await get_next_available_file_name(
self.loop, self.download_directory,
file_name or self.descriptor.suggested_file_name
)
await self.blob_manager.storage.change_file_download_dir_and_file_name(
self.stream_hash, self.download_directory, self.file_name
)
await self.update_status(ManagedStream.STATUS_RUNNING)
self.written_bytes = 0
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
await self.started_writing.wait()
def stop_tasks(self):
if self.file_output_task and not self.file_output_task.done():
self.file_output_task.cancel()
self.file_output_task = None
while self.streaming_responses:
req, response = self.streaming_responses.pop()
response.force_close()
req.transport.close()
self.downloader.stop()
self._running.clear()
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
sent = []
@@ -206,7 +454,9 @@ class ManagedStream:
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return []
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
we_have = [
blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes
]
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
@@ -229,3 +479,61 @@ class ManagedStream:
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
if not claim_info:
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
self.set_claim(claim_info, claim_info['value'])
async def _delayed_stop(self):
stalled_count = 0
while self._running.is_set():
if self.saving.is_set() or self.streaming.is_set():
stalled_count = 0
else:
stalled_count += 1
if stalled_count > 1:
log.info("stopping inactive download for lbry://%s#%s (%s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
await self.stop()
return
await asyncio.sleep(1, loop=self.loop)
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
if '=' in get_range:
get_range = get_range.split('=')[1]
start, end = get_range.split('-')
size = 0
for blob in self.descriptor.blobs[:-1]:
size += blob.length - 1
if self.stream_claim_info and self.stream_claim_info.claim.stream.source.size:
size_from_claim = int(self.stream_claim_info.claim.stream.source.size)
if not size_from_claim <= size <= size_from_claim + 16:
raise ValueError("claim contains implausible stream size")
log.debug("using stream size from claim")
size = size_from_claim
elif self.stream_claim_info:
log.debug("estimating stream size")
start = int(start)
if not 0 <= start < size:
raise HTTPRequestRangeNotSatisfiable()
end = int(end) if end else size - 1
if end >= size:
raise HTTPRequestRangeNotSatisfiable()
skip_blobs = start // 2097150
skip = skip_blobs * 2097151
skip_first_blob = start - skip
start = skip_first_blob + skip
final_size = end - start + 1
headers = {
'Accept-Ranges': 'bytes',
'Content-Range': f'bytes {start}-{end}/{size}',
'Content-Length': str(final_size),
'Content-Type': self.mime_type
}
return headers, size, skip_blobs, skip_first_blob
+5 -5
View File
@@ -4,7 +4,7 @@ import logging
import typing
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.descriptor import StreamDescriptor
REFLECTOR_V1 = 0
@@ -14,7 +14,7 @@ log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
@@ -64,7 +64,7 @@ class StreamReflectorClient(asyncio.Protocol):
async def send_descriptor(self) -> typing.Tuple[bool, typing.List[str]]: # returns a list of needed blob hashes
sd_blob = self.blob_manager.get_blob(self.descriptor.sd_hash)
assert sd_blob.get_is_verified(), "need to have a sd blob to send at this point"
assert self.blob_manager.is_blob_verified(self.descriptor.sd_hash), "need to have sd blob to send at this point"
response = await self.send_request({
'sd_blob_hash': sd_blob.blob_hash,
'sd_blob_size': sd_blob.length
@@ -80,7 +80,7 @@ class StreamReflectorClient(asyncio.Protocol):
sent_sd = True
if not needed:
for blob in self.descriptor.blobs[:-1]:
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
if self.blob_manager.is_blob_verified(blob.blob_hash, blob.length):
needed.append(blob.blob_hash)
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
self.reflected_blobs.append(sd_blob.blob_hash)
@@ -91,8 +91,8 @@ class StreamReflectorClient(asyncio.Protocol):
return sent_sd, needed
async def send_blob(self, blob_hash: str):
assert self.blob_manager.is_blob_verified(blob_hash), "need to have a blob to send at this point"
blob = self.blob_manager.get_blob(blob_hash)
assert blob.get_is_verified(), "need to have a blob to send at this point"
response = await self.send_request({
'blob_hash': blob.blob_hash,
'blob_size': blob.length
+7 -7
View File
@@ -7,7 +7,7 @@ from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.writer import HashBlobWriter
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager'):
def __init__(self, blob_manager: 'BlobManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -63,11 +63,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
return
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.open_for_writing()
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.incoming.set()
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
@@ -102,11 +102,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
return
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified():
self.writer = blob.open_for_writing()
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.incoming.set()
self.send_response({"send_blob": True})
try:
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop)
self.send_response({"received_blob": True})
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
@@ -121,7 +121,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobFileManager'):
def __init__(self, blob_manager: 'BlobManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
+255 -303
View File
@@ -5,22 +5,24 @@ import binascii
import logging
import random
from decimal import Decimal
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
DownloadDataTimeout, DownloadSDTimeout
from lbrynet.utils import generate_id
from aiohttp.web import Request
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError
from lbrynet.error import ResolveTimeout, DownloadDataTimeout
from lbrynet.utils import cache_concurrent
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import Claim
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.storage import lbc_to_dewies
from lbrynet.schema.url import URL
from lbrynet.extras.daemon.storage import lbc_to_dewies, dewies_to_lbc
from lbrynet.wallet.transaction import Output
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
from lbrynet.wallet import LbryWalletManager
from lbrynet.wallet.transaction import Transaction
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
@@ -54,8 +56,14 @@ comparison_operators = {
}
def path_or_none(p) -> typing.Optional[str]:
if not p:
return
return binascii.unhexlify(p).decode()
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
@@ -65,79 +73,23 @@ class StreamManager:
self.storage = storage
self.node = node
self.analytics_manager = analytics_manager
self.streams: typing.Set[ManagedStream] = set()
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
self.resume_downloading_task: asyncio.Task = None
self.re_reflect_task: asyncio.Task = None
self.streams: typing.Dict[str, ManagedStream] = {}
self.resume_saving_task: typing.Optional[asyncio.Task] = None
self.re_reflect_task: typing.Optional[asyncio.Task] = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.running_reflector_uploads: typing.List[asyncio.Task] = []
self.started = asyncio.Event(loop=self.loop)
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, claim_info['value'])
async def start_stream(self, stream: ManagedStream) -> bool:
"""
Resume or rebuild a partial or completed stream
"""
if not stream.running and not stream.output_file_exists:
if stream.downloader:
stream.downloader.stop()
stream.downloader = None
# the directory is gone, can happen when the folder that contains a published file is deleted
# reset the download directory to the default and update the file name
if not os.path.isdir(stream.download_directory):
stream.download_directory = self.config.download_dir
stream.downloader = self.make_downloader(
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
)
if stream.status != ManagedStream.STATUS_FINISHED:
await self.storage.change_file_status(stream.stream_hash, 'running')
stream.update_status('running')
stream.start_download(self.node)
try:
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
self.config.download_timeout)
except asyncio.TimeoutError:
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
return False
file_name = os.path.basename(stream.downloader.output_path)
output_dir = os.path.dirname(stream.downloader.output_path)
await self.storage.change_file_download_dir_and_file_name(
stream.stream_hash, output_dir, file_name
)
stream._file_name = file_name
stream.download_directory = output_dir
self.wait_for_stream_finished(stream)
return True
return True
async def stop_stream(self, stream: ManagedStream):
stream.stop_download()
if not stream.finished and stream.output_file_exists:
try:
os.remove(stream.full_path)
except OSError as err:
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
str(err))
if stream.running:
stream.update_status(ManagedStream.STATUS_STOPPED)
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
return StreamDownloader(
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
)
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = []
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
suggested_file_name: str, key: str,
content_fee: typing.Optional['Transaction']) -> typing.Optional[StreamDescriptor]:
sd_blob = self.blob_manager.get_blob(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash)
descriptor = await StreamDescriptor.recover(
@@ -145,116 +97,126 @@ class StreamManager:
)
if not descriptor:
return
to_restore.append((descriptor, sd_blob))
to_restore.append((descriptor, sd_blob, content_fee))
await asyncio.gather(*[
recover_stream(
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key'],
file_info['content_fee']
) for file_info in file_infos
])
if to_restore:
await self.storage.recover_streams(to_restore, self.config.download_dir)
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
claim: typing.Optional['StoredStreamClaim']):
sd_blob = self.blob_manager.get_blob(sd_hash)
if not sd_blob.get_is_verified():
return
# if self.blob_manager._save_blobs:
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], status: str,
claim: typing.Optional['StoredStreamClaim'], content_fee: typing.Optional['Transaction']):
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err:
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
return
if status == ManagedStream.STATUS_RUNNING:
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
else:
downloader = None
stream = ManagedStream(
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
self.loop, self.config, self.blob_manager, descriptor.sd_hash, download_directory, file_name, status,
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
analytics_manager=self.analytics_manager
)
self.streams.add(stream)
self.streams[sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
async def load_streams_from_database(self):
async def load_and_resume_streams_from_database(self):
to_recover = []
for file_info in await self.storage.get_all_lbry_files():
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_recover.append(file_info)
to_start = []
await self.storage.update_manually_removed_files_since_last_run()
for file_info in await self.storage.get_all_lbry_files():
# if the sd blob is not verified, try to reconstruct it from the database
# this could either be because the blob files were deleted manually or save_blobs was not true when
# the stream was downloaded
if not self.blob_manager.is_blob_verified(file_info['sd_hash']):
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Attempting to recover %i streams", len(to_recover))
await self.recover_streams(to_recover)
to_start = []
for file_info in await self.storage.get_all_lbry_files():
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_start.append(file_info)
log.info("Initializing %i files", len(to_start))
await asyncio.gather(*[
self.add_stream(
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
file_info['claim']
) for file_info in to_start
])
to_resume_saving = []
add_stream_tasks = []
for file_info in to_start:
file_name = path_or_none(file_info['file_name'])
download_directory = path_or_none(file_info['download_directory'])
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
add_stream_tasks.append(self.loop.create_task(self.add_stream(
file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'],
file_info['claim'], file_info['content_fee']
)))
if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop)
log.info("Started stream manager with %i files", len(self.streams))
async def resume(self):
if self.node:
await self.node.joined.wait()
else:
if not self.node:
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
t = [
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
]
if t:
log.info("resuming %i downloads", len(t))
if to_resume_saving:
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
async def resume(self, to_resume_saving):
log.info("Resuming saving %i files", len(to_resume_saving))
await asyncio.gather(
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
for (file_name, download_directory, sd_hash) in to_resume_saving),
loop=self.loop
)
async def reflect_streams(self):
while True:
if self.config.reflect_streams and self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
while sd_hashes:
stream = self.streams[sd_hashes.pop()]
if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed:
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch)
await asyncio.gather(*batch, loop=self.loop)
batch = []
if batch:
await asyncio.gather(*batch)
await asyncio.gather(*batch, loop=self.loop)
await asyncio.sleep(300, loop=self.loop)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
await self.load_and_resume_streams_from_database()
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
self.started.set()
def stop(self):
if self.resume_downloading_task and not self.resume_downloading_task.done():
self.resume_downloading_task.cancel()
if self.resume_saving_task and not self.resume_saving_task.done():
self.resume_saving_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
stream = self.streams.pop()
stream.stop_download()
_, stream = self.streams.popitem()
stream.stop_tasks()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads:
self.running_reflector_uploads.pop().cancel()
self.started.clear()
log.info("finished stopping the stream manager")
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers:
host, port = random.choice(self.config.reflector_servers)
@@ -267,9 +229,9 @@ class StreamManager:
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
stream.stop_tasks()
if stream.sd_hash in self.streams:
del self.streams[stream.sd_hash]
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor)
@@ -277,7 +239,7 @@ class StreamManager:
os.remove(stream.full_path)
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
if streams:
return streams[0]
@@ -304,66 +266,33 @@ class StreamManager:
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams:
for stream in self.streams.values():
for search, val in search_by.items():
if comparison_operators[comparison](getattr(stream, search), val):
streams.append(stream)
break
else:
streams = list(self.streams)
streams = list(self.streams.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
stream.download_id, stream.claim_name, stream.sd_hash
))
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _store_stream(self, downloader: StreamDownloader) -> int:
file_name = os.path.basename(downloader.output_path)
download_directory = os.path.dirname(downloader.output_path)
if not await self.storage.stream_exists(downloader.sd_hash):
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.storage.file_exists(downloader.sd_hash):
return await self.storage.save_downloaded_file(
downloader.descriptor.stream_hash, file_name, download_directory,
0.0
)
else:
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim) -> typing.Tuple[
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
await self.start_stream(existing[0])
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
f"download {claim_id}")
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
await self.start_stream(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
@@ -372,142 +301,165 @@ class StreamManager:
return None, existing_for_claim_id[0]
return None, None
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
download_id: str, outpoint: str, claim: Claim, resolved: typing.Dict,
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
downloader.download(self.node)
await downloader.got_descriptor.wait()
got_descriptor_time.set_result(self.loop.time() - start_time)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, outpoint
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
stream.set_claim(resolved, claim)
await stream.downloader.wrote_bytes_event.wait()
self.streams.add(stream)
return stream
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['hex']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
file_name)
), timeout
)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
def _convert_to_old_resolve_output(self, resolves):
result = {}
for url, txo in resolves.items():
if isinstance(txo, Output):
tx_height = txo.tx_ref.height
best_height = self.wallet.ledger.headers.height
result[url] = {
'name': txo.claim_name,
'value': txo.claim,
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
'claim_id': txo.claim_id,
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': dewies_to_lbc(txo.amount),
'effective_amount': txo.meta.get('effective_amount', 0),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'claim_sequence': -1,
'address': txo.get_address(self.wallet.ledger),
'valid_at_height': txo.meta.get('activation_height', None),
'timestamp': self.wallet.ledger.headers[tx_height]['timestamp'],
'supports': []
}
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
claim.stream.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
time_to_first_bytes, None if not error else error.__class__.__name__
)
)
if error:
raise error
return stream
result[url] = txo
return result
@cache_concurrent
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: typing.Optional[float] = None,
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> ManagedStream:
download_directory: typing.Optional[str] = None,
save_file: typing.Optional[bool] = None,
resolve_timeout: float = 3.0) -> ManagedStream:
timeout = timeout or self.config.download_timeout
if uri in self.starting_streams:
return await self.starting_streams[uri]
fut = asyncio.Future(loop=self.loop)
self.starting_streams[uri] = fut
start_time = self.loop.time()
resolved_time = None
stream = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
try:
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
fut.set_result(stream)
except Exception as err:
fut.set_exception(err)
try:
return await fut
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
resolved_result = self._convert_to_old_resolve_output(
await asyncio.wait_for(self.wallet.ledger.resolve([uri]), resolve_timeout)
)
except asyncio.TimeoutError:
raise ResolveTimeout(uri)
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values() if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory, node=self.node
)
return updated_stream
content_fee = None
fee_amount, fee_address = None, None
# check that the fee is payable
if not to_replace and claim.stream.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
stream = ManagedStream(
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
file_name, ManagedStream.STATUS_RUNNING, content_fee=content_fee,
analytics_manager=self.analytics_manager
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
elif fee_address:
stream.content_fee = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1')
)
log.info("paid fee of %s for %s", fee_amount, uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeout(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, dont delete stream
error = err
raise
finally:
del self.starting_streams[uri]
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__
)
)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.streams[sd_hash].stream_file(request, self.node)
@@ -4,10 +4,9 @@ import tempfile
import logging
from binascii import unhexlify
from torba.testcase import IntegrationTestCase
from torba.testcase import IntegrationTestCase, WalletNode
import lbrynet.wallet
from lbrynet.schema.claim import Claim
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
@@ -19,7 +18,7 @@ from lbrynet.extras.daemon.Components import (
)
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.reflector.server import ReflectorServer
from lbrynet.blob_exchange.server import BlobServer
@@ -59,10 +58,10 @@ class ExchangeRateManagerComponent(Component):
class CommandTestCase(IntegrationTestCase):
timeout = 180
LEDGER = lbrynet.wallet
MANAGER = LbryWalletManager
VERBOSITY = logging.WARN
blob_lru_cache_size = 0
async def asyncSetUp(self):
await super().asyncSetUp()
@@ -71,17 +70,10 @@ class CommandTestCase(IntegrationTestCase):
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
logging.getLogger('lbrynet.stream').setLevel(self.VERBOSITY)
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = True
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.known_dht_nodes = []
self.daemons = []
self.extra_wallet_nodes = []
self.extra_wallet_node_port = 5280
self.daemon = await self.add_daemon(self.wallet_node)
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
@@ -89,28 +81,13 @@ class CommandTestCase(IntegrationTestCase):
await self.confirm_tx(sendtxid)
await self.generate(5)
def wallet_maker(component_manager):
self.wallet_component = WalletComponent(component_manager)
self.wallet_component.wallet_manager = self.manager
self.wallet_component._running = True
return self.wallet_component
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
]
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
exchange_rate_manager=ExchangeRateManagerComponent
))
await self.daemon.initialize()
self.manager.old_db = self.daemon.storage
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_storage = SQLiteStorage(Config(), ':memory:')
self.server_config = Config()
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
await self.server_storage.open()
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.server.start_server(5567, '127.0.0.1')
await self.server.started_listening.wait()
@@ -122,8 +99,54 @@ class CommandTestCase(IntegrationTestCase):
async def asyncTearDown(self):
await super().asyncTearDown()
self.wallet_component._running = False
await self.daemon.stop()
for wallet_node in self.extra_wallet_nodes:
await wallet_node.stop(cleanup=True)
for daemon in self.daemons:
daemon.component_manager.get_component('wallet')._running = False
await daemon.stop(shutdown_runner=False)
async def add_daemon(self, wallet_node=None, seed=None):
if wallet_node is None:
wallet_node = WalletNode(
self.wallet_node.manager_class,
self.wallet_node.ledger_class,
port=self.extra_wallet_node_port
)
self.extra_wallet_node_port += 1
await wallet_node.start(self.conductor.spv_node, seed=seed)
self.extra_wallet_nodes.append(wallet_node)
conf = Config()
conf.data_dir = wallet_node.data_path
conf.wallet_dir = wallet_node.data_path
conf.download_dir = wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = True
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
]
def wallet_maker(component_manager):
wallet_component = WalletComponent(component_manager)
wallet_component.wallet_manager = wallet_node.manager
wallet_component._running = True
return wallet_component
daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
exchange_rate_manager=ExchangeRateManagerComponent
))
await daemon.initialize()
self.daemons.append(daemon)
wallet_node.manager.old_db = daemon.storage
return daemon
async def confirm_tx(self, txid):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
@@ -150,33 +173,64 @@ class CommandTestCase(IntegrationTestCase):
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
if confirm:
await self.generate(1)
return txid
async def blockchain_update_name(self, txid: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('updateclaim', txid, value, amount)
if confirm:
await self.generate(1)
return txid
async def out(self, awaitable):
""" Converts Daemon API call results (dictionary)
to JSON and then back to a dictionary. """
""" Serializes lbrynet API results to JSON then loads and returns it as dictionary. """
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
def sout(self, value):
""" Synchronous version of `out` method. """
return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
async def stream_create(self, name='hovercraft', bid='1.0', data=b'hi!', confirm=True, **kwargs):
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
)
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
await self.generate(1)
await self.on_transaction_dict(claim)
return claim
file = tempfile.NamedTemporaryFile()
def cleanup():
try:
file.close()
except FileNotFoundError:
pass
self.addCleanup(cleanup)
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
)
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
await self.generate(1)
await self.on_transaction_dict(claim)
return claim
async def stream_update(self, claim_id, data=None, confirm=True, **kwargs):
if data:
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
)
file = tempfile.NamedTemporaryFile()
file.write(data)
file.flush()
def cleanup():
try:
file.close()
except FileNotFoundError:
pass
self.addCleanup(cleanup)
claim = await self.out(
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
)
else:
claim = await self.out(self.daemon.jsonrpc_stream_update(claim_id, **kwargs))
self.assertIsNotNone(claim['outputs'][0]['name'])
@@ -186,6 +240,16 @@ class CommandTestCase(IntegrationTestCase):
await self.on_transaction_dict(claim)
return claim
async def stream_abandon(self, *args, confirm=True, **kwargs):
if 'blocking' not in kwargs:
kwargs['blocking'] = False
tx = await self.out(self.daemon.jsonrpc_stream_abandon(*args, **kwargs))
if confirm:
await self.on_transaction_dict(tx)
await self.generate(1)
await self.on_transaction_dict(tx)
return tx
async def publish(self, name, *args, confirm=True, **kwargs):
claim = await self.out(self.daemon.jsonrpc_publish(name, *args, **kwargs))
self.assertEqual(claim['outputs'][0]['name'], name)
@@ -213,10 +277,18 @@ class CommandTestCase(IntegrationTestCase):
await self.on_transaction_dict(channel)
return channel
async def claim_abandon(self, *args, confirm=True, **kwargs):
async def channel_abandon(self, *args, confirm=True, **kwargs):
if 'blocking' not in kwargs:
kwargs['blocking'] = False
tx = await self.out(self.daemon.jsonrpc_claim_abandon(*args, **kwargs))
tx = await self.out(self.daemon.jsonrpc_channel_abandon(*args, **kwargs))
if confirm:
await self.on_transaction_dict(tx)
await self.generate(1)
await self.on_transaction_dict(tx)
return tx
async def support_create(self, claim_id, bid='1.0', confirm=True, **kwargs):
tx = await self.out(self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs))
if confirm:
await self.on_transaction_dict(tx)
await self.generate(1)
@@ -226,5 +298,5 @@ class CommandTestCase(IntegrationTestCase):
async def resolve(self, uri):
return await self.out(self.daemon.jsonrpc_resolve(uri))
async def claim_search(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(*args, **kwargs)))['items']
async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
+98 -4
View File
@@ -14,6 +14,8 @@ import pkg_resources
import contextlib
import certifi
import aiohttp
import functools
import collections
from lbrynet.schema.claim import Claim
from lbrynet.cryptoutils import get_lbry_hash_obj
@@ -73,7 +75,7 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
@@ -100,7 +102,7 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
@@ -116,7 +118,7 @@ def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, Claim):
return stream_info.stream.sd_hash
return stream_info.stream.source.sd_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\
@@ -146,6 +148,44 @@ def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
cancel_task(tasks.pop())
def async_timed_cache(duration: int):
def wrapper(fn):
cache: typing.Dict[typing.Tuple,
typing.Tuple[typing.Any, float]] = {}
@functools.wraps(fn)
async def _inner(*args, **kwargs) -> typing.Any:
loop = asyncio.get_running_loop()
now = loop.time()
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
if key in cache and (now - cache[key][1] < duration):
return cache[key][0]
to_cache = await fn(*args, **kwargs)
cache[key] = to_cache, now
return to_cache
return _inner
return wrapper
def cache_concurrent(async_fn):
"""
When the decorated function has concurrent calls made to it with the same arguments, only run it once
"""
cache: typing.Dict = {}
@functools.wraps(async_fn)
async def wrapper(*args, **kwargs):
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
try:
return await cache[key]
finally:
cache.pop(key, None)
return wrapper
@async_timed_cache(300)
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
@@ -162,6 +202,60 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
))[0][4][0]
class LRUCache:
__slots__ = [
'capacity',
'cache'
]
def __init__(self, capacity):
self.capacity = capacity
self.cache = collections.OrderedDict()
def get(self, key):
value = self.cache.pop(key)
self.cache[key] = value
return value
def set(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def __contains__(self, item) -> bool:
return item in self.cache
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = override_lru_cache or LRUCache(cache_size)
def wrapper(async_fn):
@functools.wraps(async_fn)
async def _inner(*args, **kwargs):
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
if key in lru_cache:
return lru_cache.get(key)
concurrent_cache[key] = concurrent_cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
try:
result = await concurrent_cache[key]
lru_cache.set(key, result)
return result
finally:
concurrent_cache.pop(key, None)
return _inner
return wrapper
def get_ssl_context() -> ssl.SSLContext:
return ssl.create_default_context(
purpose=ssl.Purpose.CLIENT_AUTH, capath=certifi.where()
@@ -177,7 +271,7 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
try:
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
+23 -129
View File
@@ -1,11 +1,11 @@
import json
import logging
import binascii
from hashlib import sha256
from string import hexdigits
from torba.client.baseaccount import BaseAccount
from torba.client.basetransaction import TXORef
import ecdsa
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
log = logging.getLogger(__name__)
@@ -37,140 +37,34 @@ class Account(BaseAccount):
super().apply(d)
self.channel_keys.update(d.get('certificates', {}))
def add_channel_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.channel_keys, 'Trying to add a duplicate channel private key.'
self.channel_keys[ref.id] = private_key
def add_channel_private_key(self, private_key):
public_key_bytes = private_key.get_verifying_key().to_der()
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
def get_channel_private_key(self, ref: TXORef):
return self.channel_keys.get(ref.id)
def get_channel_private_key(self, public_key_bytes):
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
if private_key_pem:
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
async def maybe_migrate_certificates(self):
if not self.channel_keys:
return
addresses = {}
results = {
'total': 0,
'not-a-claim-tx': 0,
'migrate-success': 0,
'migrate-failed': 0,
'previous-success': 0,
'previous-corrupted': 0
}
double_hex_encoded_to_pop = []
for maybe_claim_id in list(self.channel_keys):
if ':' not in maybe_claim_id:
try:
validate_claim_id(maybe_claim_id)
continue
except Exception:
try:
maybe_claim_id_bytes = maybe_claim_id
if isinstance(maybe_claim_id_bytes, str):
maybe_claim_id_bytes = maybe_claim_id_bytes.encode()
decoded_double_hex = binascii.unhexlify(maybe_claim_id_bytes).decode()
validate_claim_id(decoded_double_hex)
if decoded_double_hex in self.channel_keys:
log.warning("don't know how to migrate certificate %s", decoded_double_hex)
else:
log.info("claim id was double hex encoded, fixing it")
double_hex_encoded_to_pop.append((maybe_claim_id, decoded_double_hex))
except Exception:
continue
for double_encoded_claim_id, correct_claim_id in double_hex_encoded_to_pop:
self.channel_keys[correct_claim_id] = self.channel_keys.pop(double_encoded_claim_id)
for maybe_claim_id in list(self.channel_keys):
results['total'] += 1
if ':' not in maybe_claim_id:
try:
validate_claim_id(maybe_claim_id)
except Exception as e:
log.warning(
"Failed to migrate claim '%s': %s",
maybe_claim_id, str(e)
)
results['migrate-failed'] += 1
continue
claims = await self.ledger.network.get_claims_by_ids(maybe_claim_id)
if maybe_claim_id not in claims:
log.warning(
"Failed to migrate claim '%s', server did not return any claim information.",
maybe_claim_id
)
results['migrate-failed'] += 1
continue
claim = claims[maybe_claim_id]
tx = None
if claim:
tx = await self.ledger.db.get_transaction(txid=claim['txid'])
else:
log.warning(maybe_claim_id)
if tx is not None:
txo = tx.outputs[claim['nout']]
if not txo.script.is_claim_involved:
results['not-a-claim-tx'] += 1
raise ValueError(
"Certificate with claim_id {} doesn't point to a valid transaction."
.format(maybe_claim_id)
)
tx_nout = '{txid}:{nout}'.format(**claim)
self.channel_keys[tx_nout] = self.channel_keys[maybe_claim_id]
del self.channel_keys[maybe_claim_id]
log.info(
"Migrated certificate with claim_id '%s' ('%s') to a new look up key %s.",
maybe_claim_id, txo.script.values['claim_name'], tx_nout
)
results['migrate-success'] += 1
else:
if claim:
addresses.setdefault(claim['address'], 0)
addresses[claim['address']] += 1
log.warning(
"Failed to migrate claim '%s', it's not associated with any of your addresses.",
maybe_claim_id
)
else:
log.warning(
"Failed to migrate claim '%s', it appears abandoned.",
maybe_claim_id
)
results['migrate-failed'] += 1
else:
try:
txid, nout = maybe_claim_id.split(':')
tx = await self.ledger.db.get_transaction(txid=txid)
if not tx:
log.warning(
"Claim migration failed to find a transaction for outpoint %s", maybe_claim_id
)
results['previous-corrupted'] += 1
continue
if tx.outputs[int(nout)].script.is_claim_involved:
results['previous-success'] += 1
else:
results['previous-corrupted'] += 1
except Exception:
log.exception("Couldn't verify certificate with look up key: %s", maybe_claim_id)
results['previous-corrupted'] += 1
channel_keys = {}
for private_key_pem in self.channel_keys.values():
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
public_key_der = private_key.get_verifying_key().to_der()
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
self.channel_keys = channel_keys
self.wallet.save()
log.info('verifying and possibly migrating certificates:')
log.info(json.dumps(results, indent=2))
if addresses:
log.warning('failed for addresses:')
log.warning(json.dumps(
[{'address': a, 'number of certificates': c} for a, c in addresses.items()],
indent=2
))
async def save_max_gap(self):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
if issubclass(self.address_generator, HierarchicalDeterministic):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
def get_balance(self, confirmations=0, include_claims=False, **constraints):
if not include_claims:
+4 -2
View File
@@ -59,11 +59,13 @@ class WalletDatabase(BaseDatabase):
channel_ids = set()
for txo in txos:
if txo.script.is_claim_name or txo.script.is_update_claim:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and my_account is not None:
txo.private_key = my_account.get_channel_private_key(txo.ref)
txo.private_key = my_account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if channel_ids:
channels = {
+51 -43
View File
@@ -1,15 +1,18 @@
import asyncio
import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from torba.client.baseledger import BaseLedger
from lbrynet.schema.uri import parse_lbry_uri, URIParseError
from torba.client.baseaccount import SingleKey
from lbrynet.schema.result import Outputs
from lbrynet.schema.url import URL
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.wallet.resolve import Resolver
from lbrynet.wallet.account import Account, validate_claim_id
from lbrynet.wallet.account import Account
from lbrynet.wallet.network import Network
from lbrynet.wallet.database import WalletDatabase
from lbrynet.wallet.transaction import Transaction
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.header import Headers, UnvalidatedHeaders
@@ -48,40 +51,35 @@ class MainNetLedger(BaseLedger):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self.resolver = Resolver(self)
@property
def resolver(self):
return Resolver(self.headers.claim_trie_root, self.headers.height, self.transaction_class,
hash160_to_address=self.hash160_to_address, network=self.network, ledger=self)
async def _inflate_outputs(self, query):
outputs = Outputs.from_base64(await query)
txs = []
if len(outputs.txs) > 0:
txs = await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))
return outputs.inflate(txs), outputs.offset, outputs.total
async def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.claim_id:
validate_claim_id(parsed_uri.claim_id)
except URIParseError as err:
return {'error': err.args[0]}
except Exception as e:
return {'error': str(e)}
try:
resolver = self.resolver
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def resolve(self, urls):
txos = (await self._inflate_outputs(self.network.resolve(urls)))[0]
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
if txo and URL.parse(url).has_stream_in_channel:
if not txo.channel or not txo.is_signed_by(txo.channel, self):
txo = None
if txo:
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return result
async def get_claim_by_claim_id(self, claim_id):
result = (await self.network.get_claims_by_ids(claim_id)).pop(claim_id, {})
return await self.resolver.get_certificate_and_validate_result(result)
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
async def get_claim_by_outpoint(self, txid, nout):
claims = (await self.network.get_claims_in_tx(txid)) or []
for claim in claims:
if claim['nout'] == nout:
return await self.resolver.get_certificate_and_validate_result(claim)
return 'claim not found'
async def get_claim_by_claim_id(self, claim_id) -> Dict[str, Output]:
for claim in (await self.claim_search(claim_id=claim_id))[0]:
return claim
async def start(self):
await super().start()
@@ -90,16 +88,26 @@ class MainNetLedger(BaseLedger):
await self._report_state()
async def _report_state(self):
for account in self.accounts:
total_receiving = len((await account.receiving.get_addresses()))
total_change = len((await account.change.get_addresses()))
channel_count = await account.get_channel_count()
claim_count = await account.get_claim_count()
balance = dewies_to_lbc(await account.get_balance())
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change, account.change.gap,
channel_count, len(account.channel_keys), claim_count)
try:
for account in self.accounts:
balance = dewies_to_lbc(await account.get_balance())
channel_count = await account.get_channel_count()
claim_count = await account.get_claim_count()
if isinstance(account.receiving, SingleKey):
log.info("Loaded single key account %s with %s LBC. "
"%d channels, %d certificates and %d claims",
account.id, balance, channel_count, len(account.channel_keys), claim_count)
else:
total_receiving = len((await account.receiving.get_addresses()))
total_change = len((await account.change.get_addresses()))
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change,
account.change.gap, channel_count, len(account.channel_keys), claim_count)
except:
log.exception(
'Failed to display wallet state, please file issue '
'for this bug along with the traceback you see below:')
class TestNetLedger(MainNetLedger):
+15 -30
View File
@@ -197,15 +197,12 @@ class LbryWalletManager(BaseWalletManager):
tx = await self.db.get_transaction(txid=txid)
if not tx:
try:
_raw = await self.ledger.network.get_transaction(txid)
raw = await self.ledger.network.get_transaction(txid)
height = await self.ledger.network.get_transaction_height(txid)
except CodeMessageError as e:
return {'success': False, 'code': e.code, 'message': e.message}
# this is a workaround for the current protocol. Should be fixed when lbryum support is over and we
# are able to use the modern get_transaction call, which accepts verbose to show height and other fields
height = await self.ledger.network.get_transaction_height(txid)
tx = self.ledger.transaction_class(unhexlify(_raw))
if tx and height > 0:
await self.ledger.maybe_verify_transaction(tx, height + 1) # off by one from server side, yes...
tx = self.ledger.transaction_class(unhexlify(raw))
await self.ledger.maybe_verify_transaction(tx, height)
return tx
@staticmethod
@@ -281,16 +278,17 @@ class LbryWalletManager(BaseWalletManager):
'is_tip': not is_my_inputs,
'nout': txo.position
})
for txo in tx.other_support_outputs:
item['support_info'].append({
'address': txo.get_address(account.ledger),
'balance_delta': dewies_to_lbc(-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position
})
if is_my_inputs:
for txo in tx.other_support_outputs:
item['support_info'].append({
'address': txo.get_address(account.ledger),
'balance_delta': dewies_to_lbc(-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position
})
for txo in tx.my_abandon_outputs:
item['abandon_info'].append({
'address': txo.get_address(account.ledger),
@@ -306,16 +304,3 @@ class LbryWalletManager(BaseWalletManager):
def save(self):
for wallet in self.wallets:
wallet.save()
def get_block(self, block_hash=None, height=None):
if height is None:
height = self.ledger.headers.height
if block_hash is None:
block_hash = self.ledger.headers.hash(height).decode()
return self.ledger.network.get_block(block_hash)
def get_claim_by_claim_id(self, claim_id):
return self.ledger.get_claim_by_claim_id(claim_id)
def get_claim_by_outpoint(self, txid, nout):
return self.ledger.get_claim_by_outpoint(txid, nout)
+7 -17
View File
@@ -3,24 +3,14 @@ from torba.client.basenetwork import BaseNetwork
class Network(BaseNetwork):
def get_block(self, block_hash):
return self.rpc('blockchain.block.get_block', block_hash)
def get_server_height(self):
return self.rpc('blockchain.block.get_server_height')
return self.rpc('blockchain.block.get_server_height', [])
def get_values_for_uris(self, block_hash, *uris):
return self.rpc('blockchain.claimtrie.getvaluesforuris', block_hash, *uris)
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def get_claims_by_ids(self, *claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', *claim_ids)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def get_claims_in_tx(self, txid):
return self.rpc('blockchain.claimtrie.getclaimsintx', txid)
def get_claims_for_name(self, name):
return self.rpc('blockchain.claimtrie.getclaimsforname', name)
def get_transaction_height(self, txid):
# 1.0 protocol specific workaround. Newer protocol should use get_transaction with verbose True
return self.rpc('blockchain.transaction.get_height', txid)
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
+143 -225
View File
@@ -1,36 +1,67 @@
import logging
from cryptography.exceptions import InvalidSignature
import asyncio
from binascii import unhexlify, hexlify
from lbrynet.utils import lru_cache_concurrent
from lbrynet.wallet.account import validate_claim_id
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
from lbrynet.schema.claim import Claim
from google.protobuf.message import DecodeError
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.url import URL
from lbrynet.wallet.claim_proofs import verify_proof, InvalidProofError
from lbrynet.wallet.transaction import Transaction
log = logging.getLogger(__name__)
class Resolver:
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network, ledger):
self.claim_trie_root = claim_trie_root
self.height = height
self.transaction_class = transaction_class
self.hash160_to_address = hash160_to_address
self.network = network
def __init__(self, ledger):
self.transaction_class = ledger.transaction_class
self.network = ledger.network
self.ledger = ledger
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
async def resolve(self, page, page_size, *uris):
uris = set(uris)
try:
for uri in uris:
for part in URL.parse(uri).parts:
if part.claim_id:
validate_claim_id(part.claim_id)
claim_trie_root = self.ledger.headers.claim_trie_root
resolutions = await self.network.get_values_for_uris(self.ledger.headers.hash().decode(), *uris)
if len(uris) > 1:
return await self._batch_handle(resolutions, uris, page, page_size, claim_trie_root)
return await self._handle_resolutions(resolutions, uris, page, page_size, claim_trie_root)
except ValueError as err:
return {'error': err.args[0]}
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def _batch_handle(self, resolutions, uris, page, page_size, claim_trie_root):
futs = []
for uri in uris:
futs.append(
asyncio.ensure_future(self._handle_resolutions(resolutions, [uri], page, page_size, claim_trie_root))
)
results = await asyncio.gather(*futs)
return dict(list(map(lambda result: list(result.items())[0], results)))
@lru_cache_concurrent(256)
async def _fetch_tx(self, txid):
return self.transaction_class(unhexlify(await self.network.get_transaction(txid)))
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size, claim_trie_root):
results = {}
for uri in requested_uris:
resolution = (resolutions or {}).get(uri, {})
if resolution:
try:
results[uri] = _handle_claim_result(
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
await self._handle_resolve_uri_response(uri, resolution, claim_trie_root, page, page_size),
uri
)
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
log.exception(err)
@@ -39,50 +70,24 @@ class Resolver:
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
return results
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
async def _handle_resolve_uri_response(self, uri, resolution, claim_trie_root, page=0, page_size=10):
result = {}
claim_trie_root = self.claim_trie_root
parsed_uri = parse_lbry_uri(uri)
certificate = None
parsed_uri = URL.parse(uri)
certificate_response = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate']['resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
height = certificate_response['height']
depth = self.height - height
certificate_result = _verify_proof(parsed_uri.name,
certificate_response = _verify_proof(parsed_uri.stream.name,
claim_trie_root,
certificate_response,
height, depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
raw=raw)
elif certificate_resolution_type == "claim_id":
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
raw=raw)
elif certificate_resolution_type == "sequence":
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
raw=raw)
else:
log.error("unknown response type: %s", certificate_resolution_type)
if 'certificate' in result:
certificate = result['certificate']
if 'unverified_claims_in_channel' in resolution:
max_results = len(resolution['unverified_claims_in_channel'])
result['claims_in_channel'] = max_results
else:
result['claims_in_channel'] = 0
else:
result['error'] = "claim not found"
result['success'] = False
result['uri'] = str(parsed_uri)
else:
certificate = None
ledger=self.ledger)
elif certificate_resolution_type not in ['winning', 'claim_id', 'sequence']:
raise Exception(f"unknown response type: {certificate_resolution_type}")
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response)
result['claims_in_channel'] = len(resolution.get('unverified_claims_in_channel', []))
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
@@ -90,27 +95,14 @@ class Resolver:
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
height = claim_response['height']
depth = self.height - height
claim_result = _verify_proof(parsed_uri.name,
claim_trie_root,
claim_response,
height, depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
certificate,
raw)
elif claim_resolution_type == "claim_id":
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate,
raw)
elif claim_resolution_type == "sequence":
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate,
raw)
else:
log.error("unknown response type: %s", claim_resolution_type)
claim_response = _verify_proof(parsed_uri.stream.name,
claim_trie_root,
claim_response,
ledger=self.ledger)
elif claim_resolution_type not in ["sequence", "winning", "claim_id"]:
raise Exception(f"unknown response type: {claim_resolution_type}")
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate_response)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
@@ -148,62 +140,55 @@ class Resolver:
# invalid signatures can only return outside a channel
if result.get('claim', {}).get('has_signature', False):
if parsed_uri.path and not result['claim']['signature_is_valid']:
if parsed_uri.has_stream and not result['claim']['signature_is_valid']:
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
return result
async def get_certificate_and_validate_result(self, claim_result):
async def parse_and_validate_claim_result(self, claim_result, certificate=None):
if not claim_result or 'value' not in claim_result:
return claim_result
certificate = None
certificate_id = Claim.from_bytes(unhexlify(claim_result['value'])).signing_channel_id
if certificate_id:
certificate = await self.network.get_claims_by_ids(certificate_id)
certificate = certificate.pop(certificate_id) if certificate else None
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
claim_result = _decode_claim_result(claim_result)
if claim_result.get('height'):
claim_result['timestamp'] = self.ledger.headers[claim_result['height']]['timestamp']
if claim_result.get('depth'):
claim_result['confirmations'] = claim_result.pop('depth')
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
if not claim_result or 'value' not in claim_result:
return claim_result
claim_result['decoded_claim'] = False
decoded = None
if not raw:
claim_value = claim_result['value']
try:
decoded = claim_result['value'] = Claim.from_bytes(unhexlify(claim_value))
claim_result['decoded_claim'] = True
except DecodeError:
pass
if decoded:
if claim_result['value']:
claim_result['has_signature'] = False
if decoded.is_signed:
claim_tx = await self.network.get_transaction(claim_result['txid'])
if claim_result['value'].is_signed:
claim_result['has_signature'] = True
claim_tx = await self._fetch_tx(claim_result['txid'])
if certificate is None:
log.info("fetching certificate to check claim signature")
certificate = await self.network.get_claims_by_ids(decoded.signing_channel_id)
channel_id = claim_result['value'].signing_channel_id
certificate = (await self.network.get_claims_by_ids([channel_id])).get(channel_id)
if not certificate:
log.warning('Certificate %s not found', decoded.signing_channel_id)
cert_tx = await self.network.get_transaction(certificate['txid']) if certificate else None
claim_result['has_signature'] = True
claim_result['signature_is_valid'] = False
validated, channel_name = validate_claim_signature_and_get_channel_name(
log.warning('Certificate %s not found', channel_id)
claim_result['channel_name'] = certificate['name'] if certificate else None
cert_tx = await self._fetch_tx(certificate['txid']) if certificate else None
claim_result['signature_is_valid'] = validate_claim_signature_and_get_channel_name(
claim_result, certificate, self.ledger, claim_tx=claim_tx, cert_tx=cert_tx
)
claim_result['channel_name'] = channel_name
if validated:
claim_result['signature_is_valid'] = True
if 'height' in claim_result and claim_result['height'] is None:
claim_result['height'] = -1
# fixme: workaround while json encoder isnt used here
if cert_tx:
channel_txo = cert_tx.outputs[certificate['nout']]
claim_result['signing_channel'] = {
'name': channel_txo.claim_name,
'claim_id': channel_txo.claim_id,
'value': channel_txo.claim
}
claim_result['is_channel_signature_valid'] = claim_result['signature_is_valid']
if 'amount' in claim_result:
claim_result = format_amount_value(claim_result)
claim_result['amount'] = dewies_to_lbc(claim_result['amount'])
claim_result['effective_amount'] = dewies_to_lbc(claim_result['effective_amount'])
claim_result['supports'] = [
{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
for (txid, nout, amount) in claim_result['supports']
]
claim_result['permanent_url'] = _get_permanent_url(
claim_result, decoded.signing_channel_id if decoded else None)
claim_result['height'] = claim_result.get('height', -1) or -1
claim_result['permanent_url'] = f"lbry://{claim_result['name']}#{claim_result['claim_id']}"
return claim_result
@@ -252,7 +237,7 @@ class Resolver:
async def iter_validate_channel_claims():
formatted_claims = []
for claim_ids in queries:
batch_result = await self.network.get_claims_by_ids(*claim_ids)
batch_result = await self.network.get_claims_by_ids(claim_ids)
for claim_id in claim_ids:
claim = batch_result[claim_id]
if claim['name'] == claim_names[claim_id]:
@@ -284,87 +269,51 @@ class Resolver:
start_position = (page - 1) * page_size
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
channel_claim_infos)
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
certificate, page_size=page_size)
upper_bound = len(claim_positions)
if not page:
return None, upper_bound
if start_position > upper_bound:
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
certificate, page_size=page_size)
return page_generator, upper_bound
# Format amount to be decimal encoded string
# Format value to be hex encoded string
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
def format_amount_value(obj):
if isinstance(obj, dict):
for k, v in obj.items():
if k in ('amount', 'effective_amount'):
if not isinstance(obj[k], float):
obj[k] = dewies_to_lbc(obj[k])
elif k == 'supports' and isinstance(v, list):
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
for (txid, nout, amount) in v]
elif isinstance(v, (list, dict)):
obj[k] = format_amount_value(v)
elif isinstance(obj, list):
obj = [format_amount_value(o) for o in obj]
return obj
def _get_permanent_url(claim_result, certificate_id):
if certificate_id:
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
else:
return f"{claim_result['name']}#{claim_result['claim_id']}"
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
def _verify_proof(name, claim_trie_root, result, ledger):
"""
Verify proof for name claim
"""
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
claim_sequence, claim_address, supports):
def _build_response(name, tx, nOut):
output = tx.outputs[nOut]
r = {
'name': name,
'value': hexlify(value),
'claim_id': claim_id,
'txid': txid,
'nout': n,
'amount': amount,
'effective_amount': effective_amount,
'height': height,
'depth': depth,
'claim_sequence': claim_sequence,
'address': claim_address,
'supports': supports
'value': hexlify(output.script.values['claim']),
'claim_id': output.claim_id,
'txid': tx.id,
'nout': nOut,
'amount': output.amount,
'effective_amount': output.amount + support_amount,
'height': result['height'],
'confirmations': result['depth'],
'claim_sequence': result['claim_sequence'],
'address': output.get_address(ledger),
'valid_at_height': result['valid_at_height'],
'timestamp': ledger.headers[result['height']]['timestamp'],
'supports': result['supports']
}
return r
def _parse_proof_result(name, result):
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
supports = result['supports']
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
if 'transaction' in result:
tx = transaction_class(raw=unhexlify(result['transaction']))
tx = ledger.transaction_class(raw=unhexlify(result['transaction']))
nOut = result['proof']['nOut']
if result['proof']['txhash'] == tx.id:
if 0 <= nOut < len(tx.outputs):
claim_output = tx.outputs[nOut]
effective_amount = claim_output.amount + support_amount
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
claim_id = result['claim_id']
claim_sequence = result['claim_sequence']
claim_script = claim_output.script
decoded_name = claim_script.values['claim_name'].decode()
decoded_value = claim_script.values['claim']
if decoded_name == name:
return _build_response(name, decoded_value, claim_id,
tx.id, nOut, claim_output.amount,
effective_amount, claim_sequence,
claim_address, supports)
if tx.outputs[nOut].claim_name == name:
return _build_response(name, tx, nOut)
return {'error': 'name in proof did not match requested name'}
outputs = len(tx['outputs'])
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
@@ -375,12 +324,8 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
return {'error': 'name is not claimed'}
if 'proof' in result:
proof_name = name
if 'name' in result:
proof_name = result['name']
name = result['name']
if 'normalized_name' in result:
proof_name = result['normalized_name']
name = result.get('name', name)
proof_name = result.get('normalized_name', name)
try:
verify_proof(result['proof'], claim_trie_root, proof_name)
except InvalidProofError:
@@ -392,48 +337,43 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
def validate_claim_signature_and_get_channel_name(claim_result, certificate_claim, ledger,
claim_tx=None, cert_tx=None):
valid_signature = False
if cert_tx and certificate_claim and claim_tx and claim_result:
tx = Transaction(unhexlify(claim_tx))
cert_tx = Transaction(unhexlify(cert_tx))
try:
is_signed = tx.outputs[claim_result['nout']].is_signed_by(
cert_tx.outputs[certificate_claim['nout']], ledger
)
except InvalidSignature:
return False, None
return is_signed, certificate_claim['name']
return False, None
valid_signature = claim_tx.outputs[claim_result['nout']].is_signed_by(
cert_tx.outputs[certificate_claim['nout']], ledger
)
if not valid_signature:
log.warning("lbry://%s#%s has an invalid signature",
claim_result['name'], claim_result['claim_id'])
return valid_signature
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
def _decode_claim_result(claim):
if 'has_signature' in claim and claim['has_signature']:
if not claim['signature_is_valid']:
log.warning("lbry://%s#%s has an invalid signature",
claim['name'], claim['claim_id'])
if 'decoded_claim' in claim:
return claim
if 'value' not in claim:
log.warning('Got an invalid claim while parsing, please report: %s', claim)
claim['hex'] = None
claim['protobuf'] = None
claim['value'] = None
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
backend_message = ' SDK message: ' + claim.get('error', '')
claim['error'] = "Failed to parse: missing value." + backend_message
return claim
try:
if not isinstance(claim['value'], Claim):
claim['value'] = Claim.from_bytes(claim['value'])
claim['hex'] = hexlify(claim['value'].to_bytes())
claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
claim['protobuf'] = hexlify(claim['value'].to_bytes())
claim['decoded_claim'] = True
except DecodeError:
claim['hex'] = claim['value']
claim['decoded_claim'] = False
claim['protobuf'] = claim['value']
claim['value'] = None
claim['error'] = "Failed to decode value"
return claim
def _handle_claim_result(results):
def _handle_claim_result(results, uri):
if not results:
#TODO: cannot determine what name we searched for here
# we should fix lbryum commands that return None
raise UnknownNameError("")
raise UnknownURI(uri)
if 'error' in results:
if results['error'] in ['name is not claimed', 'claim not found']:
@@ -446,26 +386,8 @@ def _handle_claim_result(results):
if 'outpoint' in results:
raise UnknownOutpoint(results['outpoint'])
raise Exception(results['error'])
# case where return value is {'certificate':{'txid', 'value',...},...}
if 'certificate' in results:
results['certificate'] = _decode_claim_result(results['certificate'])
# case where return value is {'claim':{'txid','value',...},...}
if 'claim' in results:
results['claim'] = _decode_claim_result(results['claim'])
# case where return value is {'txid','value',...}
# returned by queries that are not name resolve related
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
elif 'value' in results:
results = _decode_claim_result(results)
# case where there is no 'certificate', 'value', or 'claim' key
elif 'certificate' not in results:
msg = f'result in unexpected format:{results}'
assert False, msg
if not {'value', 'claim', 'certificate'}.intersection(results.keys()):
raise Exception(f'result in unexpected format:{results}')
return results
@@ -474,13 +396,9 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
# potentially triggering that.
winner = None
winner = {}
for claim in claims_in_channel:
if not claim['signature_is_valid']:
continue
if winner is None:
winner = claim
elif claim['height'] < winner['height'] or \
if not winner or claim['height'] < winner['height'] or \
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
winner = claim
return winner
winner = claim if claim['signature_is_valid'] else winner
return winner or None
+73 -124
View File
@@ -1,14 +1,61 @@
import struct
import msgpack
from lbrynet.wallet.transaction import Transaction, Output
from torba.server.hash import hash_to_hex_str
import time
from torba.server.block_processor import BlockProcessor
from lbrynet.schema.claim import Claim
from lbrynet.wallet.server.model import ClaimInfo
from lbrynet.schema.claim import Claim
from lbrynet.wallet.server.db import SQLDB
class Timer:
def __init__(self, name):
self.name = name
self.total = 0
self.count = 0
self.sub_timers = {}
self._last_start = None
def add_timer(self, name):
if name not in self.sub_timers:
self.sub_timers[name] = Timer(name)
return self.sub_timers[name]
def run(self, func, *args, forward_timer=False, timer_name=None, **kwargs):
t = self.add_timer(timer_name or func.__name__)
t.start()
try:
if forward_timer:
return func(*args, **kwargs, timer=t)
else:
return func(*args, **kwargs)
finally:
t.stop()
def start(self):
self._last_start = time.time()
return self
def stop(self):
self.total += (time.time() - self._last_start)
self.count += 1
self._last_start = None
return self
def show(self, depth=0, height=None):
if depth == 0:
print('='*100)
if height is not None:
print(f'STATISTICS AT HEIGHT {height}')
print('='*100)
else:
print(
f"{' '*depth} {self.total/60:4.2f}mins {self.name}"
# f"{self.total/self.count:.5f}sec/call, "
)
for sub_timer in self.sub_timers.values():
sub_timer.show(depth+1)
if depth == 0:
print('='*100)
class LBRYBlockProcessor(BlockProcessor):
@@ -17,126 +64,28 @@ class LBRYBlockProcessor(BlockProcessor):
super().__init__(*args, **kwargs)
if self.env.coin.NET == "regtest":
self.prefetcher.polling_delay = 0.5
self.should_validate_signatures = self.env.boolean('VALIDATE_CLAIM_SIGNATURES', False)
self.logger.info("LbryumX Block Processor - Validating signatures: {}".format(self.should_validate_signatures))
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
def advance_blocks(self, blocks):
# save height, advance blocks as usual, then hook our claim tx processing
height = self.height + 1
super().advance_blocks(blocks)
pending_undo = []
for index, block in enumerate(blocks):
undo = self.advance_claim_txs(block.transactions, height + index)
pending_undo.append((height+index, undo,))
self.db.write_undo(pending_undo)
self.sql.begin()
try:
self.timer.run(super().advance_blocks, blocks)
except:
self.logger.exception(f'Error while advancing transaction in new block.')
raise
finally:
self.sql.commit()
def advance_claim_txs(self, txs, height):
# TODO: generate claim undo info!
undo_info = []
add_undo = undo_info.append
update_inputs = set()
for etx, txid in txs:
update_inputs.clear()
tx = Transaction(etx.serialize())
for index, output in enumerate(tx.outputs):
if not output.is_claim:
continue
if output.script.is_claim_name:
add_undo(self.advance_claim_name_transaction(output, height, txid, index))
elif output.script.is_update_claim:
update_input = self.db.get_update_input(output.claim_hash, tx.inputs)
if update_input:
update_inputs.add(update_input)
add_undo(self.advance_update_claim(output, height, txid, index))
else:
info = (hash_to_hex_str(txid), output.claim_id,)
self.logger.error("REJECTED: {} updating {}".format(*info))
for txin in tx.inputs:
if txin not in update_inputs:
abandoned_claim_id = self.db.abandon_spent(txin.txo_ref.tx_ref.hash, txin.txo_ref.position)
if abandoned_claim_id:
add_undo((abandoned_claim_id, self.db.get_claim_info(abandoned_claim_id)))
return undo_info
def advance_update_claim(self, output: Output, height, txid, nout):
claim_id = output.claim_hash
claim_info = self.claim_info_from_output(output, txid, nout, height)
old_claim_info = self.db.get_claim_info(claim_id)
self.db.put_claim_id_for_outpoint(old_claim_info.txid, old_claim_info.nout, None)
if old_claim_info.cert_id:
self.db.remove_claim_from_certificate_claims(old_claim_info.cert_id, claim_id)
if claim_info.cert_id:
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
self.db.put_claim_info(claim_id, claim_info)
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
return claim_id, old_claim_info
def advance_claim_name_transaction(self, output: Output, height, txid, nout):
claim_id = output.claim_hash
claim_info = self.claim_info_from_output(output, txid, nout, height)
if claim_info.cert_id:
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
self.db.put_claim_info(claim_id, claim_info)
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
return claim_id, None
def backup_from_undo_info(self, claim_id, undo_claim_info):
"""
Undo information holds a claim state **before** a transaction changes it
There are 4 possibilities when processing it, of which only 3 are valid ones:
1. the claim is known and the undo info has info, it was an update
2. the claim is known and the undo info doesn't hold any info, it was claimed
3. the claim in unknown and the undo info has info, it was abandoned
4. the claim is unknown and the undo info does't hold info, error!
"""
undo_claim_info = ClaimInfo(*undo_claim_info) if undo_claim_info else None
current_claim_info = self.db.get_claim_info(claim_id)
if current_claim_info and undo_claim_info:
# update, remove current claim
self.db.remove_claim_id_for_outpoint(current_claim_info.txid, current_claim_info.nout)
if current_claim_info.cert_id:
self.db.remove_claim_from_certificate_claims(current_claim_info.cert_id, claim_id)
elif current_claim_info and not undo_claim_info:
# claim, abandon it
self.db.abandon_spent(current_claim_info.txid, current_claim_info.nout)
elif not current_claim_info and undo_claim_info:
# abandon, reclaim it (happens below)
pass
else:
# should never happen, unless the database got into an inconsistent state
raise Exception("Unexpected situation occurred on backup, this means the database is inconsistent. "
"Please report. Resetting the data folder (reindex) solves it for now.")
if undo_claim_info:
self.db.put_claim_info(claim_id, undo_claim_info)
if undo_claim_info.cert_id:
cert_id = self._checksig(undo_claim_info.value, undo_claim_info.address)
self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
def backup_txs(self, txs):
self.logger.info("Reorg at height {} with {} transactions.".format(self.height, len(txs)))
undo_info = msgpack.loads(self.db.claim_undo_db.get(struct.pack(">I", self.height)), use_list=False)
for claim_id, undo_claim_info in reversed(undo_info):
self.backup_from_undo_info(claim_id, undo_claim_info)
return super().backup_txs(txs)
def backup_blocks(self, raw_blocks):
self.db.batched_flush_claims()
super().backup_blocks(raw_blocks=raw_blocks)
self.db.batched_flush_claims()
async def flush(self, flush_utxos):
self.db.batched_flush_claims()
return await super().flush(flush_utxos)
def claim_info_from_output(self, output: Output, txid, nout, height):
address = self.coin.address_from_script(output.script.source)
name, value, cert_id = output.script.values['claim_name'], output.script.values['claim'], None
assert txid and address
cert_id = self._checksig(value, address)
return ClaimInfo(name, value, txid, nout, output.amount, address, height, cert_id)
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(20):
self.timer.show(height=height)
return undo
def _checksig(self, value, address):
try:
+21
View File
@@ -0,0 +1,21 @@
class FindShortestID:
__slots__ = 'short_id', 'new_id'
def __init__(self):
self.short_id = ''
self.new_id = None
def step(self, other_id, new_id):
self.new_id = new_id
for i in range(len(self.new_id)):
if other_id[i] != self.new_id[i]:
if i > len(self.short_id)-1:
self.short_id = self.new_id[:i+1]
break
def finalize(self):
return '#'+self.short_id
def register_canonical_functions(connection):
connection.create_aggregate("shortest_id", 2, FindShortestID)
-5
View File
@@ -53,8 +53,3 @@ class LBCDaemon(Daemon):
async def claimname(self, name, hexvalue, amount):
'''Claim a name, used for functional tests only.'''
return await self._send_single('claimname', (name, hexvalue, float(amount)))
@handles_errors
async def generate(self, number_of_blocks):
'''Generates regtest blocks, used for functional tests only.'''
return await self._send_single('generate', (int(number_of_blocks),))
File diff suppressed because it is too large. Load diff
-15
View File
@@ -1,15 +0,0 @@
from collections import namedtuple
import msgpack
# Classes representing data and their serializers, if any.
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
'''Claim information as its stored on database'''
@classmethod
def from_serialized(cls, serialized):
return cls(*msgpack.loads(serialized))
@property
def serialized(self):
return msgpack.dumps(self)
+28 -83
View File
@@ -7,7 +7,8 @@ from torba.server.hash import hash_to_hex_str
from torba.server.session import ElectrumX
from torba.server import util
from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH, URIParseError
from lbrynet.schema.result import Outputs
from lbrynet.schema.url import URL
from lbrynet.wallet.server.block_processor import LBRYBlockProcessor
from lbrynet.wallet.server.db import LBRYDB
@@ -22,13 +23,13 @@ class LBRYElectrumX(ElectrumX):
self.daemon = self.session_mgr.daemon
self.bp: LBRYBlockProcessor = self.session_mgr.bp
self.db: LBRYDB = self.bp.db
# fixme: lbryum specific subscribe
self.subscribe_height = False
def set_request_handlers(self, ptuple):
super().set_request_handlers(ptuple)
handlers = {
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getclaimbyid': self.claimtrie_getclaimbyid,
'blockchain.claimtrie.getclaimsforname': self.claimtrie_getclaimsforname,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
@@ -41,71 +42,16 @@ class LBRYElectrumX(ElectrumX):
'blockchain.claimtrie.getvaluesforuris': self.claimtrie_getvalueforuris,
'blockchain.claimtrie.getclaimssignedbyid': self.claimtrie_getclaimssignedbyid,
'blockchain.block.get_server_height': self.get_server_height,
'blockchain.block.get_block': self.get_block,
}
# fixme: methods we use but shouldnt be using anymore. To be removed when torba goes out
handlers.update({
'blockchain.numblocks.subscribe': self.numblocks_subscribe,
'blockchain.utxo.get_address': self.utxo_get_address,
'blockchain.transaction.broadcast':
self.transaction_broadcast_1_0,
'blockchain.transaction.get': self.transaction_get,
})
self.request_handlers.update(handlers)
async def utxo_get_address(self, tx_hash, index):
# fixme: lbryum
# Used only for electrum client command-line requests. We no
# longer index by address, so need to request the raw
# transaction. So it works for any TXO not just UTXOs.
self.assert_tx_hash(tx_hash)
try:
index = int(index)
if index < 0:
raise ValueError
except ValueError:
raise RPCError(1, "index has to be >= 0 and integer")
raw_tx = await self.daemon_request('getrawtransaction', tx_hash)
if not raw_tx:
return None
raw_tx = util.hex_to_bytes(raw_tx)
tx = self.coin.DESERIALIZER(raw_tx).read_tx()
if index >= len(tx.outputs):
return None
return self.coin.address_from_script(tx.outputs[index].pk_script)
async def claimtrie_search(self, **kwargs):
if 'claim_id' in kwargs:
self.assert_claim_id(kwargs['claim_id'])
return Outputs.to_base64(*self.db.sql.search(kwargs))
async def transaction_broadcast_1_0(self, raw_tx):
# fixme: lbryum
# An ugly API: current Electrum clients only pass the raw
# transaction in hex and expect error messages to be returned in
# the result field. And the server shouldn't be doing the client's
# user interface job here.
try:
return await self.transaction_broadcast(raw_tx)
except RPCError as e:
return e.message
async def numblocks_subscribe(self):
# fixme workaround for lbryum
'''Subscribe to get height of new blocks.'''
self.subscribe_height = True
return self.bp.height
async def notify(self, height, touched):
# fixme workaround for lbryum
await super().notify(height, touched)
if self.subscribe_height and height != self.notified_height:
self.send_notification('blockchain.numblocks.subscribe', (height,))
async def transaction_get(self, tx_hash, verbose=False):
# fixme: workaround for lbryum sending the height instead of True/False.
# fixme: lbryum_server ignored that and always used False, but this is out of spec
if verbose not in (True, False):
verbose = False
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def get_block(self, block_hash):
return await self.daemon.deserialised_block(block_hash)
async def claimtrie_resolve(self, *urls):
return Outputs.to_base64(*self.db.sql.resolve(urls))
async def get_server_height(self):
return self.bp.height
@@ -115,9 +61,7 @@ class LBRYElectrumX(ElectrumX):
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
if transaction_info and 'hex' in transaction_info and 'confirmations' in transaction_info:
# an unconfirmed transaction from lbrycrdd will not have a 'confirmations' field
height = self.db.db_height
height = height - transaction_info['confirmations']
return height
return (self.db.db_height - transaction_info['confirmations']) + 1
elif transaction_info and 'hex' in transaction_info:
return -1
return None
@@ -194,7 +138,7 @@ class LBRYElectrumX(ElectrumX):
claims = await self.daemon.getclaimsforname(name)
if claims:
claims['claims'] = [self.format_claim_from_daemon(claim, name) for claim in claims['claims']]
claims['supports_without_claims'] = claims['supports without claims']
claims['supports_without_claims'] = [] # fixme temporary
del claims['supports without claims']
claims['last_takeover_height'] = claims['nLastTakeoverHeight']
del claims['nLastTakeoverHeight']
@@ -221,14 +165,14 @@ class LBRYElectrumX(ElectrumX):
if 'name' in claim:
name = claim['name'].encode('ISO-8859-1').decode()
claim_id = claim['claimId']
raw_claim_id = unhexlify(claim_id)[::-1]
info = self.db.get_claim_info(raw_claim_id)
info = self.db.sql.get_claims(claim_id=claim['claimId'])
if not info:
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
return {}
address = info.address.decode()
supports = self.format_supports_from_daemon(claim.get('supports', []))
# fixme: temporary
#supports = self.format_supports_from_daemon(claim.get('supports', []))
supports = []
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
height = get_from_possible_keys(claim, 'height', 'nHeight')
@@ -241,7 +185,7 @@ class LBRYElectrumX(ElectrumX):
"txid": claim['txid'],
"nout": claim['n'],
"amount": amount,
"depth": self.db.db_height - height,
"depth": self.db.db_height - height + 1,
"height": height,
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
"address": address, # from index
@@ -310,21 +254,22 @@ class LBRYElectrumX(ElectrumX):
uri = uri
block_hash = block_hash
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
parsed_uri = URL.parse(uri)
except ValueError as err:
return {'error': err.args[0]}
result = {}
if parsed_uri.contains_channel:
if parsed_uri.has_channel:
certificate = None
# TODO: this is also done on the else, refactor
if parsed_uri.claim_id:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
if parsed_uri.channel.claim_id:
if len(parsed_uri.channel.claim_id) < 40:
certificate_info = self.claimtrie_getpartialmatch(
parsed_uri.channel.name, parsed_uri.channel.claim_id)
else:
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name):
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.channel.claim_id)
if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.channel.name):
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
elif parsed_uri.claim_sequence:
certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
@@ -355,7 +300,7 @@ class LBRYElectrumX(ElectrumX):
else:
claim = None
if parsed_uri.claim_id:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
if len(parsed_uri.claim_id) < 40:
claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
else:
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
+113
View File
@@ -0,0 +1,113 @@
from math import sqrt
TRENDING_WINDOW = 650 # number of blocks, ~24hr period
TRENDING_DATA_POINTS = 7 # WINDOW * DATA_POINTS = ~1 week worth of trending data
CREATE_TREND_TABLE = """
create table if not exists trend (
claim_hash bytes not null,
height integer not null,
amount integer not null,
primary key (claim_hash, height)
) without rowid;
"""
class ZScore:
__slots__ = 'count', 'total', 'power', 'last'
def __init__(self):
self.count = 0
self.total = 0
self.power = 0
self.last = None
def step(self, value):
if self.last is not None:
self.count += 1
self.total += self.last
self.power += self.last**2
self.last = value
@property
def mean(self):
return self.total / self.count
@property
def standard_deviation(self):
return sqrt((self.power / self.count) - self.mean**2)
def finalize(self):
if self.count == 0:
return self.last
return (self.last - self.mean) / (self.standard_deviation or 1)
def register_trending_functions(connection):
connection.create_aggregate("zscore", 1, ZScore)
def calculate_trending(db, height, is_first_sync, final_height):
# don't start tracking until we're at the end of initial sync
if is_first_sync and height < (final_height - (TRENDING_WINDOW*TRENDING_DATA_POINTS)):
return
if height % TRENDING_WINDOW != 0:
return
db.execute(f"""
DELETE FROM trend WHERE height < {height-(TRENDING_WINDOW*TRENDING_DATA_POINTS)}
""")
start = (height-TRENDING_WINDOW)+1
db.execute(f"""
INSERT INTO trend (claim_hash, height, amount)
SELECT claim_hash, {start}, COALESCE(
(SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
AND height >= {start}), 0
) AS support_sum
FROM claim WHERE support_sum > 0
""")
zscore = ZScore()
for (global_sum,) in db.execute("SELECT AVG(amount) FROM trend GROUP BY height"):
zscore.step(global_sum)
global_mean, global_deviation = 0, 1
if zscore.count > 0:
global_mean = zscore.mean
global_deviation = zscore.standard_deviation
db.execute(f"""
UPDATE claim SET
trending_local = COALESCE((
SELECT zscore(amount) FROM trend
WHERE claim_hash=claim.claim_hash ORDER BY height DESC
), 0),
trending_global = COALESCE((
SELECT (amount - {global_mean}) / {global_deviation} FROM trend
WHERE claim_hash=claim.claim_hash AND height = {start}
), 0),
trending_group = 0,
trending_mixed = 0
""")
# trending_group and trending_mixed determine how trending will show in query results
# normally the SQL will be: "ORDER BY trending_group, trending_mixed"
# changing the trending_group will have significant impact on trending results
# changing the value used for trending_mixed will only impact trending within a trending_group
db.execute(f"""
UPDATE claim SET
trending_group = CASE
WHEN trending_local > 0 AND trending_global > 0 THEN 4
WHEN trending_local <= 0 AND trending_global > 0 THEN 3
WHEN trending_local > 0 AND trending_global <= 0 THEN 2
WHEN trending_local <= 0 AND trending_global <= 0 THEN 1
END,
trending_mixed = CASE
WHEN trending_local > 0 AND trending_global > 0 THEN trending_global
WHEN trending_local <= 0 AND trending_global > 0 THEN trending_local
WHEN trending_local > 0 AND trending_global <= 0 THEN trending_local
WHEN trending_local <= 0 AND trending_global <= 0 THEN trending_global
END
WHERE trending_local <> 0 OR trending_global <> 0
""")
+57 -28
View File
@@ -9,11 +9,12 @@ from cryptography.hazmat.primitives.serialization import load_der_public_key
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
from ecdsa.util import sigencode_der
from cryptography.exceptions import InvalidSignature
from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput, ReadOnlyList
from torba.client.hash import hash160, sha256, Base58
from lbrynet.schema.claim import Claim
from lbrynet.schema.url import normalize_name
from lbrynet.wallet.account import Account
from lbrynet.wallet.script import InputScript, OutputScript
@@ -27,13 +28,14 @@ class Output(BaseOutput):
script: OutputScript
script_class = OutputScript
__slots__ = 'channel', 'private_key'
__slots__ = 'channel', 'private_key', 'meta'
def __init__(self, *args, channel: Optional['Output'] = None,
private_key: Optional[str] = None, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.channel = channel
self.private_key = private_key
self.meta = {}
def update_annotations(self, annotated):
super().update_annotations(annotated)
@@ -50,6 +52,10 @@ class Output(BaseOutput):
def is_claim(self) -> bool:
return self.script.is_claim_name or self.script.is_update_claim
@property
def is_support(self) -> bool:
return self.script.is_support_claim
@property
def claim_hash(self) -> bytes:
if self.script.is_claim_name:
@@ -69,6 +75,10 @@ class Output(BaseOutput):
return self.script.values['claim_name'].decode()
raise ValueError('No claim_name associated.')
@property
def normalized_name(self) -> str:
return normalize_name(self.claim_name)
@property
def claim(self) -> Claim:
if self.is_claim:
@@ -77,66 +87,83 @@ class Output(BaseOutput):
return self.script.values['claim']
raise ValueError('Only claim name and claim update have the claim payload.')
@property
def can_decode_claim(self):
try:
return self.claim
except:
return False
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
if self.channel is not None:
return "{}#{}/{}".format(
self.channel.claim_name,
self.channel.claim_id,
self.claim_name
)
return f"{self.claim_name}#{self.claim_id}"
return f"lbry://{self.claim_name}#{self.claim_id}"
raise ValueError('No claim associated.')
@property
def has_private_key(self):
return self.private_key is not None
def is_signed_by(self, channel: 'Output', ledger=None):
def get_signature_digest(self, ledger):
if self.claim.unsigned_payload:
pieces = [
Base58.decode(self.get_address(ledger)),
self.claim.unsigned_payload,
self.claim.signing_channel_hash
self.claim.signing_channel_hash[::-1]
]
else:
pieces = [
self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
self.tx_ref.tx.inputs[0].txo_ref.hash,
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]
digest = sha256(b''.join(pieces))
public_key = load_der_public_key(channel.claim.channel.public_key_bytes, default_backend())
hash = hashes.SHA256()
return sha256(b''.join(pieces))
def get_encoded_signature(self):
signature = hexlify(self.claim.signature)
r = int(signature[:int(len(signature)/2)], 16)
s = int(signature[int(len(signature)/2):], 16)
encoded_sig = sigencode_der(r, s, len(signature)*4)
public_key.verify(encoded_sig, digest, ec.ECDSA(Prehashed(hash)))
return True
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
@staticmethod
def is_signature_valid(encoded_signature, signature_digest, public_key_bytes):
try:
public_key = load_der_public_key(public_key_bytes, default_backend())
public_key.verify(encoded_signature, signature_digest, ec.ECDSA(Prehashed(hashes.SHA256())))
return True
except (ValueError, InvalidSignature):
pass
return False
def is_signed_by(self, channel: 'Output', ledger=None):
return self.is_signature_valid(
self.get_encoded_signature(),
self.get_signature_digest(ledger),
channel.claim.channel.public_key_bytes
)
def sign(self, channel: 'Output', first_input_id=None):
self.channel = channel
self.claim.signing_channel_hash = channel.claim_hash
digest = sha256(b''.join([
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]))
private_key = ecdsa.SigningKey.from_pem(channel.private_key, hashfunc=hashlib.sha256)
self.claim.signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.claim.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.script.generate()
def clear_signature(self):
self.channel = None
self.claim.clear_signature()
def generate_channel_private_key(self):
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
self.private_key = private_key.to_pem().decode()
self.claim.channel.public_key_bytes = private_key.get_verifying_key().to_der()
self.private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der()
self.script.generate()
return self.private_key
def is_channel_private_key(self, private_key_pem):
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=hashlib.sha256)
def is_channel_private_key(self, private_key):
return self.claim.channel.public_key_bytes == private_key.get_verifying_key().to_der()
@classmethod
@@ -194,15 +221,17 @@ class Transaction(BaseTransaction):
@classmethod
def claim_update(
cls, previous_claim: Output, amount: int, holding_address: str,
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
updated_claim = Output.pay_update_claim_pubkey_hash(
amount, previous_claim.claim_name, previous_claim.claim_id,
previous_claim.claim, ledger.address_to_hash160(holding_address)
claim, ledger.address_to_hash160(holding_address)
)
if signing_channel is not None:
updated_claim.sign(signing_channel, b'placeholder txid:nout')
else:
updated_claim.clear_signature()
return cls.create(
[Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account, sign=False
)
+1 -1
View File
@@ -24,6 +24,6 @@ theme:
- type: 'telegram'
link: 'https://t.me/lbryofficial'
- type: 'discord'
link: 'https://chat.lbry.io'
link: 'https://chat.lbry.com'
- type: 'reddit'
link: 'https://reddit.com/r/lbry'
+3 -1
View File
@@ -1,5 +1,7 @@
FROM debian:buster-slim
ARG TORBA_VERSION=master
RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
@@ -15,7 +17,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools wheel
COPY . /app
WORKDIR /app
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
RUN python3.7 -m pip install -e .
# Orchstr8 API
+6 -1
View File
@@ -1,6 +1,7 @@
FROM debian:buster-slim
ARG user=lbry
ARG TORBA_VERSION=master
# create an unprivileged user
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
@@ -23,7 +24,10 @@ WORKDIR /home/$user
RUN python3.7 -m pip install --upgrade pip setuptools
# get torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
# get uvloop
RUN python3.7 -m pip install --user uvloop
# copy lbrynet
RUN mkdir projects/
@@ -46,4 +50,5 @@ ENV DB_DIRECTORY=$db_dir
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
ENV MAX_SESSIONS=1000000000
ENV MAX_SEND=1000000000000000000
ENV EVENT_LOOP_POLICY=uvloop
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
+75
View File
@@ -0,0 +1,75 @@
import sys
import asyncio
from binascii import hexlify
from lbrynet.wallet.server.db import SQLDB
from lbrynet.wallet.server.coin import LBC
from lbrynet.wallet.server.daemon import Daemon
def hex_reverted(value: bytes) -> str:
return hexlify(value[::-1]).decode()
def match(name, what, value, expected):
if value != expected:
print(f'{name}: {what} mismatch, {value} is not {expected}')
return value == expected
def checkrecord(record, expected_winner, expected_claim):
assert record['is_controlling'] == record['claim_hash'], dict(record)
name = record['normalized']
claim_id = hex_reverted(record['claim_hash'])
takover = record['activation_height']
if not expected_winner:
print(f"{name} not on lbrycrd. We have {claim_id} at {takover} takeover height.")
return
if not match(name, 'claim id', claim_id, expected_winner['claimId']):
print(f"-- {name} has the wrong winner")
if not expected_claim:
print(f'{name}: {claim_id} not found, we possibly have an abandoned claim as winner')
return
match(name, 'height', record['height'], expected_claim['height'])
match(name, 'activation height', takover, expected_claim['valid at height'])
match(name, 'name', record['normalized'], expected_claim['normalized_name'])
match(name, 'amount', record['amount'], expected_claim['amount'])
match(name, 'effective amount', record['effective_amount'], expected_claim['effective amount'])
match(name, 'txid', hex_reverted(record['txo_hash'][:-4]), expected_claim['txid'])
match(name, 'nout', int.from_bytes(record['txo_hash'][-4:], 'little', signed=False), expected_claim['n'])
async def checkcontrolling(daemon: Daemon, db: SQLDB):
records, claim_ids, names, futs = [], [], [], []
for record in db.get_claims('claimtrie.claim_hash as is_controlling, claim.*', is_controlling=True):
records.append(record)
claim_id = hex_reverted(record['claim_hash'])
claim_ids.append((claim_id,))
names.append((record['normalized'],))
if len(names) > 50000:
futs.append(daemon._send_vector('getvalueforname', names[:]))
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
names.clear()
claim_ids.clear()
if names:
futs.append(daemon._send_vector('getvalueforname', names[:]))
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
names.clear()
claim_ids.clear()
while futs:
winners, claims = futs.pop(0), futs.pop(0)
for winner, claim in zip(await winners, await claims):
checkrecord(records.pop(0), winner, claim)
if __name__ == '__main__':
if not len(sys.argv) == 3:
print("usage: <db_file_path> <lbrycrd_url>")
sys.exit(1)
db_path, lbrycrd_url = sys.argv[1:] # pylint: disable=W0632
daemon = Daemon(LBC(), url=lbrycrd_url)
db = SQLDB(None, db_path)
db.open()
asyncio.get_event_loop().run_until_complete(checkcontrolling(daemon, db))
+3 -3
View File
@@ -5,7 +5,7 @@ import socket
import ipaddress
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
import logging
@@ -32,7 +32,7 @@ async def main(blob_hash: str, url: str):
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
await storage.open()
await blob_manager.setup()
@@ -47,7 +47,7 @@ async def main(blob_hash: str, url: str):
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
url = 'reflector.lbry.io:5567'
url = 'reflector.lbry.com:5567'
if len(sys.argv) > 2:
url = sys.argv[2]
asyncio.run(main(sys.argv[1], url))
+418 -7
View File
@@ -2,7 +2,380 @@ import os
import re
import json
import inspect
from lbrynet.extras.daemon.Daemon import Daemon
import tempfile
import asyncio
import time
from docopt import docopt
from binascii import unhexlify
from textwrap import indent
from lbrynet.extras.cli import set_kwargs, get_argument_parser
from lbrynet.extras.daemon.Daemon import (
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
)
from lbrynet.extras.daemon.json_response_encoder import (
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
)
from lbrynet.testcase import CommandTestCase
RETURN_DOCS = {
'Account': encode_account_doc(),
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
'Address': 'an address in base58'
}
class ExampleRecorder:
def __init__(self, test):
self.test = test
self.examples = {}
async def __call__(self, title, *command):
parser = get_argument_parser()
args, command_args = parser.parse_known_args(command)
api_method_name = args.api_method_name
parsed = docopt(args.doc, command_args)
kwargs = set_kwargs(parsed)
for k, v in kwargs.items():
if v and isinstance(v, str) and (v[0], v[-1]) == ('"', '"'):
kwargs[k] = v[1:-1]
params = json.dumps({"method": api_method_name, "params": kwargs})
method = getattr(self.test.daemon, f'jsonrpc_{api_method_name}')
result = method(**kwargs)
if asyncio.iscoroutine(result):
result = await result
output = jsonrpc_dumps_pretty(result, ledger=self.test.daemon.ledger)
self.examples.setdefault(api_method_name, []).append({
'title': title,
'curl': f"curl -d'{params}' http://localhost:5279/",
'lbrynet': 'lbrynet ' + ' '.join(command),
'python': f'requests.post("http://localhost:5279", json={params}).json()',
'output': output.strip()
})
return json.loads(output)['result']
class Examples(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.recorder = ExampleRecorder(self)
async def play(self):
r = self.recorder
# general sdk
await r(
'Get status',
'status'
)
await r(
'Get version',
'version'
)
await r(
'Get settings',
'settings', 'get'
)
# accounts
await r(
'List your accounts',
'account', 'list'
)
account = await r(
'Create an account',
'account', 'create', '"generated account"'
)
await r(
'Remove an account',
'account', 'remove', account['id']
)
await r(
'Add an account from seed',
'account', 'add', '"new account"', f"--seed=\"{account['seed']}\""
)
await r(
'Modify maximum number of times a change address can be reused',
'account', 'set', account['id'], '--change_max_uses=10'
)
# addresses
await r(
'List addresses in default account',
'address', 'list'
)
an_address = await r(
'Get an unused address',
'address', 'unused'
)
address_list_by_id = await r(
'List addresses in specified account',
'address', 'list', f"--account_id=\"{account['id']}\""
)
await r(
'Check if address is mine',
'address', 'is_mine', an_address
)
# sends/funds
transfer = await r(
'Transfer 2 LBC from default account to specific account',
'account', 'fund', f"--to_account=\"{account['id']}\"", "--amount=2.0", "--broadcast"
)
await self.on_transaction_dict(transfer)
await self.generate(1)
await self.on_transaction_dict(transfer)
await r(
'Get default account balance',
'account', 'balance'
)
txlist = await r(
'List your transactions',
'transaction', 'list'
)
await r(
'Get balance for specific account by id',
'account', 'balance', f"\"{account['id']}\""
)
spread_transaction = await r(
'Spread LBC between multiple addresses',
'account', 'fund', f"--to_account=\"{account['id']}\"", f"--from_account=\"{account['id']}\"", '--amount=1.5', '--outputs=2', '--broadcast'
)
await self.on_transaction_dict(spread_transaction)
await self.generate(1)
await self.on_transaction_dict(spread_transaction)
await r(
'Transfer all LBC to a specified account',
'account', 'fund', f"--from_account=\"{account['id']}\"", "--everything", "--broadcast"
)
# channels
channel = await r(
'Create a channel claim without metadata',
'channel', 'create', '@channel', '1.0'
)
channel_id = channel['outputs'][0]['claim_id']
await self.on_transaction_dict(channel)
await self.generate(1)
await self.on_transaction_dict(channel)
await r(
'List your channel claims',
'channel', 'list'
)
await r(
'Paginate your channel claims',
'channel', 'list', '--page=1', '--page_size=20'
)
channel = await r(
'Update a channel claim',
'channel', 'update', channel['outputs'][0]['claim_id'], '--title="New Channel"'
)
await self.on_transaction_dict(channel)
await self.generate(1)
await self.on_transaction_dict(channel)
big_channel = await r(
'Create a channel claim with all metadata',
'channel', 'create', '@bigchannel', '1.0',
'--title="Big Channel"', '--description="A channel with lots of videos."',
'--email="creator@smallmedia.com"', '--tags=music', '--tags=art',
'--languages=pt-BR', '--languages=uk', '--locations=BR', '--locations=UA::Kiyv',
'--website_url="http://smallmedia.com"', '--thumbnail_url="http://smallmedia.com/logo.jpg"',
'--cover_url="http://smallmedia.com/logo.jpg"'
)
await self.on_transaction_dict(big_channel)
await self.generate(1)
await self.on_transaction_dict(big_channel)
await self.daemon.jsonrpc_channel_abandon(big_channel['outputs'][0]['claim_id'])
await self.generate(1)
# stream claims
with tempfile.NamedTemporaryFile() as file:
file.write(b'hello world')
file.flush()
stream = await r(
'Create a stream claim without metadata',
'stream', 'create', 'astream', '1.0', file.name
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
stream_id = stream['outputs'][0]['claim_id']
stream_name = stream['outputs'][0]['name']
stream = await r(
'Update a stream claim to add channel',
'stream', 'update', stream_id,
f'--channel_id="{channel_id}"'
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
await r(
'List all your claims',
'claim', 'list'
)
await r(
'Paginate your claims',
'claim', 'list', '--page=1', '--page_size=20'
)
await r(
'List all your stream claims',
'stream', 'list'
)
await r(
'Paginate your stream claims',
'stream', 'list', '--page=1', '--page_size=20'
)
await r(
'Search for all claims in channel',
'claim', 'search', '--channel=@channel'
)
await r(
'Search for claims matching a name',
'claim', 'search', f'--name="{stream_name}"'
)
with tempfile.NamedTemporaryFile(suffix='.png') as file:
file.write(unhexlify(
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
b'9454e44ae426082'
))
file.flush()
big_stream = await r(
'Create an image stream claim with all metadata and fee',
'stream', 'create', 'blank-image', '1.0', file.name,
'--tags=blank', '--tags=art', '--languages=en', '--locations=US:NH:Manchester',
'--fee_currency=LBC', '--fee_amount=0.3',
'--title="Blank Image"', '--description="A blank PNG that is 5x7."', '--author=Picaso',
'--license="Public Domain"', '--license_url=http://public-domain.org',
'--thumbnail_url="http://smallmedia.com/thumbnail.jpg"', f'--release_time={int(time.time())}',
f'--channel_id="{channel_id}"'
)
await self.on_transaction_dict(big_stream)
await self.generate(1)
await self.on_transaction_dict(big_stream)
await self.daemon.jsonrpc_channel_abandon(big_stream['outputs'][0]['claim_id'])
await self.generate(1)
# files
file_list_result = await r(
'List local files',
'file', 'list'
)
file_uri = f"{file_list_result[0]['claim_name']}#{file_list_result[0]['claim_id']}"
await r(
'Resolve a claim',
'resolve', file_uri
)
await r(
'List files matching a parameter',
'file', 'list', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
)
await r(
'Delete a file',
'file', 'delete', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
)
await r(
'Get a file',
'get', file_uri
)
await r(
'Save a file to the downloads directory',
'file', 'save', f"--sd_hash=\"{file_list_result[0]['sd_hash']}\""
)
# blobs
bloblist = await r(
'List your local blobs',
'blob', 'list'
)
await r(
'Delete a blob',
'blob', 'delete', f"{bloblist[0]}"
)
# abandon all the things
abandon_stream = await r(
'Abandon a stream claim',
'stream', 'abandon', stream_id
)
await self.on_transaction_dict(abandon_stream)
await self.generate(1)
await self.on_transaction_dict(abandon_stream)
abandon_channel = await r(
'Abandon a channel claim',
'channel', 'abandon', channel_id
)
await self.on_transaction_dict(abandon_channel)
await self.generate(1)
await self.on_transaction_dict(abandon_channel)
with tempfile.NamedTemporaryFile() as file:
file.write(b'hello world')
file.flush()
stream = await r(
'Publish a file',
'publish', 'a-new-stream', '--bid=1.0', f'--file_path={file.name}'
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
def get_examples():
player = Examples('play')
result = player.run()
if result.errors:
for error in result.errors:
print(error[1])
raise Exception('See above for errors while running the examples.')
return player.recorder.examples
SECTIONS = re.compile("(.*?)Usage:(.*?)Options:(.*?)Returns:(.*)", re.DOTALL)
@@ -11,7 +384,27 @@ ARGUMENT_NAME = re.compile("--([^=]+)")
ARGUMENT_TYPE = re.compile("\s*\((.*?)\)(.*)")
def get_api(obj):
def get_return_def(returns):
result = returns.strip()
if (result[0], result[-1]) == ('{', '}'):
obj_type = result[1:-1]
if '[' in obj_type:
sub_type = obj_type[obj_type.index('[')+1:-1]
obj_type = obj_type[:obj_type.index('[')]
if obj_type == 'Paginated':
obj_def = encode_pagination_doc(RETURN_DOCS[sub_type])
elif obj_type == 'List':
obj_def = [RETURN_DOCS[sub_type]]
else:
raise NameError(f'Unknown return type: {obj_type}')
else:
obj_def = RETURN_DOCS[obj_type]
return indent(json.dumps(obj_def, indent=4), ' '*12)
return result
def get_api(name, examples):
obj = Daemon.callable_methods[name]
docstr = inspect.getdoc(obj).strip()
try:
@@ -45,17 +438,35 @@ def get_api(obj):
arg['description'] = ' '.join(arg['description'])
return {
'name': obj.__name__[len('jsonrpc_'):],
'name': name,
'description': description.strip(),
'arguments': arguments,
'returns': returns.strip()
'returns': get_return_def(returns),
'examples': examples
}
def write_api(f):
apis = []
for method_name in sorted(Daemon.callable_methods.keys()):
apis.append(get_api(Daemon.callable_methods[method_name]))
examples = get_examples()
api_definitions = Daemon.get_api_definitions()
apis = {
'main': {
'doc': 'Ungrouped commands.',
'commands': []
}
}
for group_name, group_doc in api_definitions['groups'].items():
apis[group_name] = {
'doc': group_doc,
'commands': []
}
for method_name, command in api_definitions['commands'].items():
if 'replaced_by' in command:
continue
apis[command['group'] or 'main']['commands'].append(get_api(
method_name,
examples.get(method_name, [])
))
json.dump(apis, f, indent=4)
+26 -24
View File
@@ -1,33 +1,35 @@
"""Set the build version to be 'qa', 'rc', 'release'"""
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
import os.path
import re
import subprocess
import sys
import os
import re
import logging
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def get_build_type(travis_tag=None):
if not travis_tag:
return "qa"
log.debug("getting build type for tag: \"%s\"", travis_tag)
if re.match('v\d+\.\d+\.\d+rc\d+$', travis_tag):
return 'rc'
elif re.match('v\d+\.\d+\.\d+$', travis_tag):
return 'release'
return 'qa'
def main():
build = get_build()
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
f.write("BUILD = '{}'\n".format(build))
def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
print('Build: rc')
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
print('Build: release')
return 'release'
print('Build: qa')
return 'qa'
except subprocess.CalledProcessError:
print("Couldn't determine build type, defaulting to qa.")
return 'qa'
build_type_path = os.path.join(root_dir, 'lbrynet', 'build_type.py')
log.debug("configuring build type file: %s", build_type_path)
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
with open(build_type_path, 'w') as f:
f.write("BUILD = \"{}\"\nBUILD_COMMIT = \"{}\"\n".format(build_type, travis_commit))
if __name__ == '__main__':
+2 -2
View File
@@ -1,7 +1,7 @@
import sys
import os
import asyncio
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.schema.address import decode_address
from lbrynet.extras.daemon.storage import SQLiteStorage
@@ -17,7 +17,7 @@ async def main(address: str):
storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite"))
await storage.open()
blob_manager = BlobFileManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
await blob_manager.setup()
server = await loop.create_server(
+43 -33
View File
@@ -11,7 +11,7 @@ from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.conf import Config
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.client import daemon_rpc
from lbrynet.extras import system_info, cli
from lbrynet.extras import system_info
def extract_uris(response):
@@ -26,7 +26,7 @@ def extract_uris(response):
async def get_frontpage_uris():
session = aiohttp.ClientSession()
try:
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
if response.status != 200:
print("API returned non 200 code!!")
return
@@ -58,7 +58,7 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, uri):
async def wait_for_done(conf, uri, timeout):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
@@ -73,11 +73,11 @@ async def wait_for_done(conf, uri):
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > 10:
if hang_count > timeout:
return False, file['blobs_completed'], file['blobs_in_stream']
async def main(uris=None, allow_fees=False):
async def main(uris=None, cmd_args=None):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
@@ -90,15 +90,16 @@ async def main(uris=None, allow_fees=False):
print("**********************************************")
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', name)
async def __resolve(name):
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
if 'error' not in resolved.get(name, {}):
if ("fee" not in resolved[name]['claim']['value']['stream']['metadata']) or allow_fees:
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
await asyncio.gather(*(__resolve(name) for name in uris))
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
first_byte_times = []
@@ -113,24 +114,29 @@ async def main(uris=None, allow_fees=False):
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri)
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except:
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, uri, cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except Exception as e:
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
failed_to_start.append(uri)
return
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
if cmd_args.exit_on_error:
return
if cmd_args.delete_after_download or cmd_args.head_blob_only:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
@@ -139,12 +145,13 @@ async def main(uris=None, allow_fees=False):
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
f"Variance: {variance(first_byte_times)}\n"
if not cmd_args.head_blob_only:
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
@@ -159,9 +166,12 @@ async def main(uris=None, allow_fees=False):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir")
parser.add_argument("--wallet_dir")
parser.add_argument("--download_directory")
#parser.add_argument("--data_dir")
#parser.add_argument("--wallet_dir")
#parser.add_argument("--download_directory")
parser.add_argument("--allow_fees", action='store_true')
args = parser.parse_args()
asyncio.run(main(allow_fees=args.allow_fees))
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=10, type=int)
parser.add_argument("--delete_after_download", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
asyncio.run(main(cmd_args=parser.parse_args()))
+9 -3
View File
@@ -1,13 +1,12 @@
set -x
# must also be updated in travis.yml
TORBA_VERSION=master
rm -rf /tmp/.wine-*
apt-get -qq update
apt-get -qq install -y git
pip install setuptools_scm
git clone https://github.com/lbryio/torba.git --depth 1
cd torba && pip install -e . && cd ..
cd lbry
@@ -15,6 +14,13 @@ cd lbry
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
git clone --depth=1 --single-branch --branch ${TORBA_VERSION} https://github.com/lbryio/torba.git
cd torba
pip install .
cd ..
rm -rf torba
pip show torba
pip install -e .
pip install pywin32
+5 -5
View File
@@ -9,22 +9,23 @@ setup(
name=__name__,
version=__version__,
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
author_email="hello@lbry.com",
url="https://lbry.com",
description="A decentralized media library and marketplace",
long_description=open(README_PATH, encoding='utf-8').read(),
long_description_content_type="text/markdown",
keywords="lbry protocol media",
license='MIT',
python_requires='>=3.6',
python_requires='>=3.7',
packages=find_packages(exclude=('tests',)),
zip_safe=False,
entry_points={
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
},
install_requires=[
'torba==0.5.5',
'aiohttp==3.5.4',
'aioupnp',
'aioupnp==0.0.12',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
@@ -35,7 +36,6 @@ setup(
'protobuf==3.6.1',
'msgpack==0.6.1',
'ecdsa==0.13',
'torba',
'pyyaml==4.2b1',
'docopt==0.6.2',
'hachoir',
+1
View File
@@ -61,6 +61,7 @@ def mock_network_loop(loop: asyncio.BaseEventLoop):
protocol = proto_lam()
transport = asyncio.DatagramTransport(extra={'socket': mock_sock})
transport.is_closing = lambda: False
transport.close = lambda: mock_sock.close()
mock_sock.sendto = sendto
transport.sendto = mock_sock.sendto
+1 -1
View File
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class AccountManagement(CommandTestCase):
+23 -26
View File
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class EpicAdventuresOfChris45(CommandTestCase):
@@ -39,9 +39,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
self.assertEqual(result, '8.989893')
# And is the channel resolvable and empty?
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam'))
self.assertIn('lbry://@spam', response)
self.assertIn('certificate', response['lbry://@spam'])
response = await self.resolve('lbry://@spam')
self.assertEqual(response['lbry://@spam']['value_type'], 'channel')
# "What goes well with spam?" ponders Chris...
# "A hovercraft with eels!" he exclaims.
@@ -64,9 +63,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
# Also checks that his new story can be found on the blockchain before
# giving the link to all his friends.
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
self.assertIn('lbry://@spam/hovercraft', response)
self.assertIn('claim', response['lbry://@spam/hovercraft'])
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(response['lbry://@spam/hovercraft']['value_type'], 'stream')
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
await self.generate(5)
@@ -81,23 +79,26 @@ class EpicAdventuresOfChris45(CommandTestCase):
# After some soul searching Chris decides that his story needs more
# heart and a better ending. He takes down the story and begins the rewrite.
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim_id, blocking=False))
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=False))
self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id)
await self.confirm_tx(abandon['txid'])
# And now checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
response = await self.resolve('lbry://@spam/hovercraft')
self.assertEqual(
response['lbry://@spam/hovercraft'],
{'error': 'lbry://@spam/hovercraft did not resolve to a claim'}
)
# After abandoning he just waits for his LBCs to be returned to his account
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.969379')
self.assertEqual(result, '8.9693455')
# Amidst all this Chris receives a call from his friend Ramsey
# who says that it is of utmost urgency that Chris transfer him
# 1 LBC to which Chris readily obliges
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id']
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address))
self.assertIn("txid", result)
@@ -108,7 +109,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
# Chris' balance was correct
self.assertEqual(result, '7.969255')
self.assertEqual(result, '7.9692215')
# Ramsey too assured him that he had received the 1 LBC and thanks him
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
@@ -134,12 +135,10 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(tx['txid'])
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
resolve_result = await self.resolve(uri)
# It obviously did! Because, blockchain baby \O/
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
self.assertEqual(resolve_result[uri]['amount'], '1.0')
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.2')
await self.generate(5)
# Now he also wanted to support the original creator of the Award Winning Novel
@@ -150,10 +149,9 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.confirm_tx(tx['txid'])
# And again checks if it went to the just right place
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
resolve_result = await self.resolve(uri)
# Which it obviously did. Because....?????
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.5')
await self.generate(5)
# Seeing the ravishing success of his novel Chris adds support to his claim too
@@ -163,8 +161,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
# It did!
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.9')
await self.generate(5)
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
@@ -183,10 +180,10 @@ class EpicAdventuresOfChris45(CommandTestCase):
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=tx['txid'], nout=0, blocking=False))
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=False))
self.assertTrue(abandon['inputs'][0]['txid'], tx['txid'])
await self.confirm_tx(abandon['txid'])
# He them checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve(uri))
self.assertNotIn('claim', response[uri])
response = await self.resolve(uri)
self.assertEqual(response[uri], {'error': f'{uri} did not resolve to a claim'})
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -29,7 +29,7 @@ class CLIIntegrationTest(AsyncioTestCase):
await self.daemon.start()
async def asyncTearDown(self):
await self.daemon.stop()
await self.daemon.stop(shutdown_runner=False)
def test_cli_status_command_with_auth(self):
actual_output = StringIO()
+105
View File
@@ -0,0 +1,105 @@
import asyncio
from aiohttp import web
from lbrynet.testcase import CommandTestCase
class FakedCommentServer:
ERRORS = {
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
self.app.add_routes([web.post('/api', self.api)])
self.runner = None
self.server = None
self.comments = []
self.comment_id = 0
def create_comment(self, **comment):
self.comment_id += 1
comment['comment_id'] = self.comment_id
self.comments.append(comment)
return comment
def get_claim_comments(self, page=1, page_size=50, **kwargs):
return self.comments[:page_size]
methods = {
'get_claim_comments': get_claim_comments,
'create_comment': create_comment,
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
if body['method'] in self.methods:
params = body.get('params', {})
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
return response
async def start(self):
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.server = web.TCPSite(self.runner, 'localhost', self.port)
await self.server.start()
async def stop(self):
await self.runner.shutdown()
await self.runner.cleanup()
async def api(self, request):
body = await request.json()
if type(body) is list or type(body) is dict:
if type(body) is list:
response = [self.process_json(part) for part in body]
else:
response = self.process_json(body)
return web.json_response(response)
else:
raise TypeError('invalid type passed')
class CommentCommands(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon.conf.comment_server = 'http://localhost:2903/api'
self.comment_server = FakedCommentServer(2903)
await self.comment_server.start()
self.addCleanup(self.comment_server.stop)
async def test_comment_create(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
self.assertEqual(0, len(await self.daemon.jsonrpc_comment_list(stream['claim_id'])))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
comment="It's 5 O'Clock Somewhere"
)
comments = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
self.assertEqual(1, len(comments))
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_name=channel2['name'],
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
self.assertEqual(2, len(comments))
self.assertEqual(comments[1]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['comment_id'], comments[1]['parent_id'])
+115
View File
@@ -0,0 +1,115 @@
import asyncio
from binascii import hexlify
from lbrynet.dht import constants
from lbrynet.dht.node import Node
from lbrynet.dht.peer import PeerManager, KademliaPeer
from torba.testcase import AsyncioTestCase
class DHTIntegrationTest(AsyncioTestCase):
async def asyncSetUp(self):
import logging
logging.getLogger('asyncio').setLevel(logging.ERROR)
logging.getLogger('lbrynet.dht').setLevel(logging.WARN)
self.nodes = []
self.known_node_addresses = []
async def setup_network(self, size: int, start_port=40000, seed_nodes=1):
for i in range(size):
node_port = start_port + i
node = Node(self.loop, PeerManager(self.loop), node_id=constants.generate_id(i),
udp_port=node_port, internal_udp_port=node_port,
peer_port=3333, external_ip='127.0.0.1')
self.nodes.append(node)
self.known_node_addresses.append(('127.0.0.1', node_port))
await node.start_listening('127.0.0.1')
self.addCleanup(node.stop)
for node in self.nodes:
node.protocol.rpc_timeout = .5
node.protocol.ping_queue._default_delay = .5
node.start('127.0.0.1', self.known_node_addresses[:seed_nodes])
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
async def test_replace_bad_nodes(self):
await self.setup_network(20)
self.assertEquals(len(self.nodes), 20)
node = self.nodes[0]
bad_peers = []
for candidate in self.nodes[1:10]:
address, port, node_id = candidate.protocol.external_ip, candidate.protocol.udp_port, candidate.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
bad_peers.append(peer)
node.protocol.add_peer(peer)
candidate.stop()
await asyncio.sleep(.3) # let pending events settle
for bad_peer in bad_peers:
self.assertIn(bad_peer, node.protocol.routing_table.get_peers())
await node.refresh_node(True)
await asyncio.sleep(.3) # let pending events settle
good_nodes = {good_node.protocol.node_id for good_node in self.nodes[10:]}
for peer in node.protocol.routing_table.get_peers():
self.assertIn(peer.node_id, good_nodes)
async def test_re_join(self):
await self.setup_network(20, seed_nodes=10)
node = self.nodes[-1]
self.assertTrue(node.joined.is_set())
self.assertTrue(node.protocol.routing_table.get_peers())
for network_node in self.nodes[:-1]:
network_node.stop()
await node.refresh_node(True)
await asyncio.sleep(.3) # let pending events settle
self.assertFalse(node.protocol.routing_table.get_peers())
for network_node in self.nodes[:-1]:
network_node.start('127.0.0.1', self.known_node_addresses)
self.assertFalse(node.protocol.routing_table.get_peers())
timeout = 20
while not node.protocol.routing_table.get_peers():
await asyncio.sleep(.1)
timeout -= 1
if not timeout:
self.fail("node didnt join back after 2 seconds")
async def test_announce_no_peers(self):
await self.setup_network(1)
node = self.nodes[0]
blob_hash = hexlify(constants.generate_id(1337)).decode()
peers = await node.announce_blob(blob_hash)
self.assertEqual(len(peers), 0)
async def test_get_token_on_announce(self):
await self.setup_network(2, seed_nodes=2)
node1, node2 = self.nodes
node1.protocol.peer_manager.clear_token(node2.protocol.node_id)
blob_hash = hexlify(constants.generate_id(1337)).decode()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
node2.protocol.node_rpc.refresh_token()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
node2.protocol.node_rpc.refresh_token()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
async def test_peer_search_removes_bad_peers(self):
# that's an edge case discovered by Tom, but an important one
# imagine that you only got bad peers and refresh will happen in one hour
# instead of failing for one hour we should be able to recover by scheduling pings to bad peers we find
await self.setup_network(2, seed_nodes=2)
node1, node2 = self.nodes
node2.stop()
# forcefully make it a bad peer but dont remove it from routing table
address, port, node_id = node2.protocol.external_ip, node2.protocol.udp_port, node2.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
self.assertTrue(node1.protocol.peer_manager.peer_is_good(peer))
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
self.assertFalse(node1.protocol.peer_manager.peer_is_good(peer))
# now a search happens, which removes bad peers while contacting them
self.assertTrue(node1.protocol.routing_table.get_peers())
await node1.peer_search(node2.protocol.node_id)
await asyncio.sleep(.3) # let pending events settle
self.assertFalse(node1.protocol.routing_table.get_peers())
+106 -38
View File
@@ -1,8 +1,10 @@
import asyncio
import logging
import os
from binascii import unhexlify, hexlify
from integration.testcase import CommandTestCase
from lbrynet.schema import Claim
from lbrynet.testcase import CommandTestCase
from lbrynet.blob_exchange.downloader import BlobDownloader
@@ -14,7 +16,7 @@ class FileCommands(CommandTestCase):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
file1, file2 = self.sout(self.daemon.jsonrpc_file_list('claim_name'))
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
@@ -26,30 +28,90 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_get('lbry://foo')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
async def test_announces(self):
# announces on publish
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
await self.stream_create('foo', '0.01')
stream = self.daemon.jsonrpc_file_list()[0]
self.assertSetEqual(
set(await self.daemon.storage.get_blobs_to_announce()),
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
)
self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True))
# announces on download
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
stream = await self.daemon.jsonrpc_get('foo')
self.assertSetEqual(
set(await self.daemon.storage.get_blobs_to_announce()),
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
)
async def test_file_list_fields(self):
await self.stream_create('foo', '0.01')
file_list = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(
file_list[0]['timestamp'],
None
)
self.assertEqual(file_list[0]['confirmations'], -1)
await self.daemon.jsonrpc_resolve('foo')
file_list = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(
file_list[0]['timestamp'],
self.ledger.headers[file_list[0]['height']]['timestamp']
)
self.assertEqual(file_list[0]['confirmations'], 1)
async def test_get_doesnt_touch_user_written_files_between_calls(self):
await self.stream_create('foo', '0.01', data=bytes([0] * (2 << 23)))
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
first_path = (await self.daemon.jsonrpc_get('lbry://foo', save_file=True)).full_path
await self.wait_files_to_complete()
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
with open(first_path, 'wb') as f:
f.write(b' ')
f.flush()
second_path = await self.daemon.jsonrpc_get('lbry://foo', save_file=True)
await self.wait_files_to_complete()
self.assertNotEquals(first_path, second_path)
async def test_file_list_updated_metadata_on_resolve(self):
await self.stream_create('foo', '0.01')
txo = (await self.daemon.resolve(['lbry://foo']))['lbry://foo']
claim = txo.claim
await self.daemon.jsonrpc_file_delete(claim_name='foo')
txid = await self.blockchain_claim_name('bar', hexlify(claim.to_bytes()).decode(), '0.01')
await self.daemon.jsonrpc_get('lbry://bar')
claim.stream.description = "fix typos, fix the world"
await self.blockchain_update_name(txid, hexlify(claim.to_bytes()).decode(), '0.01')
await self.daemon.jsonrpc_resolve('lbry://bar')
file_list = self.daemon.jsonrpc_file_list()
self.assertEqual(file_list[0].stream_claim_info.claim.stream.description, claim.stream.description)
async def test_download_different_timeouts(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
all_except_sd = [
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
]
await self.server.blob_manager.delete_blobs(all_except_sd)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didnt create a file")
await self.server.blob_manager.delete_blobs([sd_hash])
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
self.assertEqual('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
async def wait_files_to_complete(self):
while self.daemon.jsonrpc_file_list(status='running'):
while self.sout(self.daemon.jsonrpc_file_list(status='running')):
await asyncio.sleep(0.01)
async def test_filename_conflicts_management_on_resume_download(self):
await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
file_info = self.daemon.jsonrpc_file_list()[0]
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.daemon.jsonrpc_get('lbry://foo')
@@ -59,43 +121,42 @@ class FileCommands(CommandTestCase):
await self.daemon.stream_manager.start()
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
# check that internal state got through up to the file list API
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
file_info = self.daemon.jsonrpc_file_list()[0]
self.assertEqual(downloader.output_file_name, file_info['file_name'])
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
file_info = self.sout(self.daemon.jsonrpc_file_list()[0])
self.assertEqual(stream.file_name, file_info['file_name'])
# checks if what the API shows is what he have at the very internal level.
self.assertEqual(downloader.output_path, file_info['download_path'])
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
self.assertEqual(stream.full_path, file_info['download_path'])
async def test_incomplete_downloads_erases_output_file_on_stop(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
file_info = self.daemon.jsonrpc_file_list()[0]
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
blob.blob_hash for blob in blobs[1:-1]
]
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
self.assertFalse(os.path.isfile(path))
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
self.assertNotIn('error', resp)
self.assertTrue(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
self.assertTrue(os.path.isfile(path))
self.daemon.stream_manager.stop()
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
self.assertFalse(os.path.isfile(path))
async def test_incomplete_downloads_retry(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
blob.blob_hash for blob in blobs[1:-1]
]
# backup server blobs
@@ -107,10 +168,10 @@ class FileCommands(CommandTestCase):
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
# start the download
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
self.assertNotIn('error', resp)
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
self.assertEqual('running', self.daemon.jsonrpc_file_list()[0]['status'])
self.assertEqual('running', self.sout(self.daemon.jsonrpc_file_list())[0]['status'])
await self.daemon.jsonrpc_file_set_status('stop', claim_name='foo')
# recover blobs
@@ -122,14 +183,14 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_file_set_status('start', claim_name='foo')
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
file_info = self.daemon.jsonrpc_file_list()[0]
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
self.assertEqual('finished', file_info['status'])
async def test_unban_recovers_stream(self):
BlobDownloader.BAN_TIME = .5 # fixme: temporary field, will move to connection manager or a conf
BlobDownloader.BAN_FACTOR = .5 # fixme: temporary field, will move to connection manager or a conf
tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
# backup blob
@@ -143,7 +204,7 @@ class FileCommands(CommandTestCase):
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
self.server_blob_manager.blobs.clear()
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
await self.server_blob_manager.blob_completed(missing_blob)
self.server_blob_manager.blob_completed(missing_blob)
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
async def test_paid_download(self):
@@ -155,7 +216,7 @@ class FileCommands(CommandTestCase):
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
response = await self.daemon.jsonrpc_get('lbry://expensive')
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
self.assertEqual(response['error'], 'fee of 11.00000 exceeds max available balance')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
@@ -165,7 +226,7 @@ class FileCommands(CommandTestCase):
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
response = await self.daemon.jsonrpc_get('lbry://maxkey')
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
self.assertEqual(response['error'], 'fee of 111.00000 exceeds max configured to allow of 50.00000')
@@ -177,8 +238,8 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
await self.assertBalance(self.account, '9.925679')
response = await self.daemon.jsonrpc_get('lbry://icanpay')
self.assertNotIn('error', response)
await self.ledger.wait(response['tx'])
raw_content_fee = response.content_fee.raw
await self.ledger.wait(response.content_fee)
await self.assertBalance(self.account, '8.925555')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
@@ -191,3 +252,10 @@ class FileCommands(CommandTestCase):
self.assertEqual(
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
)
# restart the daemon and make sure the fee is still there
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
self.assertEqual(self.daemon.jsonrpc_file_list()[0].content_fee.raw, raw_content_fee)
@@ -36,7 +36,7 @@ class BasicTransactionTest(IntegrationTestCase):
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.media_type = "video/mp4"
stream.stream.source.media_type = "video/mp4"
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
@@ -59,9 +59,8 @@ class BasicTransactionTest(IntegrationTestCase):
self.assertEqual(d2l(await self.account.get_balance()), '7.985786')
self.assertEqual(d2l(await self.account.get_balance(include_claims=True)), '9.985786')
response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
self.assertIn('lbry://@bar/foo', response)
self.assertIn('claim', response['lbry://@bar/foo'])
response = await self.ledger.resolve(['lbry://@bar/foo'])
self.assertEqual(response['lbry://@bar/foo'].claim.claim_type, 'stream')
abandon_tx = await Transaction.create([Input.spend(stream_tx.outputs[0])], [], [self.account], self.account)
await self.broadcast(abandon_tx)
@@ -69,10 +68,10 @@ class BasicTransactionTest(IntegrationTestCase):
await self.blockchain.generate(1)
await self.ledger.wait(abandon_tx)
response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
self.assertNotIn('claim', response['lbry://@bar/foo'])
response = await self.ledger.resolve(['lbry://@bar/foo'])
self.assertIn('error', response['lbry://@bar/foo'])
# checks for expected format in inexistent URIs
response = await self.ledger.resolve(0, 10, 'lbry://404', 'lbry://@404')
self.assertEqual('URI lbry://404 cannot be resolved', response['lbry://404']['error'])
self.assertEqual('URI lbry://@404 cannot be resolved', response['lbry://@404']['error'])
response = await self.ledger.resolve(['lbry://404', 'lbry://@404'])
self.assertEqual('lbry://404 did not resolve to a claim', response['lbry://404']['error'])
self.assertEqual('lbry://@404 did not resolve to a claim', response['lbry://@404']['error'])
+317 -35
View File
@@ -1,60 +1,342 @@
from integration.testcase import CommandTestCase
import asyncio
import json
import hashlib
from unittest import skip
from binascii import hexlify, unhexlify
from lbrynet.testcase import CommandTestCase
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.schema.compat import OldClaimMessage
from torba.client.hash import sha256, Base58
class ResolveCommand(CommandTestCase):
async def test_resolve(self):
tx = await self.channel_create('@abc', '0.01')
channel_id = tx['outputs'][0]['claim_id']
def get_claim_id(self, tx):
return tx['outputs'][0]['claim_id']
async def assertResolvesToClaimId(self, name, claim_id):
other = (await self.resolve(name))[name]
if claim_id is None:
self.assertIn('error', other)
else:
self.assertEqual(claim_id, other['claim_id'])
async def test_resolve_response(self):
channel_id = self.get_claim_id(
await self.channel_create('@abc', '0.01')
)
# resolving a channel @abc
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertIn('certificate', response['lbry://@abc'])
self.assertNotIn('claim', response['lbry://@abc'])
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 0)
self.assertEqual(response['lbry://@abc']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['value_type'], 'channel')
self.assertEqual(response['lbry://@abc']['meta']['claims_in_channel'], 0)
await self.stream_create('foo', '0.01', channel_id=channel_id)
await self.stream_create('foo2', '0.01', channel_id=channel_id)
# resolving a channel @abc with some claims in it
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertIn('certificate', response['lbry://@abc'])
self.assertNotIn('claim', response['lbry://@abc'])
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 2)
response['lbry://@abc']['confirmations'] += 2
response['lbry://@abc']['meta']['claims_in_channel'] = 2
self.assertEqual(response, await self.resolve('lbry://@abc'))
# resolving claim foo within channel @abc
response = await self.resolve('lbry://@abc/foo')
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
claim = response['lbry://@abc/foo']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertTrue(claim['is_channel_signature_valid'])
self.assertEqual(
claim['timestamp'],
self.ledger.headers[claim['height']]['timestamp']
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
)
# resolving claim foo by itself
response = await self.resolve('lbry://foo')
self.assertSetEqual({'lbry://foo'}, set(response))
claim = response['lbry://foo']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
self.assertEqual(claim, (await self.resolve('lbry://foo'))['lbry://foo'])
# resolving from the given permanent url
permanent_url = response['lbry://@abc/foo']['permanent_url']
self.assertEqual(claim, (await self.resolve(permanent_url))[permanent_url])
# resolving multiple at once
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
claim = response['lbry://foo2']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo2')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
self.assertEqual(claim['name'], 'foo2')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertTrue(claim['is_channel_signature_valid'])
# resolve has correct confirmations
tx_details = await self.blockchain.get_raw_transaction(claim['txid'])
self.assertEqual(claim['confirmations'], json.loads(tx_details)['confirmations'])
# resolve handles invalid data
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
await self.generate(1)
response = await self.resolve("lbry://gibberish")
self.assertSetEqual({'lbry://gibberish'}, set(response))
claim = response['lbry://gibberish']
self.assertEqual(claim['name'], 'gibberish')
self.assertNotIn('value', claim)
async def test_winning_by_effective_amount(self):
# first one remains winner unless something else changes
claim_id1 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
claim_id2 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
claim_id3 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
# supports change the winner
await self.support_create(claim_id3, '0.09')
await self.assertResolvesToClaimId('@foo', claim_id3)
await self.support_create(claim_id2, '0.19')
await self.assertResolvesToClaimId('@foo', claim_id2)
await self.support_create(claim_id1, '0.29')
await self.assertResolvesToClaimId('@foo', claim_id1)
async def test_advanced_resolve(self):
claim_id1 = self.get_claim_id(
await self.stream_create('foo', '0.7', allow_duplicate_name=True))
claim_id2 = self.get_claim_id(
await self.stream_create('foo', '0.8', allow_duplicate_name=True))
claim_id3 = self.get_claim_id(
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo:1', claim_id1)
await self.assertResolvesToClaimId('foo:2', claim_id2)
await self.assertResolvesToClaimId('foo:3', claim_id3)
await self.assertResolvesToClaimId('foo:4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
await self.assertResolvesToClaimId('foo$3', claim_id1)
await self.assertResolvesToClaimId('foo$4', None)
async def test_partial_claim_id_resolve(self):
# add some noise
await self.channel_create('@abc', '0.1', allow_duplicate_name=True)
await self.channel_create('@abc', '0.2', allow_duplicate_name=True)
await self.channel_create('@abc', '1.0', allow_duplicate_name=True)
channel_id = self.get_claim_id(
await self.channel_create('@abc', '1.1', allow_duplicate_name=True))
await self.assertResolvesToClaimId(f'@abc', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id)
channel = (await self.claim_search(claim_id=channel_id))[0]
await self.assertResolvesToClaimId(channel['short_url'], channel_id)
await self.assertResolvesToClaimId(channel['canonical_url'], channel_id)
await self.assertResolvesToClaimId(channel['permanent_url'], channel_id)
# add some noise
await self.stream_create('foo', '0.1', allow_duplicate_name=True, channel_id=channel['claim_id'])
await self.stream_create('foo', '0.2', allow_duplicate_name=True, channel_id=channel['claim_id'])
await self.stream_create('foo', '0.3', allow_duplicate_name=True, channel_id=channel['claim_id'])
claim_id1 = self.get_claim_id(
await self.stream_create('foo', '0.7', allow_duplicate_name=True, channel_id=channel['claim_id']))
claim1 = (await self.claim_search(claim_id=claim_id1))[0]
await self.assertResolvesToClaimId('foo', claim_id1)
await self.assertResolvesToClaimId('@abc/foo', claim_id1)
await self.assertResolvesToClaimId(claim1['short_url'], claim_id1)
await self.assertResolvesToClaimId(claim1['canonical_url'], claim_id1)
await self.assertResolvesToClaimId(claim1['permanent_url'], claim_id1)
claim_id2 = self.get_claim_id(
await self.stream_create('foo', '0.8', allow_duplicate_name=True, channel_id=channel['claim_id']))
claim2 = (await self.claim_search(claim_id=claim_id2))[0]
await self.assertResolvesToClaimId('foo', claim_id2)
await self.assertResolvesToClaimId('@abc/foo', claim_id2)
await self.assertResolvesToClaimId(claim2['short_url'], claim_id2)
await self.assertResolvesToClaimId(claim2['canonical_url'], claim_id2)
await self.assertResolvesToClaimId(claim2['permanent_url'], claim_id2)
async def test_abandoned_channel_with_signed_claims(self):
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id'])
abandoned_channel_id = channel['claim_id']
await self.channel_abandon(txid=channel['txid'], nout=0)
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim_id = orphan_claim['outputs'][0]['claim_id']
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {'error': 'lbry://@abc/on-channel-claim did not resolve to a claim'}
})
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = (await self.resolve(direct_uri))[direct_uri]
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
await self.stream_abandon(claim_id=orphan_claim_id)
uri = 'lbry://@abc/on-channel-claim'
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id'])
# resolves normally
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.stream_create(
'on-channel-claim', '0.00000001', channel_id=channel['claim_id'], allow_duplicate_name=True
)
# it still resolves! but to the older claim
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
self.assertEqual(response[uri]['txid'], valid_claim['txid'])
claims = await self.claim_search(name='on-channel-claim')
self.assertEqual(2, len(claims))
self.assertEqual(
{channel['claim_id']}, {claim['signing_channel']['claim_id'] for claim in claims}
)
async def test_normalization_resolution(self):
one = 'ΣίσυφοςfiÆ'
two = 'ΣΊΣΥΦΟσFIæ'
_ = await self.stream_create(one, '0.1')
c = await self.stream_create(two, '0.2')
winner_id = c['outputs'][0]['claim_id']
r1 = await self.resolve(f'lbry://{one}')
r2 = await self.resolve(f'lbry://{two}')
self.assertEqual(winner_id, r1[f'lbry://{one}']['claim_id'])
self.assertEqual(winner_id, r2[f'lbry://{two}']['claim_id'])
async def test_resolve_old_claim(self):
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
await self.confirm_tx(channel.id)
address = channel.outputs[0].get_address(self.account.ledger)
claim = generate_signed_legacy(address, channel.outputs[0])
tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
response = await self.resolve('@olds/example')
self.assertTrue(response['@olds/example']['is_channel_signature_valid'])
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
tx = await Transaction.claim_create(
'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account
)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
response = await self.resolve('bad_example')
self.assertFalse(response['bad_example']['is_channel_signature_valid'])
response = await self.resolve('@olds/bad_example')
self.assertEqual(response, {
'@olds/bad_example': {'error': '@olds/bad_example did not resolve to a claim'}
})
async def _test_resolve_abc_foo(self):
response = await self.resolve('lbry://@abc/foo')
claim = response['lbry://@abc/foo']
self.assertIn('signing_channel', claim)
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertEqual(claim['meta']['claims_in_channel'], 0)
self.assertEqual(
claim['timestamp'],
self.ledger.headers[claim['height']]['timestamp']
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
)
@skip('this test does not work with new resolve')
async def test_resolve_lru_cache_doesnt_persist_errors(self):
original_get_transaction = self.daemon.wallet_manager.ledger.network.get_transaction
async def timeout_get_transaction(txid):
fut = self.loop.create_future()
def delayed_raise_cancelled_error():
fut.set_exception(asyncio.CancelledError())
self.loop.call_soon(delayed_raise_cancelled_error)
return await fut
tx = await self.channel_create('@abc', '0.01')
channel_id = tx['outputs'][0]['claim_id']
await self.stream_create('foo', '0.01', channel_id=channel_id)
# raise a cancelled error from get_transaction
self.daemon.wallet_manager.ledger.network.get_transaction = timeout_get_transaction
with self.assertRaises(KeyError):
await self._test_resolve_abc_foo()
# restore the real get_transaction that doesn't cancel, it should be called and the result cached
self.daemon.wallet_manager.ledger.network.get_transaction = original_get_transaction
await self._test_resolve_abc_foo()
called_again = asyncio.Event(loop=self.loop)
def check_result_cached(txid):
called_again.set()
return original_get_transaction(txid)
# check that the result was cached
self.daemon.wallet_manager.ledger.network.get_transaction = check_result_cached
await self._test_resolve_abc_foo()
self.assertFalse(called_again.is_set())
def generate_signed_legacy(address: bytes, output: Output):
decoded_address = Base58.decode(address)
claim = OldClaimMessage()
claim.ParseFromString(unhexlify(
'080110011aee04080112a604080410011a2b4865726520617265203520526561736f6e73204920e29da4e'
'fb88f204e657874636c6f7564207c20544c4722920346696e64206f7574206d6f72652061626f7574204e'
'657874636c6f75643a2068747470733a2f2f6e657874636c6f75642e636f6d2f0a0a596f752063616e206'
'6696e64206d65206f6e20746865736520736f6369616c733a0a202a20466f72756d733a2068747470733a'
'2f2f666f72756d2e6865617679656c656d656e742e696f2f0a202a20506f64636173743a2068747470733'
'a2f2f6f6666746f706963616c2e6e65740a202a2050617472656f6e3a2068747470733a2f2f7061747265'
'6f6e2e636f6d2f7468656c696e757867616d65720a202a204d657263683a2068747470733a2f2f7465657'
'37072696e672e636f6d2f73746f7265732f6f6666696369616c2d6c696e75782d67616d65720a202a2054'
'77697463683a2068747470733a2f2f7477697463682e74762f786f6e64616b0a202a20547769747465723'
'a2068747470733a2f2f747769747465722e636f6d2f7468656c696e757867616d65720a0a2e2e2e0a6874'
'7470733a2f2f7777772e796f75747562652e636f6d2f77617463683f763d4672546442434f535f66632a0'
'f546865204c696e75782047616d6572321c436f7079726967687465642028636f6e746163742061757468'
'6f722938004a2968747470733a2f2f6265726b2e6e696e6a612f7468756d626e61696c732f46725464424'
'34f535f666352005a001a41080110011a30040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc22'
'f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f32209766964656f2f6d70342a5c080110031a406'
'2b2dd4c45e364030fbfad1a6fefff695ebf20ea33a5381b947753e2a0ca359989a5cc7d15e5392a0d354c'
'0b68498382b2701b22c03beb8dcb91089031b871e72214feb61536c007cdf4faeeaab4876cb397feaf6b51'
))
claim.ClearField("publisherSignature")
digest = sha256(b''.join([
decoded_address,
claim.SerializeToString(),
output.claim_hash[::-1]
]))
signature = output.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
claim.publisherSignature.version = 1
claim.publisherSignature.signatureType = 1
claim.publisherSignature.signature = signature
claim.publisherSignature.certificateId = output.claim_hash[::-1]
return claim
+419
View File
@@ -0,0 +1,419 @@
import os
import hashlib
import aiohttp
import aiohttp.web
import asyncio
from lbrynet.utils import aiohttp_request
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.testcase import CommandTestCase
def get_random_bytes(n: int) -> bytes:
result = b''.join(hashlib.sha256(os.urandom(4)).digest() for _ in range(n // 16))
if len(result) < n:
result += os.urandom(n - len(result))
elif len(result) > n:
result = result[:-(len(result) - n)]
assert len(result) == n, (n, len(result))
return result
class RangeRequests(CommandTestCase):
async def _restart_stream_manager(self):
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
return
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
self.daemon.conf.save_blobs = save_blobs
self.daemon.conf.save_files = save_files
self.data = data
await self.stream_create('foo', '0.01', data=self.data, file_size=file_size)
if save_blobs:
self.assertTrue(len(os.listdir(self.daemon.blob_manager.blob_dir)) > 1)
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
# await self._restart_stream_manager()
await self.daemon.streaming_runner.setup()
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
self.daemon.conf.streaming_port)
await site.start()
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
async def _test_range_requests(self):
name = 'foo'
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}'
async with aiohttp_request('get', url) as req:
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
self.assertEqual(content_length, len(streamed_bytes))
return streamed_bytes, content_range, content_length
async def test_range_requests_2_byte(self):
self.data = b'hi'
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(15, content_length)
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_15_byte(self):
self.data = b'123456789abcdef'
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(15, content_length)
self.assertEqual(15, len(streamed))
self.assertEqual(self.data, streamed)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_0_padded_bytes(self, size: int = (MAX_BLOB_SIZE - 1) * 4,
expected_range: str = 'bytes 0-8388603/8388604', padding=b'',
file_size=0):
self.data = get_random_bytes(size)
await self._setup_stream(self.data, file_size=file_size)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(len(self.data + padding), content_length)
self.assertEqual(streamed, self.data + padding)
self.assertEqual(expected_range, content_range)
async def test_range_requests_1_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 1, padding=b'\x00'
)
async def test_range_requests_2_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 2, padding=b'\x00' * 2
)
async def test_range_requests_14_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 14, padding=b'\x00' * 14
)
async def test_range_requests_no_padding_size_from_claim(self):
size = ((MAX_BLOB_SIZE - 1) * 4) - 14
await self.test_range_requests_0_padded_bytes(size, padding=b'', file_size=size,
expected_range=f"bytes 0-{size-1}/{size}")
async def test_range_requests_15_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 15, padding=b'\x00' * 15
)
async def test_forbidden(self):
self.data = get_random_bytes(1000)
await self._setup_stream(self.data, file_size=1000)
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/foo'
self.daemon.conf.streaming_get = False
async with aiohttp_request('get', url) as req:
self.assertEqual(403, req.status)
async def test_range_requests_last_block_of_last_blob_padding(self):
self.data = get_random_bytes(((MAX_BLOB_SIZE - 1) * 4) - 16)
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(len(self.data), content_length)
self.assertEqual(streamed, self.data)
self.assertEqual('bytes 0-8388587/8388588', content_range)
async def test_streaming_only_with_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
# test that repeated range requests do not create duplicate files
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
# test that a range request after restart does not create a duplicate file
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
async def test_streaming_only_without_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_blobs=False)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
# test that repeated range requests do not create duplicate files
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
# test that a range request after restart does not create a duplicate file
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
async def test_stream_and_save_file_with_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_files=True)
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
full_path = stream.full_path
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
for _ in range(3):
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
async def test_stream_and_save_file_without_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_files=True)
self.daemon.conf.save_blobs = False
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
full_path = stream.full_path
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
async def test_switch_save_blobs_while_running(self):
await self.test_streaming_only_without_blobs()
self.daemon.conf.save_blobs = True
blobs_in_stream = self.daemon.jsonrpc_file_list()[0].blobs_in_stream
sd_hash = self.daemon.jsonrpc_file_list()[0].sd_hash
start_file_count = len(os.listdir(self.daemon.blob_manager.blob_dir))
await self._test_range_requests()
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
# switch back
self.daemon.conf.save_blobs = False
await self._test_range_requests()
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=sd_hash)
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
await self._test_range_requests()
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(blobs_in_stream, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
async def test_file_save_streaming_only_save_blobs(self):
await self.test_streaming_only_with_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
self.server.stop_server()
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNotNone(stream.full_path)
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=stream.sd_hash)
async def test_file_save_stop_before_finished_streaming_only(self, wait_for_start_writing=False):
await self.test_streaming_only_with_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
self.server.stop_server()
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
path = stream.full_path
self.assertIsNotNone(path)
if wait_for_start_writing:
await stream.started_writing.wait()
self.assertTrue(os.path.isfile(path))
await self._restart_stream_manager()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNotNone(stream.full_path)
self.assertFalse(os.path.isfile(path))
if wait_for_start_writing:
await stream.started_writing.wait()
self.assertTrue(os.path.isfile(path))
async def test_file_save_stop_before_finished_streaming_only_wait_for_start(self):
return await self.test_file_save_stop_before_finished_streaming_only(wait_for_start_writing=True)
async def test_file_save_streaming_only_dont_save_blobs(self):
await self.test_streaming_only_without_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
class RangeRequestsLRUCache(CommandTestCase):
blob_lru_cache_size = 32
async def _request_stream(self):
name = 'foo'
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}'
async with aiohttp_request('get', url) as req:
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
self.assertEqual(content_length, len(streamed_bytes))
self.assertEqual(15, content_length)
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed_bytes)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_with_blob_lru_cache(self):
self.data = b'hi'
self.daemon.conf.save_blobs = False
self.daemon.conf.save_files = False
await self.stream_create('foo', '0.01', data=self.data, file_size=0)
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
await self.daemon.streaming_runner.setup()
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
self.daemon.conf.streaming_port)
await site.start()
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
await self._request_stream()
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
self.server.stop_server()
# running with cache size 0 gets through without errors without
# this since the server doesnt stop immediately
await asyncio.sleep(1, loop=self.loop)
await self._request_stream()
+1 -1
View File
@@ -54,7 +54,7 @@ class AccountSynchronization(AsyncioTestCase):
async def asyncTearDown(self):
self.wallet_component._running = False
await self.daemon.stop()
await self.daemon.stop(shutdown_runner=False)
@mock.patch('time.time', mock.Mock(return_value=12345))
def test_sync(self):
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class TransactionCommandsTestCase(CommandTestCase):
@@ -17,7 +17,7 @@ class TransactionCommandsTestCase(CommandTestCase):
sendtxid = await self.blockchain.send_to_address(change_address, 10)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.id, sendtxid)
self.assertEqual(tx.height, -2)
self.assertEqual(tx.height, -1)
await self.generate(1)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.height, self.ledger.headers.height)
+209 -20
View File
@@ -3,34 +3,223 @@ import tempfile
import shutil
import os
from torba.testcase import AsyncioTestCase
from lbrynet.error import InvalidDataError, InvalidBlobHashError
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import BlobFile, BlobBuffer, AbstractBlob
class TestBlobfile(AsyncioTestCase):
async def test_create_blob(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
class TestBlob(AsyncioTestCase):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
async def asyncSetUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
self.loop = asyncio.get_running_loop()
self.config = Config()
self.storage = SQLiteStorage(self.config, ":memory:", self.loop)
self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.config)
await self.storage.open()
def _get_blob(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(self.loop, self.blob_hash, len(self.blob_bytes), self.blob_manager.blob_completed,
blob_directory=blob_directory)
self.assertFalse(blob.get_is_verified())
self.addCleanup(blob.close)
return blob
async def _test_create_blob(self, blob_class=AbstractBlob, blob_directory=None):
blob = self._get_blob(blob_class, blob_directory)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
await asyncio.sleep(0, loop=self.loop) # wait for the db save task
return blob
async def _test_close_writers_on_finished(self, blob_class=AbstractBlob, blob_directory=None):
blob = self._get_blob(blob_class, blob_directory=blob_directory)
writers = [blob.get_blob_writer('1.2.3.4', port) for port in range(5)]
self.assertEqual(5, len(blob.writers))
# test that writing too much causes the writer to fail with InvalidDataError and to be removed
with self.assertRaises(InvalidDataError):
writers[1].write(self.blob_bytes * 2)
await writers[1].finished
await asyncio.sleep(0, loop=self.loop)
self.assertEqual(4, len(blob.writers))
# write the blob
other = writers[2]
writers[3].write(self.blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertEqual(0, len(blob.writers))
with self.assertRaises(IOError):
other.write(self.blob_bytes)
def _test_ioerror_if_length_not_set(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(
self.loop, self.blob_hash, blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=blob_directory
)
self.addCleanup(blob.close)
writer = blob.get_blob_writer()
with self.assertRaises(IOError):
writer.write(b'')
async def _test_invalid_blob_bytes(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=blob_directory
)
self.addCleanup(blob.close)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes[:-4] + b'fake')
with self.assertRaises(InvalidBlobHashError):
await writer.finished
async def test_add_blob_buffer_to_db(self):
blob = await self._test_create_blob(BlobBuffer)
db_status = await self.storage.get_blob_status(blob.blob_hash)
self.assertEqual(db_status, 'pending')
async def test_add_blob_file_to_db(self):
blob = await self._test_create_blob(BlobFile, self.tmp_dir)
db_status = await self.storage.get_blob_status(blob.blob_hash)
self.assertEqual(db_status, 'finished')
async def test_invalid_blob_bytes(self):
await self._test_invalid_blob_bytes(BlobBuffer)
await self._test_invalid_blob_bytes(BlobFile, self.tmp_dir)
def test_ioerror_if_length_not_set(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self._test_ioerror_if_length_not_set(BlobBuffer)
self._test_ioerror_if_length_not_set(BlobFile, tmp_dir)
async def test_create_blob_file(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
blob = await self._test_create_blob(BlobFile, tmp_dir)
self.assertIsInstance(blob, BlobFile)
self.assertTrue(os.path.isfile(blob.file_path))
for _ in range(2):
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
async def test_create_blob_buffer(self):
blob = await self._test_create_blob(BlobBuffer)
self.assertIsInstance(blob, BlobBuffer)
self.assertIsNotNone(blob._verified_bytes)
# check we can only read the bytes once, and that the buffer is torn down
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
self.assertIsNone(blob._verified_bytes)
with self.assertRaises(OSError):
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
self.assertIsNone(blob._verified_bytes)
async def test_close_writers_on_finished(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
await self._test_close_writers_on_finished(BlobBuffer)
await self._test_close_writers_on_finished(BlobFile, tmp_dir)
async def test_concurrency_and_premature_closes(self):
blob_directory = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(blob_directory))
blob = self._get_blob(BlobBuffer, blob_directory=blob_directory)
writer = blob.get_blob_writer('1.1.1.1', 1337)
self.assertEqual(1, len(blob.writers))
with self.assertRaises(OSError):
blob.get_blob_writer('1.1.1.1', 1337)
writer.close_handle()
self.assertTrue(blob.writers[('1.1.1.1', 1337)].closed())
writer = blob.get_blob_writer('1.1.1.1', 1337)
self.assertEqual(blob.writers[('1.1.1.1', 1337)], writer)
writer.close_handle()
await asyncio.sleep(0.000000001) # flush callbacks
self.assertEqual(0, len(blob.writers))
async def test_delete(self):
blob_buffer = await self._test_create_blob(BlobBuffer)
self.assertIsInstance(blob_buffer, BlobBuffer)
self.assertIsNotNone(blob_buffer._verified_bytes)
self.assertTrue(blob_buffer.get_is_verified())
blob_buffer.delete()
self.assertIsNone(blob_buffer._verified_bytes)
self.assertFalse(blob_buffer.get_is_verified())
loop = asyncio.get_event_loop()
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, tmp_dir, storage)
blob_file = await self._test_create_blob(BlobFile, tmp_dir)
self.assertIsInstance(blob_file, BlobFile)
self.assertTrue(os.path.isfile(blob_file.file_path))
self.assertTrue(blob_file.get_is_verified())
blob_file.delete()
self.assertFalse(os.path.isfile(blob_file.file_path))
self.assertFalse(blob_file.get_is_verified())
await storage.open()
await blob_manager.setup()
async def test_delete_corrupt(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes)
await blob.verified.wait()
blob.close()
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
self.assertTrue(blob.get_is_verified())
# add the blob on the server
blob = blob_manager.get_blob(blob_hash, len(blob_bytes))
self.assertEqual(blob.get_is_verified(), False)
self.assertNotIn(blob_hash, blob_manager.completed_blob_hashes)
with open(blob.file_path, 'wb+') as f:
f.write(b'\x00')
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
self.assertFalse(blob.get_is_verified())
self.assertFalse(os.path.isfile(blob.file_path))
writer = blob.open_for_writing()
writer.write(blob_bytes)
await blob.finished_writing.wait()
self.assertTrue(os.path.isfile(blob.file_path), True)
self.assertEqual(blob.get_is_verified(), True)
self.assertIn(blob_hash, blob_manager.completed_blob_hashes)
def test_invalid_blob_hash(self):
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, '', len(self.blob_bytes))
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'x' * 96, len(self.blob_bytes))
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'a' * 97, len(self.blob_bytes))
async def _test_close_reader(self, blob_class=AbstractBlob, blob_directory=None):
blob = await self._test_create_blob(blob_class, blob_directory)
reader = blob.reader_context()
self.assertEqual(0, len(blob.readers))
async def read_blob_buffer():
with reader as read_handle:
self.assertEqual(1, len(blob.readers))
await asyncio.sleep(2, loop=self.loop)
self.assertEqual(0, len(blob.readers))
return read_handle.read()
self.loop.call_later(1, blob.close)
with self.assertRaises(ValueError) as err:
read_task = self.loop.create_task(read_blob_buffer())
await read_task
self.assertEqual(err.exception, ValueError("I/O operation on closed file"))
async def test_close_reader(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
await self._test_close_reader(BlobBuffer)
await self._test_close_reader(BlobFile, tmp_dir)
+23 -20
View File
@@ -5,50 +5,53 @@ import os
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
class TestBlobManager(AsyncioTestCase):
async def test_sync_blob_manager_on_startup(self):
loop = asyncio.get_event_loop()
async def setup_blob_manager(self, save_blobs=True):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.config = Config(save_blobs=save_blobs)
self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite"))
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, tmp_dir, storage)
async def test_sync_blob_file_manager_on_startup(self):
await self.setup_blob_manager(save_blobs=True)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
with open(os.path.join(blob_manager.blob_dir, blob_hash), 'wb') as f:
with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f:
f.write(blob_bytes)
# it should not have been added automatically on startup
await storage.open()
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# make sure we can add the blob
await blob_manager.blob_completed(blob_manager.get_blob(blob_hash, len(blob_bytes)))
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes)))
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# stop the blob manager and restart it, make sure the blob is there
blob_manager.stop()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
self.blob_manager.stop()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# test that the blob is removed upon the next startup after the file being manually deleted
blob_manager.stop()
self.blob_manager.stop()
# manually delete the blob file and restart the blob manager
os.remove(os.path.join(blob_manager.blob_dir, blob_hash))
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash))
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# check that the deleted blob was updated in the database
self.assertEqual(
'pending', (
await storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
)
)
+83 -17
View File
@@ -9,9 +9,9 @@ from lbrynet.blob_exchange.serialization import BlobRequest
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.blob_exchange.client import request_blob
from lbrynet.dht.peer import KademliaPeer, PeerManager
# import logging
@@ -35,13 +35,13 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
reflector_servers=[])
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
reflector_servers=[])
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
@@ -51,6 +51,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
await self.server_blob_manager.setup()
self.server.start_server(33333, '127.0.0.1')
self.addCleanup(self.server.stop_server)
await self.server.started_listening.wait()
@@ -58,21 +59,25 @@ class TestBlobExchange(BlobExchangeTestBase):
async def _add_blob_to_server(self, blob_hash: str, blob_bytes: bytes):
# add the blob on the server
server_blob = self.server_blob_manager.get_blob(blob_hash, len(blob_bytes))
writer = server_blob.open_for_writing()
writer = server_blob.get_blob_writer()
writer.write(blob_bytes)
await server_blob.finished_writing.wait()
await server_blob.verified.wait()
self.assertTrue(os.path.isfile(server_blob.file_path))
self.assertEqual(server_blob.get_is_verified(), True)
self.assertTrue(writer.closed())
async def _test_transfer_blob(self, blob_hash: str):
client_blob = self.client_blob_manager.get_blob(blob_hash)
# download the blob
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
self.server_from_client.tcp_port, 2, 3)
await client_blob.finished_writing.wait()
downloaded, transport = await request_blob(self.loop, client_blob, self.server_from_client.address,
self.server_from_client.tcp_port, 2, 3)
self.assertIsNotNone(transport)
self.addCleanup(transport.close)
await client_blob.verified.wait()
self.assertEqual(client_blob.get_is_verified(), True)
self.assertTrue(downloaded)
client_blob.close()
async def test_transfer_sd_blob(self):
sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02"
@@ -92,9 +97,11 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
second_client_conf = Config()
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
@@ -102,7 +109,7 @@ class TestBlobExchange(BlobExchangeTestBase):
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
second_client_blob = second_client_blob_manager.get_blob(blob_hash)
# download the blob
await asyncio.gather(
@@ -112,9 +119,65 @@ class TestBlobExchange(BlobExchangeTestBase):
),
self._test_transfer_blob(blob_hash)
)
await second_client_blob.finished_writing.wait()
await second_client_blob.verified.wait()
self.assertEqual(second_client_blob.get_is_verified(), True)
async def test_blob_writers_concurrency(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.server_blob_manager.get_blob(blob_hash)
write_blob = blob._write_blob
write_called_count = 0
def wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
write_blob(blob_bytes)
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
writer2 = blob.get_blob_writer(peer_port=2)
reader1_ctx_before_write = blob.reader_context()
with self.assertRaises(OSError):
blob.get_blob_writer(peer_port=2)
with self.assertRaises(OSError):
with blob.reader_context():
pass
blob.set_length(len(mock_blob_bytes))
results = {}
def check_finished_callback(writer, num):
def inner(writer_future: asyncio.Future):
results[num] = writer_future.result()
writer.finished.add_done_callback(inner)
check_finished_callback(writer1, 1)
check_finished_callback(writer2, 2)
def write_task(writer):
async def _inner():
writer.write(mock_blob_bytes)
return self.loop.create_task(_inner())
await asyncio.gather(write_task(writer1), write_task(writer2), loop=self.loop)
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
with reader1_ctx_before_write as f:
self.assertEqual(mock_blob_bytes, f.read())
with blob.reader_context() as f:
self.assertEqual(mock_blob_bytes, f.read())
with blob.reader_context() as f:
blob.close()
with self.assertRaises(ValueError):
f.read()
self.assertListEqual([], blob.readers)
async def test_host_different_blobs_to_multiple_peers_at_once(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
@@ -124,9 +187,12 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_conf = Config()
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
@@ -143,7 +209,7 @@ class TestBlobExchange(BlobExchangeTestBase):
server_from_second_client.tcp_port, 2, 3
),
self._test_transfer_blob(sd_hash),
second_client_blob.finished_writing.wait()
second_client_blob.verified.wait()
)
self.assertEqual(second_client_blob.get_is_verified(), True)
+76 -2
View File
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from lbrynet import utils
import unittest
import asyncio
from lbrynet import utils
from torba.testcase import AsyncioTestCase
class CompareVersionTest(unittest.TestCase):
@@ -61,3 +62,76 @@ class SdHashTests(unittest.TestCase):
}
}
self.assertIsNone(utils.get_sd_hash(claim))
class CacheConcurrentDecoratorTests(AsyncioTestCase):
def setUp(self):
self.called = []
self.finished = []
self.counter = 0
@utils.cache_concurrent
async def foo(self, arg1, arg2=None, delay=1):
self.called.append((arg1, arg2, delay))
await asyncio.sleep(delay, loop=self.loop)
self.counter += 1
self.finished.append((arg1, arg2, delay))
return object()
async def test_gather_duplicates(self):
result = await asyncio.gather(
self.loop.create_task(self.foo(1)), self.loop.create_task(self.foo(1)), loop=self.loop
)
self.assertEqual(1, len(self.called))
self.assertEqual(1, len(self.finished))
self.assertEqual(1, self.counter)
self.assertIs(result[0], result[1])
self.assertEqual(2, len(result))
async def test_one_cancelled_all_cancel(self):
t1 = self.loop.create_task(self.foo(1))
self.loop.call_later(0.1, t1.cancel)
with self.assertRaises(asyncio.CancelledError):
await asyncio.gather(
t1, self.loop.create_task(self.foo(1)), loop=self.loop
)
self.assertEqual(1, len(self.called))
self.assertEqual(0, len(self.finished))
self.assertEqual(0, self.counter)
async def test_error_after_success(self):
def cause_type_error():
self.counter = ""
self.loop.call_later(0.1, cause_type_error)
t1 = self.loop.create_task(self.foo(1))
t2 = self.loop.create_task(self.foo(1))
with self.assertRaises(TypeError):
await t2
self.assertEqual(1, len(self.called))
self.assertEqual(0, len(self.finished))
self.assertTrue(t1.done())
self.assertEqual("", self.counter)
# test that the task is run fresh, it should not error
self.counter = 0
t3 = self.loop.create_task(self.foo(1))
self.assertTrue((await t3))
self.assertEqual(1, self.counter)
# the previously failed call should still raise if awaited
with self.assertRaises(TypeError):
await t1
self.assertEqual(1, self.counter)
async def test_break_it(self):
t1 = self.loop.create_task(self.foo(1))
t2 = self.loop.create_task(self.foo(1))
t3 = self.loop.create_task(self.foo(2, delay=0))
t3.add_done_callback(lambda _: t2.cancel())
with self.assertRaises(asyncio.CancelledError):
await asyncio.gather(t1, t2, t3)
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff. Show more