Compare commits

...
263 Commits
Author SHA1 Message Date
Victor Shyba 27afc759ff fix peer search using inexistent variable, fix test to cover it 2019-05-10 14:23:40 -03:00
Victor Shyba 7abb28cb3e raise ban cap to 30 minutes 2019-05-10 14:23:40 -03:00
Jack Robison 7d2f592bb5 Merge pull request #2107 from lbryio/fix-update-file-we-dont-have
fix stream_update failing on metadata only updates for files we don't have
2019-05-10 12:28:53 -04:00
Jack Robison 1a182bc9bd bytes/str 2019-05-10 11:50:20 -04:00
Jack Robison 324cbdcdb2 only save the content claim if stream_hash isn't None
-stream_hash being none means we don't have the file in the database
2019-05-10 11:50:20 -04:00
Lex Berezhny 6c0c0ee322 added logging to publish command 2019-05-10 11:43:34 -04:00
Jack Robison 1e54271d8a Merge pull request #2105 from lbryio/fix-store-content-fee
fix store content fee
2019-05-09 18:47:30 -04:00
Jack Robison 54b9ed6cb2 fix store content fee 2019-05-09 18:14:54 -04:00
Lex Berezhny 558a02d187 correctly format lat/long and fee amounts in JSON RPC response 2019-05-09 16:19:49 -04:00
Jack Robison 3567b331c0 v0.37.0rc6 2019-05-09 11:52:34 -04:00
Jack Robison 9e4cf3c995 Merge pull request #2048 from lbryio/timeout_settings
Adjust default max_connections_per_download and peer_connect_timeout settings
2019-05-09 11:14:59 -04:00
Thomas Zarebczan db1d547b19 adjust the default max_connections_per_download and peer_connect_timeout 2019-05-09 10:38:32 -04:00
Jack Robison c31efd780b docs 2019-05-09 10:32:07 -04:00
Jack Robison 7cdb7987e3 Merge pull request #2102 from lbryio/shutdown_fast
Set a shutdown timeout for TCPSite to avoid hanging on exit
2019-05-09 10:24:13 -04:00
Victor Shyba ec07037e88 set shutdown_timeout on TCPSite 2019-05-09 01:01:07 -03:00
Lex Berezhny 9bb7b0571e fix integrat tests 2019-05-09 00:00:39 -04:00
Lex Berezhny d98162a118 stream_update --replace also clears the channel 2019-05-09 00:00:39 -04:00
Jack Robison c5ae59bff7 Merge pull request #2100 from lbryio/junction_fixes
DHT junction fixes for cases that can lock up
2019-05-08 23:52:54 -04:00
Victor Shyba efbf2f49a9 fix generator junction 2019-05-08 23:02:38 -03:00
Victor Shyba 77c69f661d add locking cases scenarios 2019-05-08 23:02:38 -03:00
Victor Shyba 2e09d2adfa close search on transport close 2019-05-08 23:02:38 -03:00
Jack Robison 8620a0e424 Merge pull request #2095 from lbryio/add-content-fee-and-saved-file-to-db
add saved_file and content_fee columns to file table
2019-05-08 20:37:56 -04:00
Jack Robison e2c123456b write decrypted blob data from run_in_executor 2019-05-08 16:07:32 -04:00
Jack Robison c9be79f062 prevent re-migrating 10 to 11 2019-05-08 15:13:07 -04:00
Jack Robison a2a1ef34ab fix tests 2019-05-08 14:55:09 -04:00
Jack Robison 761a2b235c set written_bytes, download_path, file_name, and download_directory to None if the file doesn't exist 2019-05-08 14:11:44 -04:00
Jack Robison 3ca2fee592 use stream size from claim if available 2019-05-08 14:11:39 -04:00
Jack Robison a480b2d25f fix race when closing blob client protocol after getting a response 2019-05-08 12:21:11 -04:00
Jack Robison d7032b12d7 add saved_file and content_fee columns to file table
-drop not null constraints for file_name and download_directory

-add migrator
2019-05-08 10:26:06 -04:00
Jack Robison 84381ff76c Merge pull request #2093 from lbryio/fix_announce_on_storage
Refactor announcer + fixups for last db migrator code
2019-05-07 10:46:12 -04:00
Victor Shyba 007dd43861 re-raise cancelled errors 2019-05-07 11:10:49 -03:00
Victor Shyba 1bc4e4b702 typo on logging, announcements logging is info, not warning 2019-05-07 11:10:49 -03:00
Victor Shyba b91f27219d use fetchall as there is an update while iterating 2019-05-07 11:10:49 -03:00
Victor Shyba dc4560cc9a refactor announcer 2019-05-07 11:10:49 -03:00
Victor Shyba 84b471d486 limit batch size on announcer test so it covers batching logic 2019-05-07 11:10:49 -03:00
Victor Shyba c6a3c05a0c add missing migrator line 2019-05-07 11:10:49 -03:00
Jack Robison dd665a758d Merge pull request #2065 from lbryio/streaming-bug-fixes
Streaming bug fixes
2019-05-07 10:10:39 -04:00
Jack Robison 32dc1297ec fix infinite loop 2019-05-06 18:46:26 -04:00
Jack Robison bef244ba77 dont make duplicate files 2019-05-06 18:46:26 -04:00
Jack Robison 3c231d6215 handle get with file_name/download_directory/save_file arg for a stream that is already running 2019-05-06 18:46:26 -04:00
Jack Robison a212cf6ba4 set content_fee attribute 2019-05-06 18:46:26 -04:00
Jack Robison bc060f1378 pay fee after downloading the sd blob/adding to the db 2019-05-06 18:46:26 -04:00
Jack Robison 22c701fd50 fix completed field for an item in file list 2019-05-06 18:46:26 -04:00
Jack Robison 4f0e8fce6e add optional download_directory argument to get 2019-05-06 18:46:26 -04:00
Jack Robison f642cfe9dd fix resuming save tasks on startup 2019-05-06 18:46:26 -04:00
Jack Robison 24e073680b add connection id workaround 2019-05-06 18:46:26 -04:00
Jack Robison b2f63a1545 fix tests 2019-05-06 18:46:26 -04:00
Jack Robison f506b3e6d4 tests 2019-05-06 18:46:26 -04:00
Jack Robison f7412514b3 fix partial content 0 padding 2019-05-06 18:46:26 -04:00
Jack Robison bd6a609b30 fix logging 2019-05-06 18:46:26 -04:00
Jack Robison 4e32b69d1d don't set running streams as stopped on startup 2019-05-06 18:46:26 -04:00
Jack Robison 1116c7f29e logging 2019-05-06 18:46:26 -04:00
Jack Robison d80c671cf2 look victor 2019-05-06 18:46:26 -04:00
Jack Robison 14b12cbea2 block /stream and /get requests on the stream manager starting 2019-05-06 18:46:26 -04:00
Jack Robison 1f7feafb67 force close open streaming requests and api calls on shutdown 2019-05-06 18:46:26 -04:00
Jack Robison cbe689ea7f logging 2019-05-06 18:46:26 -04:00
Jack Robison 9099ee2e8e fix/refactor starting and stopping files
-move partial content handling into ManagedStream
-add delayed stop test
2019-05-06 18:46:26 -04:00
Jack Robison b134e0c9c9 fix blob_get 2019-05-06 18:46:26 -04:00
Jack Robison f8c0e80cfc add __slots__ to BlobInfo, StreamDescriptor, and KademliaPeer 2019-05-06 18:46:26 -04:00
Jack Robison 1f266ebdad add logging for streams we already have 2019-05-06 18:46:26 -04:00
Jack Robison b09e1f8825 fix written_bytes never being 0 2019-05-06 18:46:26 -04:00
Jack Robison c496278a5b Merge pull request #2092 from lbryio/fix_announce_on_storage
Set should_announce=1 to head and descriptor blobs when saving a stream to database
2019-05-06 18:45:36 -04:00
Victor Shyba cacc65d45e fix edge case from test for single blob stream 2019-05-06 18:18:12 -03:00
Victor Shyba 2247b354b3 integration test for announcing both on publish and download 2019-05-06 17:27:12 -03:00
Victor Shyba 43111f0497 bump db revision 2019-05-06 17:19:40 -03:00
Victor Shyba 6c142b4477 add migrator for previous bugs 2019-05-06 17:12:04 -03:00
Victor Shyba e8683a24bd fix should_announce being set on stream creation 2019-05-06 16:49:52 -03:00
Victor Shyba a4d17d1705 re-enable tests for announcing on stream creation 2019-05-06 16:49:03 -03:00
Baltazar Gomez a030e13548 fix incorrect media-type of .cbr archives 2019-05-05 20:57:36 -04:00
Jack Robison 32b8601c38 Merge pull request #2083 from lbryio/fix_get_params
make save_file a parameter instead of a flag so it obeys conf
2019-05-05 19:44:29 -04:00
Victor Shyba f5bd28bfdb make save_file a parameter instead of a flag so it obeys conf 2019-05-05 19:46:42 -03:00
Lex Berezhny 6cbbe23c7c account_list defaults to 0 confirmations for balance 2019-05-04 18:08:33 -04:00
Lex Berezhny d7c2e4e9b3 fixed bug where lbrynet failed to start with single-key accounts in the wallet 2019-05-04 17:56:39 -04:00
Jack Robison d61ddbb950 Merge pull request #2076 from lbryio/download_improv
make ttfb configurable, fix race condition on writers, ban on time scales, remove idle code for getting unstuck on long downloads, fix foreign key error and improve downloader loop checks
2019-05-02 17:18:16 -04:00
Victor Shyba cf7bb6a391 fix tests and stream being deleted on data timeout 2019-05-02 16:41:48 -03:00
Victor Shyba 68d7328918 always close the write handle 2019-05-02 12:20:49 -03:00
Victor Shyba ff2985cc80 close protocol if transport reuse brings responses before requests 2019-05-02 12:13:33 -03:00
Victor Shyba ac69fcbc5f fix possible division by zero 2019-05-02 11:52:09 -03:00
Victor Shyba 609cf42868 cleanup downloader code 2019-05-01 18:04:45 -03:00
Victor Shyba e43b29fcd1 if the current writer is closed, its fine to open a new one 2019-05-01 15:09:11 -03:00
Victor Shyba 6594b7c50c test concurrency through races on getting a writer 2019-05-01 15:08:47 -03:00
Victor Shyba 971f2a1cef log download speed only if downloading full streams 2019-05-01 14:31:11 -03:00
Victor Shyba fc88261cc6 add head_blob_only to ttfb 2019-05-01 13:59:45 -03:00
Victor Shyba 895b3992c9 set type for stall download timeout 2019-05-01 13:26:51 -03:00
Victor Shyba 41951d8ec2 improve ttfb script confs 2019-05-01 13:17:13 -03:00
Victor Shyba ae2121c5c4 fix tests and new test 2019-05-01 13:16:45 -03:00
Victor Shyba 771fe935f4 test get-delete-get respects existing files 2019-05-01 13:16:32 -03:00
Victor Shyba 60f7766cf7 small adjustment to new_peer_or_finished 2019-05-01 02:55:51 -03:00
Victor Shyba 63bd983012 pylint 2019-05-01 00:42:51 -03:00
Victor Shyba 7ffce1040c fix foreign key error 2019-04-30 19:51:19 -03:00
Victor Shyba ac6835fa60 dont get stuck 2019-04-30 19:51:02 -03:00
Victor Shyba 3cf5c536c0 improve ban timing and downloader loop friction 2019-04-30 17:56:29 -03:00
Lex Berezhny 85cc1a4763 v0.37.0rc5 2019-04-29 17:22:47 -04:00
Lex Berezhny 180cfcfb72 fix chris45 tests 2019-04-29 16:01:59 -04:00
Lex Berezhny 864c1dd901 fix test_claim_commands integration tests 2019-04-29 16:01:59 -04:00
Lex Berezhny aeaa7ba986 switched from sha256 for unencrypted file hash to sha384 2019-04-29 16:01:59 -04:00
Lex Berezhny b8325c1f89 regenerated docs 2019-04-29 09:49:39 -04:00
Lex Berezhny a36936b5ff v0.37.0rc4 2019-04-29 00:27:53 -04:00
Victor Shyba 7cf245a0e9 encode timestamp and confirmations into file list 2019-04-29 00:26:14 -04:00
Victor Shyba 9f83cce574 test confirmations and timestamp on file list, being updated on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 30b113d64a depth -> confirmations 2019-04-29 00:26:14 -04:00
Victor Shyba ccd2023d37 add timestamp field to resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 3127cd11cf test for timestamp field on resolve 2019-04-29 00:26:14 -04:00
Victor Shyba 6e483cba8d tests: add confirm parameters, like the other calls 2019-04-28 22:53:43 -04:00
Victor Shyba 46c03c0cb2 refactor: move generate to blockchain call 2019-04-28 22:53:43 -04:00
Victor Shyba c2ca0a9467 tests: replace sleep with the proper generate call 2019-04-28 22:53:43 -04:00
Victor Shyba b350e74aca fix stream update callback to update the main dict 2019-04-28 22:53:43 -04:00
Victor Shyba de9473f75a test resolve updates file list 2019-04-28 22:53:43 -04:00
Lex Berezhny bfc107fdb3 removed blockchain.block.get_block from wallet server API 2019-04-27 18:03:35 -04:00
Lex Berezhny 6a2efa54ab dropped block_show command from API, users should run a full node if they need access to blocks 2019-04-27 18:03:35 -04:00
Lex Berezhny 72f0e1ca14 added timestamp field to transaction output results from JSON RPC calls 2019-04-27 17:10:49 -04:00
Lex Berezhny 9a1262d557 regenerate docs 2019-04-27 13:13:20 -04:00
Lex Berezhny 706a8e52dc v0.37.0rc3 2019-04-27 13:12:27 -04:00
Lex Berezhny f3c8776878 fixed --preview mode on stream_create/update 2019-04-27 13:11:32 -04:00
Lex Berezhny 539ea04039 fixed signing streams using --channel_name argument 2019-04-27 11:52:29 -04:00
Lex Berezhny 8f96b07559 updating stream claim with new file now correctly updates file list removing old file 2019-04-27 10:55:20 -04:00
Lex Berezhny 0a1e97fe23 cli test fix 2019-04-26 23:37:10 -04:00
Lex Berezhny da637b9c0e added --replace option to stream_update and channel_update, publish defaults to --replace 2019-04-26 23:37:10 -04:00
Lex Berezhny 3618de1227 added --clear_fee ability to stream update command 2019-04-26 17:52:29 -04:00
Victor Shyba 3cb8a63bf9 add channel_name parameter on claim search 2019-04-26 17:30:21 -04:00
Victor Shyba bad99de5b3 add tests for channel_name, which should work for urls too 2019-04-26 17:30:21 -04:00
Lex Berezhny 3d1efbc9bb v0.37.0rc2 2019-04-25 20:40:07 -04:00
Victor Shyba 7bdef3146b fix ttfb 2019-04-25 20:38:33 -04:00
Victor Shyba 2145bfcabf install uvloop on wallet server docker image and default to it 2019-04-25 20:07:10 -04:00
Lex Berezhny b88832d0a9 regenerated docs/api.json 2019-04-24 23:03:30 -04:00
Lex Berezhny db08882f20 v0.37.0rc1 2019-04-24 21:35:59 -04:00
Jack Robison 6f52d36b22 Merge pull request #2020 from lbryio/download-range-requests
Support HTTP 206 partial content requests for streaming downloads
2019-04-24 14:21:41 -04:00
Jack Robison 056a629e87 remove unused property 2019-04-24 13:39:06 -04:00
Jack Robison 1ffb99f9f9 rename streaming_only setting -> save_files 2019-04-24 12:44:12 -04:00
Jack Robison a8e612773d test 2019-04-24 09:48:04 -04:00
Jack Robison 67088b2ca2 test_blob_writers_concurrency 2019-04-24 09:48:04 -04:00
Jack Robison dab49e812f set writer result or error before calling close_handle 2019-04-24 09:48:04 -04:00
Jack Robison 0f3f38d8a3 pylint 2019-04-24 09:48:04 -04:00
Jack Robison 7ba433c2e5 fix setting blob verified event and canceling blob writers
-raise OSError if multiple writers for one peer are opened for a blob
2019-04-24 09:48:04 -04:00
Jack Robison bb7649e8ad test file_save stopped before finished writing with streaming_only on/off 2019-04-24 09:48:04 -04:00
Jack Robison 832537a5cf set deleted downloads as streaming mode on startup 2019-04-24 09:48:04 -04:00
Jack Robison 3234d70270 add peer address/port to hash blob writer 2019-04-24 09:48:04 -04:00
Jack Robison 432fe444f8 fix cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 861c25716e add unit tests for cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Victor Shyba 9ba6653475 raise proper error when sd is not available 2019-04-24 09:48:04 -04:00
Jack Robison 8107e97304 remove debugging /streams endpoint 2019-04-24 09:48:04 -04:00
Jack Robison e2663f6d57 remove unnecessary except/raise 2019-04-24 09:48:04 -04:00
Jack Robison 35479dcbd2 add file_save integration tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a997277aa reduce the amount of Blob objects created 2019-04-24 09:48:04 -04:00
Jack Robison a7d94f4514 add file_save 2019-04-24 09:48:04 -04:00
Jack Robison a0e3338177 only re-reflect streams if the sd blob is readable 2019-04-24 09:48:04 -04:00
Jack Robison d5a353066c missing await 2019-04-24 09:48:04 -04:00
Jack Robison bc29fc4465 type 2019-04-24 09:48:04 -04:00
Jack Robison 987f175fcc logging 2019-04-24 09:48:04 -04:00
Jack Robison 5b1398b588 rename test_streaming.py 2019-04-24 09:48:04 -04:00
Jack Robison 873b58efc9 update example_daemon_settings.yml 2019-04-24 09:48:04 -04:00
Jack Robison 01a81fceed more streaming_only + save_blobs integration testing 2019-04-24 09:48:04 -04:00
Jack Robison a14b0aa573 tests 2019-04-24 09:48:04 -04:00
Jack Robison a1abb7c8fb test range requests 2019-04-24 09:48:04 -04:00
Jack Robison 46b166952b non async blob_completed callback, tests 2019-04-24 09:48:04 -04:00
Jack Robison 1375153fda fix publish updates not being saved to the db, don't query the db for the stream hash 2019-04-24 09:48:04 -04:00
Jack Robison 1a14157d37 fix unawaited task 2019-04-24 09:48:04 -04:00
Jack Robison 40b9fa38b0 update mock dht 2019-04-24 09:48:04 -04:00
Jack Robison 4d0e9e0428 relative http redirect from /get -> /stream 2019-04-24 09:48:04 -04:00
Jack Robison 3a53ef5690 catch TransportNotConnected 2019-04-24 09:48:04 -04:00
Jack Robison 33a68b5cef handle dht is_closing 2019-04-24 09:48:04 -04:00
Jack Robison 63bf084a6a cancel dht response future 2019-04-24 09:48:04 -04:00
Jack Robison a882f2e8c5 merge conflicts 2019-04-24 09:48:04 -04:00
Jack Robison 119e0ec408 cancelled error 2019-04-24 09:48:04 -04:00
Jack Robison 7d782d516a track and close blob readers 2019-04-24 09:48:04 -04:00
Jack Robison d3eac48035 tests, add __slots__ to AbstractBlob 2019-04-24 09:48:04 -04:00
Jack Robison 60c9ae64b4 endpoint to call get and redirect to the partial content stream 2019-04-24 09:48:04 -04:00
Jack Robison 2d314dce60 add save_file argument to get 2019-04-24 09:48:04 -04:00
Jack Robison 287b89db66 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 7a160f7335 fix tests 2019-04-24 09:48:04 -04:00
Jack Robison 04ddee7d89 blob tests 2019-04-24 09:48:04 -04:00
Jack Robison 3a916a8e8e tests 2019-04-24 09:48:04 -04:00
Jack Robison f125468ebf fix claim not being saved to the db 2019-04-24 09:48:04 -04:00
Jack Robison f0e17cff9a support streaming downloads / range requests 2019-04-24 09:48:04 -04:00
Jack Robison 93267efe0b remove StreamAssembler, refactor ManagedStream and StreamDownloader (WIP) 2019-04-24 09:48:04 -04:00
Jack Robison c663e5a3cf add cache_concurrent decorator 2019-04-24 09:48:04 -04:00
Jack Robison 676f0015aa refactor BlobFile into AbstractBlob, BlobFile, and BlobBuffer classes 2019-04-24 09:48:04 -04:00
Jack Robison d44a79ada2 cache resolve_host, add async_timed_cache decorator 2019-04-24 09:48:04 -04:00
Jack Robison d7ed8ffaa6 add streaming_only and save_blobs settings 2019-04-24 09:48:04 -04:00
Jack Robison b230d8fbcb rename BlobFileManager -> BlobManager 2019-04-24 09:48:04 -04:00
Lex Berezhny 7d2428fd17 regenerate api.json 2019-04-23 23:49:39 -04:00
Lex Berezhny 4157045a41 block_show command example 2019-04-23 23:49:39 -04:00
Lex Berezhny 58987857cf v0.36.0 2019-04-22 18:47:58 -04:00
Lex Berezhny fa8858325c add lbry:// to permanent_url response field 2019-04-22 18:41:11 -04:00
Lex Berezhny 3357a419ab ability to overide the file name, hash and size when updating a stream claim 2019-04-22 18:05:49 -04:00
Lex Berezhny a782adea75 regenerate api doc 2019-04-22 14:06:42 -04:00
Lex Berezhny e8991e6d52 improve account locking/unlocing CLI docs 2019-04-22 14:06:42 -04:00
Lex Berezhny 13bd81c2b6 v0.36.0rc4 2019-04-21 23:11:23 -04:00
Victor Shyba 21241b98a4 workaround for having new encoder fields on resolve so app can use them 2019-04-21 22:51:48 -04:00
Lex Berezhny bf467e06c5 added create stream image detection test 2019-04-21 22:12:02 -04:00
Lex Berezhny 886ed6415a added examples which include all metadata passed to create stream and create channel 2019-04-21 21:34:13 -04:00
Lex Berezhny 97db241109 v0.36.0rc3 2019-04-21 20:20:53 -04:00
Lex Berezhny 216379918b renamed txo sub_type to value_type in txo json encoder output 2019-04-21 17:40:40 -04:00
Lex Berezhny fcab4eb9bd make sure to use python3 when running set_build.py 2019-04-21 16:31:46 -04:00
Lex Berezhny 19f300cc45 added protobuf to get and file_list commands 2019-04-21 14:28:43 -04:00
Lex Berezhny 26fec098e4 fix stream unit tests 2019-04-21 14:28:43 -04:00
Lex Berezhny 7824de655e updated stream manager to pull protobuf instead of hex from resolve response 2019-04-21 14:28:43 -04:00
Lex Berezhny 6ef1b1dd59 resolve command also returns protobuf 2019-04-21 14:28:43 -04:00
Lex Berezhny 9fae8f5153 added --include_protobuf to add hexlified protobuf to RPC responses 2019-04-21 14:28:43 -04:00
Lex Berezhny f660691e78 v0.36.0rc2 2019-04-21 00:50:27 -04:00
Lex Berezhny bf60683ba6 update to protobuf change where ClaimList defaults to collection list type 2019-04-21 00:45:51 -04:00
Lex Berezhny 68215699f7 fix chri45 test 2019-04-21 00:45:51 -04:00
Lex Berezhny 1644ffbf22 regenerated docs 2019-04-21 00:45:51 -04:00
Lex Berezhny 71f5061848 exposed new protobuf fields in APIs and updated tests 2019-04-21 00:45:51 -04:00
Lex Berezhny 61cadb5cbe v0.36.0rc1 2019-04-20 02:34:27 -04:00
Lex Berezhny 2034104cfc test fixes 2019-04-20 02:33:41 -04:00
Lex Berezhny 633f725634 stream manager test fix 2019-04-20 02:33:41 -04:00
Lex Berezhny 8bf2e41c95 --contact_email -> --email and --homepage_url -> --website_url 2019-04-20 02:33:41 -04:00
Lex Berezhny fa9a4a75c6 update to an even newer protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny 90bef98bc3 use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 35c43647aa use TXORef.hash instead of TXORef.id when signing claim with channel 2019-04-20 02:33:41 -04:00
Lex Berezhny 449a583286 default to claimType == 1 when it is missing for old claim protobufs 2019-04-20 02:33:41 -04:00
Lex Berezhny b1f813782a regenerated protobuf python code 2019-04-20 02:33:41 -04:00
Lex Berezhny 60080cbd19 updating to new protobufs 2019-04-20 02:33:41 -04:00
Victor Shyba 665dba62c7 simplify save from resolve, deduplicate items 2019-04-20 01:19:40 -04:00
Victor Shyba 76dea9e89b batch save supports 2019-04-20 01:19:40 -04:00
Victor Shyba ebc9ed01c5 make lru_cache a power of two, following py docs 2019-04-20 01:19:40 -04:00
Victor Shyba 55116763e9 cache transactions 2019-04-20 01:19:40 -04:00
Victor Shyba e1a6b4e3d6 fix dict handling mistake 2019-04-20 01:19:40 -04:00
Victor Shyba 911ca8c37e cache and share txs fetches while validating concurrently on batches 2019-04-20 01:19:40 -04:00
jessop 7f5deaf6c8 adds api examples to docs 2019-04-16 14:18:23 -04:00
Lex Berezhny 0200756218 put generated commands in groups 2019-04-12 14:24:16 -04:00
ポール ウェッブ a63442f843 Fixes and updated gitignore 2019-04-12 11:53:37 -04:00
Oleg Silkin 74b3da79aa Adds integration tests 2019-04-07 15:27:59 -04:00
Oleg Silkin b6a4589da5 Adds comment group to lbrynet with commands list and create 2019-04-07 15:27:59 -04:00
Oleg Silkin fbbb0a9931 Adds server URL to configuration settings 2019-04-07 15:27:59 -04:00
Oleg Silkin a94d65d522 Adds utility functions for making batch jsonrpc requests to comment server 2019-04-07 15:27:59 -04:00
Oleg Silkin fa2ead002e Fixes incompatible attrs issue when running pip install -e . 2019-04-06 18:43:34 -04:00
Thomas Zarebczan 8dfee4867a Update python3 > python, add mac instructions 2019-04-06 17:10:32 -04:00
Thomas Zarebczan 6614ab8e79 updated with new venv command 2019-04-06 17:10:32 -04:00
Thomas Zarebczan d32c1dfbb3 update install for windowz
Didn't need to install any build tools
Also, venv commands are a bit different.
2019-04-06 17:10:32 -04:00
Lex Berezhny 9f42b958fa run examples fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 574a912f7d run examples as part of ci 2019-04-06 17:08:15 -04:00
Lex Berezhny 2cfcc3cd45 test fix 2019-04-06 17:08:15 -04:00
Lex Berezhny 9af625e627 test fixes 2019-04-06 17:08:15 -04:00
Lex Berezhny 1871c94003 new examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 93220c3874 daemon json rpc return type docs 2019-04-06 17:08:15 -04:00
Lex Berezhny 362ed3f1c8 remove unused import 2019-04-06 17:08:15 -04:00
Lex Berezhny 62a182b8bb automated example generator 2019-04-06 17:08:15 -04:00
Lex Berezhny 6f76d9e24a moved integration testcase.py into lbrynet so that it can be used from scripts 2019-04-06 17:08:15 -04:00
Lex Berezhny 55e83fe05c regenerate api.json 2019-04-06 17:08:15 -04:00
Lex Berezhny 2f0c7d6a3f added title to examples 2019-04-06 17:08:15 -04:00
Lex Berezhny 59e1df9efa generate api examples 2019-04-06 17:08:15 -04:00
Victor Shyba fd56dc68cd fix depth on resolve, add tests 2019-04-05 09:40:26 -04:00
Victor Shyba a0a810edc3 make permanent url name#claim_id on torba too 2019-04-05 09:40:26 -04:00
Victor Shyba c1d3b5984d fix old hash reverse confusion 2019-04-05 09:40:26 -04:00
Victor Shyba 1fb719dd61 fix valid_at_height not being shown for name resolutions, which fixes tests 2019-04-05 09:40:26 -04:00
Victor Shyba b87316fb02 make permanent url name#id only for now 2019-04-05 09:40:26 -04:00
Victor Shyba 65f3d78084 fix invalid signatures for missing certificates 2019-04-05 09:40:26 -04:00
Victor Shyba 47bc1526f7 test invalid signature to be False, not accepting None 2019-04-05 09:40:26 -04:00
Victor Shyba 3acabec4bb fixes from review 2019-04-03 00:51:05 -04:00
Victor Shyba 36a62d3756 port orchstr8 fix to our test case as its lbrynet specific 2019-04-03 00:51:05 -04:00
Victor Shyba 2e22473ce3 pylint and smaller fixes 2019-04-03 00:51:05 -04:00
Victor Shyba 0b0fd64642 further refactor resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba ca17d8597d move resolve from ledger to resolve.py 2019-04-03 00:51:05 -04:00
Victor Shyba d16e2716c8 handle invalid data 2019-04-03 00:51:05 -04:00
Victor Shyba 89c80cedfd test resolve for invalid data 2019-04-03 00:51:05 -04:00
Thomas Zarebczan 021e30ffde fix: extension check
disregard the first period, otherwise the `filename.` test fails.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan eb2cd8fec1 stream type mapping
Mapping extensions to stream types. I went with document / model / binary for the ones we were missing. This can be adjusted later on if not correct, but at least we'll have the main ones. Added extensions for lbry, cbz, cbr also.
2019-04-01 22:56:40 -04:00
Thomas Zarebczan 6356818557 fix: time to first byte script
Fix with for metadata
2019-04-01 16:14:53 -04:00
YULIUS KURNIAWAN KRISTIANTO 8bf724a78b Update CONTRIBUTING.md 2019-04-01 16:12:49 -04:00
YULIUS KURNIAWAN KRISTIANTO 65d0a924c1 Update README.md 2019-04-01 16:12:49 -04:00
Lex Berezhny ef7bd00f22 round USD fees to nearest penny 2019-03-31 18:14:13 -04:00
92 changed files with 8715 additions and 4590 deletions

No files matched your search

+3
View File
@@ -1,3 +1,4 @@
.DS_Store
/build
/dist
/.tox
@@ -8,3 +9,5 @@
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+12 -1
View File
@@ -35,6 +35,17 @@ jobs:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
@@ -69,7 +80,7 @@ jobs:
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- python scripts/set_build.py
- python3 scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
+1 -1
View File
@@ -1,3 +1,3 @@
## Contributing to LBRY
https://lbry.io/faq/contributing
https://lbry.tech/contribute
+11 -12
View File
@@ -13,15 +13,6 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
After installing python 3, you'll need to install some additional libraries depending on your operating system.
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
### macOS
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
@@ -29,15 +20,17 @@ macOS users will need to install [xcode command line tools](https://developer.xa
Remaining dependencies can then be installed by running:
```
brew install python3 protobuf
brew install python protobuf
```
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
### Linux
On Ubuntu (we recommend 18.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
@@ -52,8 +45,14 @@ To install:
git clone https://github.com/lbryio/lbry.git
cd lbry
virtualenv lbry-venv --python=python3.7
Creating venv:
python -m venv lbry-venv
Activating lbry-venv on Linux/Mac:
source lbry-venv/bin/activate
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
+3 -3
View File
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
+2634 -1332
View File
File diff suppressed because it is too large. Load diff
+4 -3
View File
@@ -12,12 +12,13 @@ blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
delete_blobs_on_remove: True
save_blobs: true
save_files: false
dht_node_port: 4444
peer_port: 3333
use_upnp: True
use_upnp: true
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - blob_server
# - dht
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.35.0"
__version__ = "0.37.0rc6"
version = tuple(__version__.split('.'))
+270 -132
View File
@@ -4,6 +4,8 @@ import asyncio
import binascii
import logging
import typing
import contextlib
from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
@@ -21,10 +23,6 @@ log = logging.getLogger(__name__)
_hexmatch = re.compile("^[a-f,0-9]+$")
def is_valid_hashcharacter(char: str) -> bool:
return len(char) == 1 and _hexmatch.match(char)
def is_valid_blobhash(blobhash: str) -> bool:
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@@ -46,143 +44,74 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
return encrypted, digest.hexdigest()
class BlobFile:
"""
A chunk of data available on the network which is specified by a hashsum
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
if len(data) != length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
This class is used to create blobs on the local filesystem
when we already know the blob hash before hand (i.e., when downloading blobs)
Also can be used for reading from blobs on the local filesystem
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
class AbstractBlob:
"""
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
This class is non-io specific
"""
__slots__ = [
'loop',
'blob_hash',
'length',
'blob_completed_callback',
'blob_directory',
'writers',
'verified',
'writing',
'readers'
]
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self.loop = loop
self.blob_hash = blob_hash
self.length = length
self.blob_dir = blob_dir
self.file_path = os.path.join(blob_dir, self.blob_hash)
self.writers: typing.List[HashBlobWriter] = []
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=loop)
self.blob_write_lock = asyncio.Lock(loop=loop)
if self.file_exists:
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
self.length = length
self.verified.set()
self.finished_writing.set()
self.saved_verified_blob = False
self.blob_completed_callback = blob_completed_callback
self.blob_directory = blob_directory
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
self.readers: typing.List[typing.BinaryIO] = []
@property
def file_exists(self):
return os.path.isfile(self.file_path)
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
def __del__(self):
if self.writers or self.readers:
log.warning("%s not closed before being garbage collected", self.blob_hash)
self.close()
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
raise NotImplementedError()
@contextlib.contextmanager
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
with self._reader_context() as reader:
try:
error = finished.exception()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
self.writers.remove(writer)
if not error: # the blob downloaded, cancel all the other download attempts and set the result
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
log.exception("something else")
raise error
return callback
self.readers.append(reader)
yield reader
finally:
if reader in self.readers:
self.readers.remove(reader)
async def save_verified_blob(self, writer, verified_bytes: bytes):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
async with self.blob_write_lock:
if self.verified.is_set():
return
await self.loop.run_in_executor(None, _save_verified)
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers.append(writer)
fut.add_done_callback(self.writer_finished(writer))
return writer
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
"""
with open(self.file_path, 'rb') as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def close(self):
while self.writers:
self.writers.pop().finished.cancel()
def delete(self):
self.close()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
self.verified.clear()
self.finished_writing.clear()
self.length = None
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with open(self.file_path, "rb") as f:
buff = f.read()
if len(buff) != self.length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
@classmethod
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_dir, blob_hash, length)
writer = blob.open_for_writing()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def set_length(self, length):
def set_length(self, length) -> None:
if self.length is not None and length == self.length:
return
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
@@ -190,8 +119,217 @@ class BlobFile:
return
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
def get_length(self):
def get_length(self) -> typing.Optional[int]:
return self.length
def get_is_verified(self):
def get_is_verified(self) -> bool:
return self.verified.is_set()
def is_readable(self) -> bool:
return self.verified.is_set()
def is_writeable(self) -> bool:
return not self.writing.is_set()
def write_blob(self, blob_bytes: bytes):
if not self.is_writeable():
raise OSError("cannot open blob for writing")
try:
self.writing.set()
self._write_blob(blob_bytes)
finally:
self.writing.clear()
def close(self):
while self.writers:
peer, writer = self.writers.popitem()
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
writer.finished.cancel()
while self.readers:
reader = self.readers.pop()
if reader:
reader.close()
def delete(self):
self.close()
self.verified.clear()
self.length = None
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
"""
if not self.is_readable():
raise OSError('blob files cannot be read')
with self.reader_context() as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with self.reader_context() as reader:
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
writer = blob.get_blob_writer()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers[(peer_address, peer_port)] = writer
def remove_writer(_):
if (peer_address, peer_port) in self.writers:
del self.writers[(peer_address, peer_port)]
fut.add_done_callback(remove_writer)
def writer_finished_callback(finished: asyncio.Future):
try:
err = finished.exception()
if err:
raise err
verified_bytes = finished.result()
while self.writers:
_, other = self.writers.popitem()
if other is not writer:
other.close_handle()
self.save_verified_blob(verified_bytes)
except (InvalidBlobHashError, InvalidDataError) as error:
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
pass
fut.add_done_callback(writer_finished_callback)
return writer
class BlobBuffer(AbstractBlob):
"""
An in-memory only blob
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self._verified_bytes: typing.Optional[BytesIO] = None
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError("cannot open blob for reading")
try:
yield self._verified_bytes
finally:
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
return super().delete()
def __del__(self):
super().__del__()
if self._verified_bytes:
self.delete()
class BlobFile(AbstractBlob):
"""
A blob existing on the local file system
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
if not blob_directory or not os.path.isdir(blob_directory):
raise OSError(f"invalid blob directory '{blob_directory}'")
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
if self.file_exists:
file_size = int(os.stat(self.file_path).st_size)
if length and length != file_size:
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
self.delete()
else:
self.length = file_size
self.verified.set()
@property
def file_exists(self):
return os.path.isfile(self.file_path)
def is_writeable(self) -> bool:
return super().is_writeable() and not os.path.isfile(self.file_path)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
return super().get_blob_writer(peer_address, peer_port)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
handle = open(self.file_path, 'rb')
try:
yield handle
finally:
handle.close()
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def delete(self):
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> BlobInfo:
if not blob_dir or not os.path.isdir(blob_dir):
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
return await super().create_from_unencrypted(
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
)
+7
View File
@@ -2,6 +2,13 @@ import typing
class BlobInfo:
__slots__ = [
'blob_hash',
'blob_num',
'length',
'iv',
]
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
self.blob_hash = blob_hash
self.blob_num = blob_num
+60 -22
View File
@@ -2,18 +2,19 @@ import os
import typing
import asyncio
import logging
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobFileManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
class BlobManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
@@ -27,16 +28,59 @@ class BlobFileManager:
self._node_data_store = node_data_store
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, BlobFile] = {}
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
buffer = self.blobs.pop(blob_hash)
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
if buffer.is_readable():
with buffer.reader_context() as reader:
self.blobs[blob_hash].write_blob(reader.read())
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
return self.blobs[blob_hash]
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
def get_files_in_blob_dir() -> typing.Set[str]:
if not self.blob_dir:
return set()
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
return True
def stop(self):
@@ -45,37 +89,31 @@ class BlobFileManager:
blob.close()
self.completed_blob_hashes.clear()
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
return self.blobs[blob_hash]
def get_stream_descriptor(self, sd_hash):
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
async def blob_completed(self, blob: BlobFile):
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
if blob.blob_hash is None:
raise Exception("Blob hash is None")
if not blob.length:
raise Exception("Blob has a length of 0")
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
if isinstance(blob, BlobFile):
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
else:
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
blobs = [self.get_blob(b) for b in blob_hashes]
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
if blob_hash not in self.blobs:
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
+3 -2
View File
@@ -44,16 +44,15 @@ class HashBlobWriter:
self._hashsum.update(data)
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
self.close_handle()
return
self.buffer.write(data)
if self.len_so_far == expected_length:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
@@ -62,6 +61,8 @@ class HashBlobWriter:
self.close_handle()
def close_handle(self):
if not self.finished.done():
self.finished.cancel()
if self.buffer is not None:
self.buffer.close()
self.buffer = None
+36 -16
View File
@@ -4,8 +4,9 @@ import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
@@ -17,15 +18,18 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.peer_timeout = peer_timeout
self.transport: asyncio.Transport = None
self.transport: typing.Optional[asyncio.Transport] = None
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._response_fut: typing.Optional[asyncio.Future] = None
self.buf = b''
# this is here to handle the race when the downloader is closed right as response_fut gets a result
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
@@ -35,6 +39,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if not self._response_fut:
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
return self.close()
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
@@ -63,8 +70,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def _write(self, data):
def _write(self, data: bytes):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
@@ -87,6 +93,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
@@ -95,6 +102,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
@@ -133,6 +144,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return self._blob_bytes_received, self.close()
def close(self):
self.closed.set()
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
@@ -145,18 +157,21 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
self.closed.clear()
blob_hash = blob.blob_hash
if blob.get_is_verified() or not blob.is_writeable():
return 0, self.transport
try:
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self._blob_bytes_received = 0
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
# i'm not sure how to fix this race condition - jack
log.exception(e)
return self._blob_bytes_received, self.transport
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
@@ -165,6 +180,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
except asyncio.CancelledError:
self.close()
raise
finally:
if self.writer and not self.writer.closed():
self.writer.close_handle()
self.writer = None
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
@@ -177,9 +196,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.close()
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None)\
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
@@ -196,7 +216,7 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: s
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or blob.file_exists:
if blob.get_is_verified() or not blob.is_writeable():
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
+41 -56
View File
@@ -1,21 +1,23 @@
import asyncio
import typing
import logging
from lbrynet.utils import drain_tasks
from lbrynet.utils import cache_concurrent
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
log = logging.getLogger(__name__)
class BlobDownloader:
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
BAN_CAP = 30.0*60.0 # maximum time a node remains banned before being retried
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
@@ -24,21 +26,16 @@ class BlobDownloader:
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.failures: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.time_since_last_blob = loop.time()
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'BlobFile'):
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
# the safe net side is that any failure will reset the peer score, triggering the race back
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
# for peer, task in self.active_connections.items():
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
# return False
return not (blob.get_is_verified() or blob.file_exists)
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
if blob.get_is_verified():
return
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
@@ -46,92 +43,80 @@ class BlobDownloader:
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
)
if bytes_received == blob.get_length():
self.time_since_last_blob = self.loop.time()
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = self.failures.get(peer, 0) + 1
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = 0
self.connections[peer] = transport
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
self.scores[peer] = rough_speed
elapsed = self.loop.time() - start
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 0
async def new_peer_or_finished(self, blob: 'BlobFile'):
async def get_and_re_add_peers():
try:
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
self.peer_queue.put_nowait(new_peers)
except asyncio.TimeoutError:
pass
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
active_tasks = list(self.active_connections.values())
try:
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
finally:
drain_tasks(tasks)
async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
def cleanup_active(self):
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
self.clearbanned()
def clearbanned(self):
now = self.loop.time()
if now - self.time_since_last_blob > 60.0:
return
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
self.peer_queue.put_nowait(forgiven)
for banned_peer in forgiven:
self.ignored.pop(banned_peer)
self.ignored = dict((
(peer, when) for (peer, when) in self.ignored.items()
if (now - when) < min(self.BAN_CAP, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
))
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
@cache_concurrent
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
connection_id: int = 0) -> 'AbstractBlob':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
self.is_running.set()
try:
while not blob.get_is_verified():
batch: typing.List['KademliaPeer'] = []
while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set()
while not self.peer_queue.empty():
batch.extend(self.peer_queue.get_nowait())
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
batch.update(self.peer_queue.get_nowait())
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in batch:
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
await self.new_peer_or_finished(blob)
await self.new_peer_or_finished()
self.cleanup_active()
if batch:
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
else:
self.clearbanned()
blob.close()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
while self.active_connections:
self.active_connections.popitem()[1].cancel()
blob.close()
def close(self):
self.scores.clear()
self.ignored.clear()
self.is_running.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
blob_hash: str) -> 'BlobFile':
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
blob_hash: str) -> 'AbstractBlob':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
+3 -3
View File
@@ -8,13 +8,13 @@ from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPr
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
log = logging.getLogger(__name__)
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -94,7 +94,7 @@ class BlobServerProtocol(asyncio.Protocol):
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
+8 -2
View File
@@ -480,10 +480,12 @@ class Config(CLIConfig):
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
@@ -493,7 +495,7 @@ class Config(CLIConfig):
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 8,
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
@@ -528,12 +530,16 @@ class Config(CLIConfig):
('lbrynet4.lbry.io', 4444) # ASIA
])
# comment server uses a single string because requests are made to the /api resource
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
+33 -44
View File
@@ -13,55 +13,44 @@ class BlobAnnouncer:
self.loop = loop
self.node = node
self.storage = storage
self.pending_call: asyncio.Handle = None
self.announce_task: asyncio.Task = None
self.running = False
self.announce_queue: typing.List[str] = []
async def _announce(self, batch_size: typing.Optional[int] = 10):
if not batch_size:
return
if not self.node.joined.is_set():
await self.node.joined.wait()
blob_hashes = await self.storage.get_blobs_to_announce()
if blob_hashes:
self.announce_queue.extend(blob_hashes)
log.info("%i blobs to announce", len(blob_hashes))
batch = []
while len(self.announce_queue):
cnt = 0
announced = []
while self.announce_queue and cnt < batch_size:
blob_hash = self.announce_queue.pop()
announced.append(blob_hash)
batch.append(self.node.announce_blob(blob_hash))
cnt += 1
to_await = []
while batch:
to_await.append(batch.pop())
if to_await:
await asyncio.gather(*tuple(to_await), loop=self.loop)
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
if self.running:
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
async def _submit_announcement(self, blob_hash):
try:
peers = len(await self.node.announce_blob(blob_hash))
if peers > 4:
return blob_hash
else:
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
def announce(self, batch_size: typing.Optional[int] = 10):
self.announce_task = self.loop.create_task(self._announce(batch_size))
async def _announce(self, batch_size: typing.Optional[int] = 10):
while batch_size:
if not self.node.joined.is_set():
await self.node.joined.wait()
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
while len(self.announce_queue):
log.info("%i blobs to announce", len(self.announce_queue))
announced = await asyncio.gather(*[
self._submit_announcement(
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
], loop=self.loop)
announced = list(filter(None, announced))
if announced:
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
await asyncio.sleep(60)
def start(self, batch_size: typing.Optional[int] = 10):
if self.running:
raise Exception("already running")
self.running = True
self.announce(batch_size)
assert not self.announce_task or self.announce_task.done(), "already running"
self.announce_task = self.loop.create_task(self._announce(batch_size))
def stop(self):
self.running = False
if self.pending_call:
if not self.pending_call.cancelled():
self.pending_call.cancel()
self.pending_call = None
if self.announce_task:
if not (self.announce_task.done() or self.announce_task.cancelled()):
self.announce_task.cancel()
self.announce_task = None
if self.announce_task and not self.announce_task.done():
self.announce_task.cancel()
-1
View File
@@ -201,7 +201,6 @@ class Node:
try:
async with peer_generator as junction:
yield junction
await peer_generator.finished.wait()
finally:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
+9
View File
@@ -135,6 +135,15 @@ class PeerManager:
class KademliaPeer:
__slots__ = [
'loop',
'_node_id',
'address',
'udp_port',
'tcp_port',
'protocol_version',
]
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
if node_id is not None:
@@ -26,12 +26,11 @@ class AsyncGeneratorJunction:
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
self.loop = loop
self.__iterator_queue = asyncio.Queue(loop=loop)
self.result_queue = queue or asyncio.Queue(loop=loop)
self.tasks: typing.List[asyncio.Task] = []
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
self.can_iterate = asyncio.Event(loop=self.loop)
self.finished = asyncio.Event(loop=self.loop)
@property
def running(self):
@@ -42,15 +41,16 @@ class AsyncGeneratorJunction:
try:
async for item in iterator:
self.result_queue.put_nowait(item)
self.__iterator_queue.put_nowait(item)
finally:
self.running_iterators[iterator] = False
if not self.running:
self.__iterator_queue.put_nowait(StopAsyncIteration)
while True:
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
self.running_iterators[async_gen] = True
self.tasks.append(self.loop.create_task(iterate(async_gen)))
if not self.can_iterate.is_set():
self.can_iterate.set()
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
"""
@@ -62,14 +62,10 @@ class AsyncGeneratorJunction:
return self
async def __anext__(self):
if not self.can_iterate.is_set():
await self.can_iterate.wait()
if not self.running:
raise StopAsyncIteration()
try:
return await self.result_queue.get()
finally:
self.awaiting = None
result = await self.__iterator_queue.get()
if result is StopAsyncIteration:
raise result
return result
def aclose(self):
async def _aclose():
@@ -80,8 +76,6 @@ class AsyncGeneratorJunction:
self.running_iterators[iterator] = False
drain_tasks(self.tasks)
raise StopAsyncIteration()
if not self.finished.is_set():
self.finished.set()
return self.loop.create_task(_aclose())
async def __aenter__(self):
+4 -2
View File
@@ -5,7 +5,7 @@ from itertools import chain
import typing
import logging
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.error import RemoteException, TransportNotConnected
from lbrynet.dht.protocol.distance import Distance
from typing import TYPE_CHECKING
@@ -169,6 +169,8 @@ class IterativeFinder:
log.warning(str(err))
self.active.discard(peer)
return
except TransportNotConnected:
return self.aclose()
except RemoteException:
return
return await self._handle_probe_result(peer, response)
@@ -215,7 +217,7 @@ class IterativeFinder:
await self._search_round()
if self.running:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration):
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
if self.running:
self.loop.call_soon(self.aclose)
+5 -1
View File
@@ -524,6 +524,10 @@ class KademliaProtocol(DatagramProtocol):
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except asyncio.CancelledError:
if not response_fut.done():
response_fut.cancel()
raise
except (asyncio.TimeoutError, RemoteException):
self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
@@ -540,7 +544,7 @@ class KademliaProtocol(DatagramProtocol):
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
ErrorDatagram]):
if not self.transport:
if not self.transport or self.transport.is_closing():
raise TransportNotConnected()
data = message.bencode()
+6
View File
@@ -32,6 +32,12 @@ class DownloadDataTimeout(Exception):
self.download = download
class ResolveTimeout(Exception):
def __init__(self, uri):
super().__init__(f'Failed to resolve "{uri}" within the timeout')
self.uri = uri
class RequestCanceledError(Exception):
pass
+3 -6
View File
@@ -25,7 +25,7 @@ def display(data):
print(json.dumps(data, indent=2))
async def execute_command(conf, method, params):
async def execute_command(conf, method, params, callback=display):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
@@ -33,12 +33,9 @@ async def execute_command(conf, method, params):
try:
data = await resp.json()
if 'result' in data:
display(data['result'])
return callback(data['result'])
elif 'error' in data:
if 'message' in data['error']:
display(data['error']['message'])
else:
display(data['error'])
return callback(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
+11 -9
View File
@@ -16,7 +16,7 @@ from lbrynet import utils
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.dht.node import Node
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.extras.daemon.Component import Component
@@ -63,7 +63,7 @@ class DatabaseComponent(Component):
@staticmethod
def get_current_db_revision():
return 9
return 11
@property
def revision_filename(self):
@@ -278,10 +278,10 @@ class BlobComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_manager: BlobFileManager = None
self.blob_manager: BlobManager = None
@property
def component(self) -> typing.Optional[BlobFileManager]:
def component(self) -> typing.Optional[BlobManager]:
return self.blob_manager
async def start(self):
@@ -291,8 +291,10 @@ class BlobComponent(Component):
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
data_store = dht_node.protocol.data_store
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
storage, data_store)
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
if not os.path.isdir(blob_dir):
os.mkdir(blob_dir)
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
return await self.blob_manager.setup()
async def stop(self):
@@ -451,7 +453,7 @@ class PeerProtocolServerComponent(Component):
async def start(self):
log.info("start blob server")
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = self.conf.tcp_port
address = await wallet.get_unused_address()
@@ -485,7 +487,7 @@ class UPnPComponent(Component):
while True:
if now:
await self._maintain_redirects()
await asyncio.sleep(360)
await asyncio.sleep(360, loop=self.component_manager.loop)
async def _maintain_redirects(self):
# setup the gateway if necessary
@@ -528,7 +530,7 @@ class UPnPComponent(Component):
self.upnp_redirects.update(upnp_redirects)
except (asyncio.TimeoutError, UPnPError):
self.upnp = None
return self._maintain_redirects()
return
elif self.upnp: # check existing redirects are still active
found = set()
mappings = await self.upnp.get_redirects()
File diff suppressed because it is too large. Load diff
+3 -24
View File
@@ -1,27 +1,6 @@
import aiohttp
import logging
from lbrynet.conf import Config
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
from lbrynet.extras.cli import execute_command
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
async def daemon_rpc(conf: Config, method: str, *args, **kwargs):
async with aiohttp.ClientSession() as session:
message = {'method': method, 'params': [args, kwargs]}
async with session.get(conf.api_connection_url, json=message) as resp:
data = await resp.json()
if 'result' in data:
return data['result']
elif 'error' in data:
raise JSONRPCException(data['error'])
def daemon_rpc(conf: Config, method: str, **kwargs):
return execute_command(conf, method, kwargs, callback=lambda data: data)
+31
View File
@@ -0,0 +1,31 @@
import logging
import aiohttp
log = logging.getLogger(__name__)
def rpc_body(method: str, rpc_id: any, **params) -> dict:
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
async def jsonrpc_post(url: str, method: str, **params) -> any:
clean = params.pop('clean', True)
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
if clean:
if 'error' in response:
return response['error']
return response['result']
else:
return response
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
headers = {'Content-Type': 'application/json'}
complete = []
batch_size = max(batch_size, 50)
for i in range(0, len(calls), batch_size):
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
complete += await response.json()
if clean:
complete = [body['result'] if 'result' in body else None for body in complete]
return complete
+142 -39
View File
@@ -5,41 +5,116 @@ from datetime import datetime
from json import JSONEncoder
from ecdsa import BadSignatureError
from lbrynet.schema.claim import Claim
from lbrynet.wallet.ledger import MainNetLedger
from lbrynet.wallet.ledger import MainNetLedger, Account
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.stream.managed_stream import ManagedStream
log = logging.getLogger(__name__)
def encode_txo_doc():
return {
'txid': "hash of transaction in hex",
'nout': "position in the transaction",
'height': "block where transaction was recorded",
'amount': "value of the txo as a decimal",
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'payment'",
'name': "when type is 'claim' or 'support', this is the claim name",
'claim_id': "when type is 'claim' or 'support', this is the claim id",
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
'protobuf': "hex encoded raw protobuf version of 'value' field",
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
'signing_channel': "for signed claims only, metadata of signing channel",
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
}
def encode_tx_doc():
return {
'txid': "hash of transaction in hex",
'height': "block where transaction was recorded",
'inputs': [encode_txo_doc()],
'outputs': [encode_txo_doc()],
'total_input': "sum of inputs as a decimal",
'total_output': "sum of outputs, sans fee, as a decimal",
'total_fee': "fee amount",
'hex': "entire transaction encoded in hex",
}
def encode_account_doc():
return {
'id': 'account_id',
'is_default': 'this account is used by default',
'ledger': 'name of crypto currency and network',
'name': 'optional account name',
'seed': 'human friendly words from which account can be recreated',
'encrypted': 'if account is encrypted',
'private_key': 'extended private key',
'public_key': 'extended public key',
'address_generator': 'settings for generating addresses',
'modified_on': 'date of last modification to account settings'
}
def encode_file_doc():
return {
'completed': '(bool) true if download is completed',
'file_name': '(str) name of file',
'download_directory': '(str) download directory',
'points_paid': '(float) credit paid to download file',
'stopped': '(bool) true if download is stopped',
'stream_hash': '(str) stream hash of file',
'stream_name': '(str) stream name',
'suggested_file_name': '(str) suggested file name',
'sd_hash': '(str) sd hash of file',
'download_path': '(str) download path of file',
'mime_type': '(str) mime type of file',
'key': '(str) key attached to file',
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
'total_bytes': '(int) file upper bound size in bytes',
'written_bytes': '(int) written size in bytes',
'blobs_completed': '(int) number of fully downloaded blobs',
'blobs_in_stream': '(int) total blobs on stream',
'blobs_remaining': '(int) total blobs remaining to download',
'status': '(str) downloader status',
'claim_id': '(str) None if claim is not found else the claim id',
'txid': '(str) None if claim is not found else the transaction id',
'nout': '(int) None if claim is not found else the transaction output index',
'outpoint': '(str) None if claim is not found else the tx and output',
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
}
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
self.include_protobuf = include_protobuf
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Account):
return self.encode_account(obj)
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, Claim):
claim_dict = obj.to_dict()
if obj.is_stream:
claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
fee = claim_dict['stream'].get('fee', {})
if 'address' in fee:
fee['address'] = obj.stream.fee.address
if 'amount' in fee:
fee['amount'] = obj.stream.fee.amount
if 'languages' in claim_dict['stream']:
claim_dict['stream']['languages'] = obj.stream.langtags
elif obj.is_channel:
claim_dict['channel']['public_key'] = obj.channel.public_key
if 'languages' in claim_dict['channel']:
claim_dict['channel']['languages'] = obj.channel.langtags
return claim_dict
return self.encode_claim(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -66,50 +141,56 @@ class JSONResponseEncoder(JSONEncoder):
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'height': tx_height,
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_name:
output['type'] = 'claim'
output['claim_op'] = 'create'
elif txo.script.is_update_claim:
output['type'] = 'claim'
output['claim_op'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'payment'
if txo.script.is_claim_involved:
output.update({
'name': txo.claim_name,
'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url,
})
if txo.script.is_claim_name or txo.script.is_update_claim:
claim = txo.claim
output['value'] = claim
if claim.is_signed:
output['valid_signature'] = None
if check_signature and txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
output['value'] = txo.claim
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.channel is not None:
output['signing_channel'] = {
'name': txo.channel.claim_name,
'claim_id': txo.channel.claim_id,
'value': txo.channel.claim
}
if check_signature and txo.claim.is_signed:
output['is_channel_signature_valid'] = False
try:
output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
except BadSignatureError:
output['valid_signature'] = False
pass
except ValueError:
log.exception(
'txo.id: %s, txo.channel.id:%s, output: %s',
txo.id, txo.channel.id, output
)
output['valid_signature'] = False
if txo.script.is_claim_name:
output['type'] = 'claim'
elif txo.script.is_update_claim:
output['type'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'basic'
return output
def encode_input(self, txi):
@@ -117,3 +198,25 @@ class JSONResponseEncoder(JSONEncoder):
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
def encode_account(self, account):
result = account.to_dict()
result['id'] = account.id
result.pop('certificates', None)
result['is_default'] = self.ledger.accounts[0] == account
return result
def encode_file(self, managed_stream):
file = managed_stream.as_dict()
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
file.update({
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
})
return file
@staticmethod
def encode_claim(claim):
return getattr(claim, claim.claim_type).to_dict()
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
from .migrate7to8 import do_migration
elif current == 8:
from .migrate8to9 import do_migration
elif current == 9:
from .migrate9to10 import do_migration
elif current == 10:
from .migrate10to11 import do_migration
else:
raise Exception("DB migration of version {} to {} is not available".format(current,
current+1))
@@ -0,0 +1,54 @@
import sqlite3
import os
import binascii
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
current_columns = []
for col_info in cursor.execute("pragma table_info('file');").fetchall():
current_columns.append(col_info[1])
if 'content_fee' in current_columns or 'saved_file' in current_columns:
connection.close()
print("already migrated")
return
cursor.execute(
"pragma foreign_keys=off;"
)
cursor.execute("""
create table if not exists new_file (
stream_hash text primary key not null references stream,
file_name text,
download_directory text,
blob_data_rate real not null,
status text not null,
saved_file integer not null,
content_fee text
);
""")
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
saved_file = 0
if download_dir != '{stream}' and file_name != '{stream}':
try:
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
binascii.unhexlify(file_name).decode())):
saved_file = 1
else:
download_dir, file_name = None, None
except Exception:
download_dir, file_name = None, None
else:
download_dir, file_name = None, None
cursor.execute(
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
)
cursor.execute("drop table file")
cursor.execute("alter table new_file rename to file")
connection.commit()
connection.close()
@@ -0,0 +1,20 @@
import sqlite3
import os
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
query = "select stream_hash, sd_hash from main.stream"
for stream_hash, sd_hash in cursor.execute(query).fetchall():
head_blob_hash = cursor.execute(
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
(stream_hash,)
).fetchone()
if not head_blob_hash:
continue
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
connection.commit()
connection.close()
+160 -79
View File
@@ -8,6 +8,7 @@ import time
from torba.client.basedatabase import SQLiteMixin
from lbrynet.conf import Config
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet.wallet.transaction import Transaction
from lbrynet.schema.claim import Claim
from lbrynet.dht.constants import data_expiration
from lbrynet.blob.blob_info import BlobInfo
@@ -54,10 +55,6 @@ class StoredStreamClaim:
def nout(self) -> typing.Optional[int]:
return None if not self.outpoint else int(self.outpoint.split(":")[1])
@property
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.claim else self.claim.claim_dict['stream']['metadata']
def as_dict(self) -> typing.Dict:
return {
"name": self.claim_name,
@@ -118,8 +115,8 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name, *claim_args) in _batched_select(
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
@@ -145,7 +142,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
"key": stream_key,
"stream_name": stream_name, # hex
"suggested_file_name": suggested_file_name, # hex
"claim": claim
"claim": claim,
"saved_file": bool(saved_file),
"content_fee": None if not raw_content_fee else Transaction(
binascii.unhexlify(raw_content_fee)
)
}
)
for claim_name, claim_id in _batched_select(
@@ -157,21 +158,12 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
# add all blobs, except the last one, which is empty
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
@@ -183,6 +175,11 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
# ensure should_announce is set regardless if insert was ignored
transaction.execute(
"update blob set should_announce=1 where blob_hash in (?, ?)",
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
)
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
@@ -195,13 +192,21 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
data_payment_rate: float, status: str) -> int:
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
content_fee: typing.Optional[Transaction]) -> int:
if not file_name and not download_directory:
encoded_file_name, encoded_download_dir = None, None
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
transaction.execute(
"insert or replace into file values (?, ?, ?, ?, ?)",
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?)",
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
)
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
@@ -250,10 +255,12 @@ class SQLiteStorage(SQLiteMixin):
create table if not exists file (
stream_hash text primary key not null references stream,
file_name text not null,
download_directory text not null,
file_name text,
download_directory text,
blob_data_rate real not null,
status text not null
status text not null,
saved_file integer not null,
content_fee text
);
create table if not exists content_claim (
@@ -296,27 +303,28 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # blob functions # # # # # # # # #
def add_completed_blob(self, blob_hash: str, length: int):
def _add_blob(transaction: sqlite3.Connection):
transaction.execute(
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
def _add_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
(blob_hash, length, 0, 0, "pending", 0, 0)
[
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
for blob_hash, length in blob_hashes_and_lengths
]
)
transaction.execute(
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
)
return self.db.run(_add_blob)
if finished:
transaction.executemany(
"update blob set status='finished' where blob.blob_hash=?", [
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
]
)
return await self.db.run(_add_blobs)
def get_blob_status(self, blob_hash: str):
return self.run_and_return_one_or_none(
"select status from blob where blob_hash=?", blob_hash
)
def add_known_blob(self, blob_hash: str, length: int):
return self.db.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
)
def should_announce(self, blob_hash: str):
return self.run_and_return_one_or_none(
"select should_announce from blob where blob_hash=?", blob_hash
@@ -419,6 +427,26 @@ class SQLiteStorage(SQLiteMixin):
}
return self.db.run(_sync_blobs)
def sync_files_to_blobs(self):
def _sync_blobs(transaction: sqlite3.Connection):
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
transaction.execute(
"select distinct sb.stream_hash from stream_blob sb "
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
).fetchall()
)
return self.db.run(_sync_blobs)
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
def _set_streaming(transaction: sqlite3.Connection):
transaction.executemany(
"update file set file_name=null, download_directory=null where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
return self.db.run(_set_streaming)
# # # # # # # # # stream functions # # # # # # # # #
async def stream_exists(self, sd_hash: str) -> bool:
@@ -481,20 +509,53 @@ class SQLiteStorage(SQLiteMixin):
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
)
def get_stream_info_for_sd_hash(self, sd_blob_hash):
return self.run_and_return_one_or_none(
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
sd_blob_hash
)
def delete_stream(self, descriptor: 'StreamDescriptor'):
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
# # # # # # # # # file stuff # # # # # # # # #
def save_downloaded_file(self, stream_hash, file_name, download_directory,
data_payment_rate) -> typing.Awaitable[int]:
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float,
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
return self.save_published_file(
stream_hash, file_name, download_directory, data_payment_rate, status="running"
stream_hash, file_name, download_directory, data_payment_rate, status="running",
content_fee=content_fee
)
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
status="finished") -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], data_payment_rate: float,
status: str = "finished",
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
content_fee)
async def update_manually_removed_files_since_last_run(self):
"""
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1"
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
else:
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
)
return await self.db.run(update_manually_removed_files)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
@@ -503,24 +564,45 @@ class SQLiteStorage(SQLiteMixin):
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
stream_hash
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
file_name: typing.Optional[str]):
if not file_name or not download_dir:
encoded_file_name, encoded_download_dir = None, None
else:
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
encoded_download_dir, encoded_file_name, stream_hash,
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
binascii.hexlify(content_fee.raw), stream_hash,
))
async def set_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
stream_hash,
))
async def clear_saved_file(self, stream_hash: str):
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
stream_hash,
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
typing.Optional[Transaction]]],
download_directory: str):
def _recover(transaction: sqlite3.Connection):
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
for descriptor, sd_blob in descriptors_and_sds:
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
for descriptor, sd_blob, content_fee in descriptors_and_sds:
content_claim = transaction.execute(
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
).fetchone()
delete_stream(transaction, descriptor) # this will also delete the content claim
store_stream(transaction, sd_blob, descriptor)
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
download_directory, 0.0, 'stopped')
download_directory, 0.0, 'stopped', content_fee=content_fee)
if content_claim:
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
@@ -539,16 +621,18 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # support functions # # # # # # # # #
def save_supports(self, claim_id, supports):
def save_supports(self, claim_id_to_supports: dict):
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
def _save_support(transaction):
transaction.execute("delete from support where claim_id=?", (claim_id,))
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
)
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
for claim_id, supports in claim_id_to_supports.items():
for support in supports:
transaction.execute(
"insert into support values (?, ?, ?, ?)",
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
support.get('address', ""))
)
return self.db.run(_save_support)
def get_supports(self, *claim_ids):
@@ -576,7 +660,7 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # claim functions # # # # # # # # #
async def save_claims(self, claim_infos):
support_callbacks = []
claim_id_to_supports = {}
update_file_callbacks = []
def _save_claims(transaction):
@@ -591,7 +675,7 @@ class SQLiteStorage(SQLiteMixin):
sequence = claim_info['claim_sequence']
certificate_id = claim_info['value'].signing_channel_id
try:
source_hash = claim_info['value'].stream.sd_hash
source_hash = claim_info['value'].stream.source.sd_hash
except (AttributeError, ValueError):
source_hash = None
serialized = binascii.hexlify(claim_info['value'].to_bytes())
@@ -602,7 +686,7 @@ class SQLiteStorage(SQLiteMixin):
# if this response doesn't have support info don't overwrite the existing
# support info
if 'supports' in claim_info:
support_callbacks.append((claim_id, claim_info['supports']))
claim_id_to_supports[claim_id] = claim_info['supports']
if not source_hash:
continue
stream_hash = transaction.execute(
@@ -632,23 +716,20 @@ class SQLiteStorage(SQLiteMixin):
await self.db.run(_save_claims)
if update_file_callbacks:
await asyncio.wait(update_file_callbacks)
if support_callbacks:
await asyncio.wait([
self.save_supports(*args) for args in support_callbacks
])
if claim_id_to_supports:
await self.save_supports(claim_id_to_supports)
def save_claims_for_resolve(self, claim_infos):
to_save = []
to_save = {}
for info in claim_infos:
if 'value' in info:
if info['value']:
to_save.append(info)
to_save[info['claim_id']] = info
else:
if 'certificate' in info and info['certificate']['value']:
to_save.append(info['certificate'])
if 'claim' in info and info['claim']['value']:
to_save.append(info['claim'])
return self.save_claims(to_save)
for key in ('certificate', 'claim'):
if info.get(key, {}).get('value'):
to_save[info[key]['claim_id']] = info[key]
return self.save_claims(to_save.values())
@staticmethod
def _save_content_claim(transaction, claim_outpoint, stream_hash):
@@ -671,7 +752,7 @@ class SQLiteStorage(SQLiteMixin):
if not known_sd_hash:
raise Exception("stream not found")
# check the claim contains the same sd hash
if known_sd_hash[0] != claim.stream.sd_hash:
if known_sd_hash[0] != claim.stream.source.sd_hash:
raise Exception("stream mismatch")
# if there is a current claim associated to the file, check that the new claim is an update to it
+521
View File
@@ -0,0 +1,521 @@
import json
import logging
import os.path
import hashlib
from typing import Tuple, List
from string import ascii_letters
from decimal import Decimal, ROUND_UP
from binascii import hexlify, unhexlify
from google.protobuf.json_format import MessageToDict
from torba.client.hash import Base58
from torba.client.constants import COIN
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.schema.base import Metadata, BaseMessageList
from lbrynet.schema.types.v2.claim_pb2 import (
Fee as FeeMessage,
Location as LocationMessage,
Language as LanguageMessage
)
log = logging.getLogger(__name__)
def calculate_sha384_file_hash(file_path):
sha384 = hashlib.sha384()
with open(file_path, 'rb') as f:
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
sha384.update(chunk)
return sha384.digest()
class Dimmensional(Metadata):
__slots__ = ()
@property
def width(self) -> int:
return self.message.width
@width.setter
def width(self, width: int):
self.message.width = width
@property
def height(self) -> int:
return self.message.height
@height.setter
def height(self, height: int):
self.message.height = height
@property
def dimensions(self) -> Tuple[int, int]:
return self.width, self.height
@dimensions.setter
def dimensions(self, dimensions: Tuple[int, int]):
self.message.width, self.message.height = dimensions
def _extract(self, file_metadata, field):
try:
setattr(self, field, file_metadata.getValues(field)[0])
except:
log.exception(f'Could not extract {field} from file metadata.')
def update(self, file_metadata=None, height=None, width=None):
if height is not None:
self.height = height
elif file_metadata:
self._extract(file_metadata, 'height')
if width is not None:
self.width = width
elif file_metadata:
self._extract(file_metadata, 'width')
class Playable(Metadata):
__slots__ = ()
@property
def duration(self) -> int:
return self.message.duration
@duration.setter
def duration(self, duration: int):
self.message.duration = duration
def update(self, file_metadata=None, duration=None):
if duration is not None:
self.duration = duration
elif file_metadata:
try:
self.duration = file_metadata.getValues('duration')[0].seconds
except:
log.exception('Could not extract duration from file metadata.')
class Image(Dimmensional):
__slots__ = ()
class Audio(Playable):
__slots__ = ()
class Video(Dimmensional, Playable):
__slots__ = ()
def update(self, file_metadata=None, height=None, width=None, duration=None):
Dimmensional.update(self, file_metadata, height, width)
Playable.update(self, file_metadata, duration)
class Source(Metadata):
__slots__ = ()
def update(self, file_path=None):
if file_path is not None:
self.name = os.path.basename(file_path)
self.media_type, stream_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.size = os.path.getsize(file_path)
if self.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
return stream_type
@property
def name(self) -> str:
return self.message.name
@name.setter
def name(self, name: str):
self.message.name = name
@property
def size(self) -> int:
return self.message.size
@size.setter
def size(self, size: int):
self.message.size = size
@property
def media_type(self) -> str:
return self.message.media_type
@media_type.setter
def media_type(self, media_type: str):
self.message.media_type = media_type
@property
def file_hash(self) -> str:
return hexlify(self.message.hash).decode()
@file_hash.setter
def file_hash(self, file_hash: str):
self.message.hash = unhexlify(file_hash.encode())
@property
def file_hash_bytes(self) -> bytes:
return self.message.hash
@file_hash_bytes.setter
def file_hash_bytes(self, file_hash_bytes: bytes):
self.message.hash = file_hash_bytes
@property
def sd_hash(self) -> str:
return hexlify(self.message.sd_hash).decode()
@sd_hash.setter
def sd_hash(self, sd_hash: str):
self.message.sd_hash = unhexlify(sd_hash.encode())
@property
def sd_hash_bytes(self) -> bytes:
return self.message.sd_hash
@sd_hash_bytes.setter
def sd_hash_bytes(self, sd_hash: bytes):
self.message.sd_hash = sd_hash
@property
def url(self) -> str:
return self.message.url
@url.setter
def url(self, url: str):
self.message.url = url
class Fee(Metadata):
__slots__ = ()
def update(self, address: str = None, currency: str = None, amount=None):
if address is not None:
self.address = address
if currency is not None and amount is not None:
currency = currency.lower()
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
setattr(self, currency, Decimal(amount))
@property
def currency(self) -> str:
return FeeMessage.Currency.Name(self.message.currency)
@property
def address(self) -> str:
return Base58.encode(self.message.address)
@address.setter
def address(self, address: str):
self.message.address = Base58.decode(address)
@property
def address_bytes(self) -> bytes:
return self.message.address
@address_bytes.setter
def address_bytes(self, address: bytes):
self.message.address = address
@property
def amount(self) -> Decimal:
if self.currency == 'LBC':
return self.lbc
if self.currency == 'BTC':
return self.btc
if self.currency == 'USD':
return self.usd
DEWIES = Decimal(COIN)
@property
def lbc(self) -> Decimal:
if self.message.currency != FeeMessage.LBC:
raise ValueError('LBC can only be returned for LBC fees.')
return Decimal(self.message.amount / self.DEWIES)
@lbc.setter
def lbc(self, amount: Decimal):
self.dewies = int(amount * self.DEWIES)
@property
def dewies(self) -> int:
if self.message.currency != FeeMessage.LBC:
raise ValueError('Dewies can only be returned for LBC fees.')
return self.message.amount
@dewies.setter
def dewies(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.LBC
SATOSHIES = Decimal(COIN)
@property
def btc(self) -> Decimal:
if self.message.currency != FeeMessage.BTC:
raise ValueError('BTC can only be returned for BTC fees.')
return Decimal(self.message.amount / self.SATOSHIES)
@btc.setter
def btc(self, amount: Decimal):
self.satoshis = int(amount * self.SATOSHIES)
@property
def satoshis(self) -> int:
if self.message.currency != FeeMessage.BTC:
raise ValueError('Satoshies can only be returned for BTC fees.')
return self.message.amount
@satoshis.setter
def satoshis(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.BTC
PENNIES = Decimal('100.0')
PENNY = Decimal('0.01')
@property
def usd(self) -> Decimal:
if self.message.currency != FeeMessage.USD:
raise ValueError('USD can only be returned for USD fees.')
return Decimal(self.message.amount / self.PENNIES)
@usd.setter
def usd(self, amount: Decimal):
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
@property
def pennies(self) -> int:
if self.message.currency != FeeMessage.USD:
raise ValueError('Pennies can only be returned for USD fees.')
return self.message.amount
@pennies.setter
def pennies(self, amount: int):
self.message.amount = amount
self.message.currency = FeeMessage.USD
class ClaimReference(Metadata):
__slots__ = ()
@property
def claim_id(self) -> str:
return hexlify(self.claim_hash[::-1]).decode()
@claim_id.setter
def claim_id(self, claim_id: str):
self.claim_hash = unhexlify(claim_id)[::-1]
@property
def claim_hash(self) -> bytes:
return self.message.claim_hash
@claim_hash.setter
def claim_hash(self, claim_hash: bytes):
self.message.claim_hash = claim_hash
class ClaimList(BaseMessageList[ClaimReference]):
__slots__ = ()
item_class = ClaimReference
@property
def _message(self):
return self.message.claim_references
def append(self, value):
self.add().claim_id = value
@property
def ids(self) -> List[str]:
return [c.claim_id for c in self]
class Language(Metadata):
__slots__ = ()
@property
def langtag(self) -> str:
langtag = []
if self.language:
langtag.append(self.language)
if self.script:
langtag.append(self.script)
if self.region:
langtag.append(self.region)
return '-'.join(langtag)
@langtag.setter
def langtag(self, langtag: str):
parts = langtag.split('-')
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2:
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@property
def language(self) -> str:
if self.message.language:
return LanguageMessage.Language.Name(self.message.language)
@language.setter
def language(self, language: str):
self.message.language = LanguageMessage.Language.Value(language)
@property
def script(self) -> str:
if self.message.script:
return LanguageMessage.Script.Name(self.message.script)
@script.setter
def script(self, script: str):
self.message.script = LanguageMessage.Script.Value(script)
@property
def region(self) -> str:
if self.message.region:
return LocationMessage.Country.Name(self.message.region)
@region.setter
def region(self, region: str):
self.message.region = LocationMessage.Country.Value(region)
class LanguageList(BaseMessageList[Language]):
__slots__ = ()
item_class = Language
def append(self, value: str):
self.add().langtag = value
class Location(Metadata):
__slots__ = ()
def from_value(self, value):
if isinstance(value, str) and value.startswith('{'):
value = json.loads(value)
if isinstance(value, dict):
for key, val in value.items():
setattr(self, key, val)
elif isinstance(value, str):
parts = value.split(':')
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
country = parts and parts.pop(0)
if country:
self.country = country
state = parts and parts.pop(0)
if state:
self.state = state
city = parts and parts.pop(0)
if city:
self.city = city
code = parts and parts.pop(0)
if code:
self.code = code
latitude = parts and parts.pop(0)
if latitude:
self.latitude = latitude
longitude = parts and parts.pop(0)
if longitude:
self.longitude = longitude
else:
raise ValueError(f'Could not parse country value: {value}')
def to_dict(self):
d = MessageToDict(self.message)
if self.message.longitude:
d['longitude'] = self.longitude
if self.message.latitude:
d['latitude'] = self.latitude
return d
@property
def country(self) -> str:
if self.message.country:
return LocationMessage.Country.Name(self.message.country)
@country.setter
def country(self, country: str):
self.message.country = LocationMessage.Country.Value(country)
@property
def state(self) -> str:
return self.message.state
@state.setter
def state(self, state: str):
self.message.state = state
@property
def city(self) -> str:
return self.message.city
@city.setter
def city(self, city: str):
self.message.city = city
@property
def code(self) -> str:
return self.message.code
@code.setter
def code(self, code: str):
self.message.code = code
GPS_PRECISION = Decimal('10000000')
@property
def latitude(self) -> str:
if self.message.latitude:
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
@latitude.setter
def latitude(self, latitude: str):
latitude = Decimal(latitude)
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
self.message.latitude = int(latitude * self.GPS_PRECISION)
@property
def longitude(self) -> str:
if self.message.longitude:
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
@longitude.setter
def longitude(self, longitude: str):
longitude = Decimal(longitude)
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
self.message.longitude = int(longitude * self.GPS_PRECISION)
class LocationList(BaseMessageList[Location]):
__slots__ = ()
item_class = Location
def append(self, value):
self.add().from_value(value)
+50 -3
View File
@@ -1,4 +1,5 @@
from binascii import hexlify, unhexlify
from typing import List, Iterator, TypeVar, Generic
from google.protobuf.message import DecodeError
from google.protobuf.json_format import MessageToDict
@@ -21,9 +22,10 @@ class Signable:
self.unsigned_payload = None
self.signing_channel_hash = None
@property
def is_undetermined(self):
return self.message.WhichOneof('type') is None
def clear_signature(self):
self.signature = None
self.unsigned_payload = None
self.signing_channel_hash = None
@property
def signing_channel_id(self):
@@ -72,3 +74,48 @@ class Signable:
def __bytes__(self):
return self.to_bytes()
class Metadata:
__slots__ = 'message',
def __init__(self, message):
self.message = message
I = TypeVar('I')
class BaseMessageList(Metadata, Generic[I]):
__slots__ = ()
item_class = None
@property
def _message(self):
return self.message
def add(self) -> I:
return self.item_class(self._message.add())
def extend(self, values: List[str]):
for value in values:
self.append(value)
def append(self, value: str):
raise NotImplemented
def __len__(self):
return len(self._message)
def __iter__(self) -> Iterator[I]:
for item in self._message:
yield self.item_class(item)
def __getitem__(self, item) -> I:
return self.item_class(self._message[item])
def __delitem__(self, key):
del self._message[key]
+251 -603
View File
@@ -1,76 +1,85 @@
import os.path
import json
from string import ascii_letters
from typing import List, Tuple, Iterator, TypeVar, Generic
from decimal import Decimal
import logging
from typing import List
from binascii import hexlify, unhexlify
from google.protobuf.json_format import MessageToDict
from google.protobuf.message import DecodeError
from hachoir.core.log import log as hachoir_log
from hachoir.parser import createParser as binary_file_parser
from hachoir.metadata import extractMetadata as binary_file_metadata
from hachoir.core.log import log as hachoir_log
from torba.client.hash import Base58
from torba.client.constants import COIN
from lbrynet.schema import compat
from lbrynet.schema.base import Signable
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.schema.types.v2.claim_pb2 import (
Claim as ClaimMessage,
Fee as FeeMessage,
Location as LocationMessage,
Language as LanguageMessage
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
from lbrynet.schema.attrs import (
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
LanguageList, LocationList, ClaimList, ClaimReference
)
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
hachoir_log.use_print = False
log = logging.getLogger(__name__)
class Claim(Signable):
STREAM = 'stream'
CHANNEL = 'channel'
COLLECTION = 'collection'
REPOST = 'repost'
__slots__ = 'version',
message_class = ClaimMessage
def __init__(self, claim_message=None):
super().__init__(claim_message)
def __init__(self, message=None):
super().__init__(message)
self.version = 2
@property
def claim_type(self) -> str:
return self.message.WhichOneof('type')
def get_message(self, type_name):
message = getattr(self.message, type_name)
if self.claim_type is None:
message.SetInParent()
if self.claim_type != type_name:
raise ValueError(f'Claim is not a {type_name}.')
return message
@property
def is_stream(self):
return self.message.WhichOneof('type') == 'stream'
@property
def is_channel(self):
return self.message.WhichOneof('type') == 'channel'
@property
def stream_message(self):
if self.is_undetermined:
self.message.stream.SetInParent()
if not self.is_stream:
raise ValueError('Claim is not a stream.')
return self.message.stream
return self.claim_type == self.STREAM
@property
def stream(self) -> 'Stream':
return Stream(self)
@property
def channel_message(self):
if self.is_undetermined:
self.message.channel.SetInParent()
if not self.is_channel:
raise ValueError('Claim is not a channel.')
return self.message.channel
def is_channel(self):
return self.claim_type == self.CHANNEL
@property
def channel(self) -> 'Channel':
return Channel(self)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
@property
def is_repost(self):
return self.claim_type == self.REPOST
@property
def repost(self) -> 'Repost':
return Repost(self)
@property
def is_collection(self):
return self.claim_type == self.COLLECTION
@property
def collection(self) -> 'Collection':
return Collection(self)
@classmethod
def from_bytes(cls, data: bytes) -> 'Claim':
@@ -89,450 +98,78 @@ class Claim(Signable):
return claim
I = TypeVar('I')
class BaseMessageList(Generic[I]):
__slots__ = 'message',
item_class = None
def __init__(self, message):
self.message = message
def add(self) -> I:
return self.item_class(self.message.add())
def extend(self, values: List[str]):
for value in values:
self.append(value)
def append(self, value: str):
raise NotImplemented
def __len__(self):
return len(self.message)
def __iter__(self) -> Iterator[I]:
for lang in self.message:
yield self.item_class(lang)
def __getitem__(self, item) -> I:
return self.item_class(self.message[item])
class Dimmensional:
__slots__ = ()
@property
def width(self) -> int:
return self.message.width
@width.setter
def width(self, width: int):
self.message.width = width
@property
def height(self) -> int:
return self.message.height
@height.setter
def height(self, height: int):
self.message.height = height
@property
def dimensions(self) -> Tuple[int, int]:
return self.width, self.height
@dimensions.setter
def dimensions(self, dimensions: Tuple[int, int]):
self.message.width, self.message.height = dimensions
class Playable:
__slots__ = ()
@property
def duration(self) -> int:
return self.message.duration
@duration.setter
def duration(self, duration: int):
self.message.duration = duration
def set_duration_from_path(self, file_path):
try:
file_metadata = binary_file_metadata(binary_file_parser(file_path))
self.duration = file_metadata.getValues('duration')[0].seconds
except:
pass
class Image(Dimmensional):
__slots__ = 'message',
def __init__(self, image_message):
self.message = image_message
class Video(Dimmensional, Playable):
__slots__ = 'message',
def __init__(self, video_message):
self.message = video_message
class Audio(Playable):
__slots__ = 'message',
def __init__(self, audio_message):
self.message = audio_message
class File:
__slots__ = '_file',
def __init__(self, file_message):
self._file = file_message
@property
def name(self) -> str:
return self._file.name
@name.setter
def name(self, name: str):
self._file.name = name
@property
def size(self) -> int:
return self._file.size
@size.setter
def size(self, size: int):
self._file.size = size
class Fee:
__slots__ = '_fee',
def __init__(self, fee_message):
self._fee = fee_message
@property
def currency(self) -> str:
return FeeMessage.Currency.Name(self._fee.currency)
@property
def address(self) -> str:
return Base58.encode(self._fee.address)
@address.setter
def address(self, address: str):
self._fee.address = Base58.decode(address)
@property
def address_bytes(self) -> bytes:
return self._fee.address
@address_bytes.setter
def address_bytes(self, address: bytes):
self._fee.address = address
@property
def amount(self) -> Decimal:
if self.currency == 'LBC':
return self.lbc
if self.currency == 'BTC':
return self.btc
if self.currency == 'USD':
return self.usd
DEWIES = Decimal(COIN)
@property
def lbc(self) -> Decimal:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('LBC can only be returned for LBC fees.')
return Decimal(self._fee.amount / self.DEWIES)
@lbc.setter
def lbc(self, amount: Decimal):
self.dewies = int(amount * self.DEWIES)
@property
def dewies(self) -> int:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('Dewies can only be returned for LBC fees.')
return self._fee.amount
@dewies.setter
def dewies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.LBC
SATOSHIES = Decimal(COIN)
@property
def btc(self) -> Decimal:
if self._fee.currency != FeeMessage.BTC:
raise ValueError('BTC can only be returned for BTC fees.')
return Decimal(self._fee.amount / self.SATOSHIES)
@btc.setter
def btc(self, amount: Decimal):
self.satoshis = int(amount * self.SATOSHIES)
@property
def satoshis(self) -> int:
if self._fee.currency != FeeMessage.BTC:
raise ValueError('Satoshies can only be returned for BTC fees.')
return self._fee.amount
@satoshis.setter
def satoshis(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.BTC
PENNIES = Decimal(100.0)
@property
def usd(self) -> Decimal:
if self._fee.currency != FeeMessage.USD:
raise ValueError('USD can only be returned for USD fees.')
return Decimal(self._fee.amount / self.PENNIES)
@usd.setter
def usd(self, amount: Decimal):
self.pennies = int(amount * self.PENNIES)
@property
def pennies(self) -> int:
if self._fee.currency != FeeMessage.USD:
raise ValueError('Pennies can only be returned for USD fees.')
return self._fee.amount
@pennies.setter
def pennies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.USD
class Language:
__slots__ = 'message',
def __init__(self, message):
self.message = message
@property
def langtag(self) -> str:
langtag = []
if self.language:
langtag.append(self.language)
if self.script:
langtag.append(self.script)
if self.region:
langtag.append(self.region)
return '-'.join(langtag)
@langtag.setter
def langtag(self, langtag: str):
parts = langtag.split('-')
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2:
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@property
def language(self) -> str:
if self.message.language:
return LanguageMessage.Language.Name(self.message.language)
@language.setter
def language(self, language: str):
self.message.language = LanguageMessage.Language.Value(language)
@property
def script(self) -> str:
if self.message.script:
return LanguageMessage.Script.Name(self.message.script)
@script.setter
def script(self, script: str):
self.message.script = LanguageMessage.Script.Value(script)
@property
def region(self) -> str:
if self.message.region:
return LocationMessage.Country.Name(self.message.region)
@region.setter
def region(self, region: str):
self.message.region = LocationMessage.Country.Value(region)
class LanguageList(BaseMessageList[Language]):
__slots__ = ()
item_class = Language
def append(self, value: str):
self.add().langtag = value
class Location:
__slots__ = 'message',
def __init__(self, message):
self.message = message
def from_value(self, value):
if isinstance(value, str) and value.startswith('{'):
value = json.loads(value)
if isinstance(value, dict):
for key, val in value.items():
setattr(self, key, val)
elif isinstance(value, str):
parts = value.split(':')
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
country = parts and parts.pop(0)
if country:
self.country = country
state = parts and parts.pop(0)
if state:
self.state = state
city = parts and parts.pop(0)
if city:
self.city = city
code = parts and parts.pop(0)
if code:
self.code = code
latitude = parts and parts.pop(0)
if latitude:
self.latitude = latitude
longitude = parts and parts.pop(0)
if longitude:
self.longitude = longitude
else:
raise ValueError(f'Could not parse country value: {value}')
@property
def country(self) -> str:
if self.message.country:
return LocationMessage.Country.Name(self.message.country)
@country.setter
def country(self, country: str):
self.message.country = LocationMessage.Country.Value(country)
@property
def state(self) -> str:
return self.message.state
@state.setter
def state(self, state: str):
self.message.state = state
@property
def city(self) -> str:
return self.message.city
@city.setter
def city(self, city: str):
self.message.city = city
@property
def code(self) -> str:
return self.message.code
@code.setter
def code(self, code: str):
self.message.code = code
GPS_PRECISION = Decimal('10000000')
@property
def latitude(self) -> str:
if self.message.latitude:
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
@latitude.setter
def latitude(self, latitude: str):
latitude = Decimal(latitude)
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
self.message.latitude = int(latitude * self.GPS_PRECISION)
@property
def longitude(self) -> str:
if self.message.longitude:
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
@longitude.setter
def longitude(self, longitude: str):
longitude = Decimal(longitude)
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
self.message.longitude = int(longitude * self.GPS_PRECISION)
class LocationList(BaseMessageList[Location]):
__slots__ = ()
item_class = Location
def append(self, value):
self.add().from_value(value)
class BaseClaimSubType:
class BaseClaim:
__slots__ = 'claim', 'message'
def __init__(self, claim: Claim):
claim_type = None
object_fields = 'thumbnail',
repeat_fields = 'tags', 'languages', 'locations'
def __init__(self, claim: Claim = None):
self.claim = claim or Claim()
self.message = self.claim.get_message(self.claim_type)
def to_dict(self):
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
claim.update(claim.pop(self.claim_type))
if 'languages' in claim:
claim['languages'] = self.langtags
if 'locations' in claim:
claim['locations'] = [l.to_dict() for l in self.locations]
return claim
def update(self, **kwargs):
for key in list(kwargs):
for field in self.object_fields:
if key.startswith(f'{field}_'):
attr = getattr(self, field)
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
continue
for l in self.repeat_fields:
field = getattr(self, l)
if kwargs.pop(f'clear_{l}', False):
del field[:]
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
field.append(items)
elif isinstance(items, list):
field.extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
for key, value in kwargs.items():
setattr(self, key, value)
@property
def title(self) -> str:
return self.message.title
return self.claim.message.title
@title.setter
def title(self, title: str):
self.message.title = title
self.claim.message.title = title
@property
def description(self) -> str:
return self.message.description
return self.claim.message.description
@description.setter
def description(self, description: str):
self.message.description = description
self.claim.message.description = description
@property
def thumbnail_url(self) -> str:
return self.message.thumbnail_url
@thumbnail_url.setter
def thumbnail_url(self, thumbnail_url: str):
self.message.thumbnail_url = thumbnail_url
def thumbnail(self) -> Source:
return Source(self.claim.message.thumbnail)
@property
def tags(self) -> List:
return self.message.tags
return self.claim.message.tags
@property
def languages(self) -> LanguageList:
return LanguageList(self.message.languages)
return LanguageList(self.claim.message.languages)
@property
def langtags(self) -> List[str]:
@@ -540,151 +177,79 @@ class BaseClaimSubType:
@property
def locations(self) -> LocationList:
return LocationList(self.message.locations)
return LocationList(self.claim.message.locations)
class Stream(BaseClaim):
__slots__ = ()
claim_type = Claim.STREAM
object_fields = BaseClaim.object_fields + ('source',)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
claim = super().to_dict()
if 'source' in claim:
if 'hash' in claim['source']:
claim['source']['hash'] = self.source.file_hash
if 'sd_hash' in claim['source']:
claim['source']['sd_hash'] = self.source.sd_hash
fee = claim.get('fee', {})
if 'address' in fee:
fee['address'] = self.fee.address
if 'amount' in fee:
fee['amount'] = str(self.fee.amount)
stream_type = self.message.WhichOneof('type')
if stream_type:
claim['stream_type'] = stream_type
return claim
def update(self, **kwargs):
for l in ('tags', 'languages', 'locations'):
if kwargs.pop(f'clear_{l}', False):
self.message.ClearField('tags')
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
getattr(self, l).append(items)
elif isinstance(items, list):
getattr(self, l).extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
if kwargs.pop('clear_fee', False):
self.message.ClearField('fee')
else:
self.fee.update(
kwargs.pop('fee_address', None),
kwargs.pop('fee_currency', None),
kwargs.pop('fee_amount', None)
)
for key, value in kwargs.items():
setattr(self, key, value)
if 'sd_hash' in kwargs:
self.source.sd_hash = kwargs.pop('sd_hash')
if 'file_name' in kwargs:
self.source.name = kwargs.pop('file_name')
if 'file_hash' in kwargs:
self.source.file_hash = kwargs.pop('file_hash')
stream_type = None
if file_path is not None:
stream_type = self.source.update(file_path=file_path)
elif self.source.name:
self.source.media_type, stream_type = guess_media_type(self.source.name)
elif self.source.media_type:
stream_type = guess_stream_type(self.source.media_type)
class Channel(BaseClaimSubType):
if 'file_size' in kwargs:
self.source.size = kwargs.pop('file_size')
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.channel_message
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
@public_key.setter
def public_key(self, sd_public_key: str):
self.message.public_key = unhexlify(sd_public_key.encode())
@property
def public_key_bytes(self) -> bytes:
return self.message.public_key
@public_key_bytes.setter
def public_key_bytes(self, public_key: bytes):
self.message.public_key = public_key
@property
def contact_email(self) -> str:
return self.message.contact_email
@contact_email.setter
def contact_email(self, contact_email: str):
self.message.contact_email = contact_email
@property
def homepage_url(self) -> str:
return self.message.homepage_url
@homepage_url.setter
def homepage_url(self, homepage_url: str):
self.message.homepage_url = homepage_url
@property
def cover_url(self) -> str:
return self.message.cover_url
@cover_url.setter
def cover_url(self, cover_url: str):
self.message.cover_url = cover_url
class Stream(BaseClaimSubType):
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.stream_message
def update(
self, file_path=None, stream_type=None,
fee_currency=None, fee_amount=None, fee_address=None,
**kwargs):
duration_was_not_set = True
sub_types = ('image', 'video', 'audio')
for key in list(kwargs.keys()):
for sub_type in sub_types:
if key.startswith(f'{sub_type}_'):
stream_type = sub_type
sub_obj = getattr(self, sub_type)
sub_obj_attr = key[len(f'{sub_type}_'):]
setattr(sub_obj, sub_obj_attr, kwargs.pop(key))
if sub_obj_attr == 'duration':
duration_was_not_set = False
break
if stream_type is not None:
if stream_type not in sub_types:
raise Exception(
f"stream_type of '{stream_type}' is not valid, must be one of: {sub_types}"
)
sub_obj = getattr(self, stream_type)
if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
sub_obj.set_duration_from_path(file_path)
if stream_type in ('image', 'video', 'audio'):
media = getattr(self, stream_type)
media_args = {'file_metadata': None}
if file_path is not None:
try:
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
except:
log.exception('Could not read file metadata.')
if isinstance(media, Playable):
media_args['duration'] = duration
if isinstance(media, Dimmensional):
media_args['height'] = height
media_args['width'] = width
media.update(**media_args)
super().update(**kwargs)
if file_path is not None:
self.media_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.file.size = os.path.getsize(file_path)
if self.file.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
if fee_amount and fee_currency:
if fee_address:
self.fee.address = fee_address
if fee_currency.lower() == 'lbc':
self.fee.lbc = Decimal(fee_amount)
elif fee_currency.lower() == 'btc':
self.fee.btc = Decimal(fee_amount)
elif fee_currency.lower() == 'usd':
self.fee.usd = Decimal(fee_amount)
else:
raise Exception(f'Unknown currency type: {fee_currency}')
@property
def sd_hash(self) -> str:
return hexlify(self.message.sd_hash).decode()
@sd_hash.setter
def sd_hash(self, sd_hash: str):
self.message.sd_hash = unhexlify(sd_hash.encode())
@property
def sd_hash_bytes(self) -> bytes:
return self.message.sd_hash
@sd_hash_bytes.setter
def sd_hash_bytes(self, sd_hash: bytes):
self.message.sd_hash = sd_hash
@property
def author(self) -> str:
return self.message.author
@@ -717,14 +282,6 @@ class Stream(BaseClaimSubType):
def release_time(self, release_time: int):
self.message.release_time = release_time
@property
def media_type(self) -> str:
return self.message.media_type
@media_type.setter
def media_type(self, media_type: str):
self.message.media_type = media_type
@property
def fee(self) -> Fee:
return Fee(self.message.fee)
@@ -734,8 +291,12 @@ class Stream(BaseClaimSubType):
return self.message.HasField('fee')
@property
def file(self) -> File:
return File(self.message.file)
def source(self) -> Source:
return Source(self.message.source)
@property
def stream_type(self) -> str:
return self.message.WhichOneof('type')
@property
def image(self) -> Image:
@@ -748,3 +309,90 @@ class Stream(BaseClaimSubType):
@property
def audio(self) -> Audio:
return Audio(self.message.audio)
class Channel(BaseClaim):
__slots__ = ()
claim_type = Claim.CHANNEL
object_fields = BaseClaim.object_fields + ('cover',)
repeat_fields = BaseClaim.repeat_fields + ('featured',)
def to_dict(self):
claim = super().to_dict()
claim['public_key'] = self.public_key
if 'featured' in claim:
claim['featured'] = self.featured.ids
return claim
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
@public_key.setter
def public_key(self, sd_public_key: str):
self.message.public_key = unhexlify(sd_public_key.encode())
@property
def public_key_bytes(self) -> bytes:
return self.message.public_key
@public_key_bytes.setter
def public_key_bytes(self, public_key: bytes):
self.message.public_key = public_key
@property
def email(self) -> str:
return self.message.email
@email.setter
def email(self, email: str):
self.message.email = email
@property
def website_url(self) -> str:
return self.message.website_url
@website_url.setter
def website_url(self, website_url: str):
self.message.website_url = website_url
@property
def cover(self) -> Source:
return Source(self.message.cover)
@property
def featured(self) -> ClaimList:
return ClaimList(self.message.featured)
class Repost(BaseClaim):
__slots__ = ()
claim_type = Claim.REPOST
@property
def reference(self) -> ClaimReference:
return ClaimReference(self.message)
class Collection(BaseClaim):
__slots__ = ()
claim_type = Claim.COLLECTION
repeat_fields = BaseClaim.repeat_fields + ('claims',)
def to_dict(self):
claim = super().to_dict()
if 'claim_references' in claim:
claim['claim_references'] = self.claims.ids
return claim
@property
def claims(self) -> ClaimList:
return ClaimList(self.message)
+21 -16
View File
@@ -1,18 +1,26 @@
import json
from decimal import Decimal
from google.protobuf.message import DecodeError
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
def from_old_json_schema(claim, payload: bytes):
value = json.loads(payload)
try:
value = json.loads(payload)
except:
raise DecodeError('Could not parse JSON.')
stream = claim.stream
stream.media_type = value.get('content_type', value.get('content-type', 'application/octet-stream'))
stream.source.sd_hash = value['sources']['lbry_sd_hash']
stream.source.media_type = value.get(
'content_type', value.get('content-type', 'application/octet-stream'))
stream.title = value.get('title', '')
stream.description = value.get('description', '')
stream.thumbnail_url = value.get('thumbnail', '')
if value.get('thumbnail', ''):
stream.thumbnail.url = value.get('thumbnail', '')
stream.author = value.get('author', '')
stream.license = value.get('license', '')
stream.license_url = value.get('license_url', '')
@@ -24,7 +32,6 @@ def from_old_json_schema(claim, payload: bytes):
stream.languages.append(language)
except:
pass
stream.sd_hash = value['sources']['lbry_sd_hash']
if value.get('nsfw', False):
stream.tags.append('mature')
if "fee" in value:
@@ -37,7 +44,7 @@ def from_old_json_schema(claim, payload: bytes):
elif currency == 'BTC':
stream.fee.btc = Decimal(fee[currency]['amount'])
else:
raise ValueError(f'Unknown currency: {currency}')
raise DecodeError(f'Unknown currency: {currency}')
stream.fee.address = fee[currency]['address']
return claim
@@ -45,18 +52,21 @@ def from_old_json_schema(claim, payload: bytes):
def from_types_v1(claim, payload: bytes):
old = OldClaimMessage()
old.ParseFromString(payload)
if old.claimType == 1:
if old.claimType == 2:
channel = claim.channel
channel.public_key_bytes = old.certificate.publicKey
else:
stream = claim.stream
stream.title = old.stream.metadata.title
stream.description = old.stream.metadata.description
stream.author = old.stream.metadata.author
stream.license = old.stream.metadata.license
stream.license_url = old.stream.metadata.licenseUrl
stream.thumbnail_url = old.stream.metadata.thumbnail
stream.thumbnail.url = old.stream.metadata.thumbnail
if old.stream.metadata.HasField('language'):
stream.languages.add().message.language = old.stream.metadata.language
stream.media_type = old.stream.source.contentType
stream.sd_hash_bytes = old.stream.source.source
stream.source.media_type = old.stream.source.contentType
stream.source.sd_hash_bytes = old.stream.source.source
if old.stream.metadata.nsfw:
stream.tags.append('mature')
if old.stream.metadata.HasField('fee'):
@@ -70,17 +80,12 @@ def from_types_v1(claim, payload: bytes):
elif currency == 'BTC':
stream.fee.btc = Decimal(fee.amount)
else:
raise ValueError(f'Unsupported currency: {currency}')
raise DecodeError(f'Unsupported currency: {currency}')
if old.HasField('publisherSignature'):
sig = old.publisherSignature
claim.signature = sig.signature
claim.signature_type = KeyType.Name(sig.signatureType)
claim.signing_channel_hash = sig.certificateId
claim.signing_channel_hash = sig.certificateId[::-1]
old.ClearField("publisherSignature")
claim.unsigned_payload = old.SerializeToString()
elif old.claimType == 2:
channel = claim.channel
channel.public_key_bytes = old.certificate.publicKey
else:
raise ValueError('claimType must be 1 for Streams and 2 for Channel')
return claim
+160 -146
View File
@@ -1,160 +1,174 @@
import os
types_map = {
# http://www.iana.org/assignments/media-types
'.a': 'application/octet-stream',
'.ai': 'application/postscript',
'.aif': 'audio/x-aiff',
'.aifc': 'audio/x-aiff',
'.aiff': 'audio/x-aiff',
'.au': 'audio/basic',
'.avi': 'video/x-msvideo',
'.bat': 'text/plain',
'.bcpio': 'application/x-bcpio',
'.bin': 'application/octet-stream',
'.bmp': 'image/bmp',
'.c': 'text/plain',
'.cdf': 'application/x-netcdf',
'.cpio': 'application/x-cpio',
'.csh': 'application/x-csh',
'.css': 'text/css',
'.csv': 'text/csv',
'.dll': 'application/octet-stream',
'.doc': 'application/msword',
'.dot': 'application/msword',
'.dvi': 'application/x-dvi',
'.eml': 'message/rfc822',
'.eps': 'application/postscript',
'.epub': 'application/epub+zip',
'.etx': 'text/x-setext',
'.exe': 'application/octet-stream',
'.gif': 'image/gif',
'.gtar': 'application/x-gtar',
'.h': 'text/plain',
'.hdf': 'application/x-hdf',
'.htm': 'text/html',
'.html': 'text/html',
'.ico': 'image/vnd.microsoft.icon',
'.ief': 'image/ief',
'.iges': 'model/iges',
'.jpe': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.jpg': 'image/jpeg',
'.js': 'application/javascript',
'.json': 'application/json',
'.ksh': 'text/plain',
'.latex': 'application/x-latex',
'.m1v': 'video/mpeg',
'.m3u': 'application/vnd.apple.mpegurl',
'.m3u8': 'application/vnd.apple.mpegurl',
'.man': 'application/x-troff-man',
'.markdown': 'text/markdown',
'.md': 'text/markdown',
'.me': 'application/x-troff-me',
'.mht': 'message/rfc822',
'.mhtml': 'message/rfc822',
'.mif': 'application/x-mif',
'.mov': 'video/quicktime',
'.movie': 'video/x-sgi-movie',
'.mp2': 'audio/mpeg',
'.mp3': 'audio/mpeg',
'.mp4': 'video/mp4',
'.mpa': 'video/mpeg',
'.mpe': 'video/mpeg',
'.mpeg': 'video/mpeg',
'.mpg': 'video/mpeg',
'.ms': 'application/x-troff-ms',
'.nc': 'application/x-netcdf',
'.nws': 'message/rfc822',
'.o': 'application/octet-stream',
'.obj': 'application/octet-stream',
'.oda': 'application/oda',
'.p12': 'application/x-pkcs12',
'.p7c': 'application/pkcs7-mime',
'.pbm': 'image/x-portable-bitmap',
'.pdf': 'application/pdf',
'.pfx': 'application/x-pkcs12',
'.pgm': 'image/x-portable-graymap',
'.pl': 'text/plain',
'.png': 'image/png',
'.pnm': 'image/x-portable-anymap',
'.pot': 'application/vnd.ms-powerpoint',
'.ppa': 'application/vnd.ms-powerpoint',
'.ppm': 'image/x-portable-pixmap',
'.pps': 'application/vnd.ms-powerpoint',
'.ppt': 'application/vnd.ms-powerpoint',
'.ps': 'application/postscript',
'.pwz': 'application/vnd.ms-powerpoint',
'.py': 'text/x-python',
'.pyc': 'application/x-python-code',
'.pyo': 'application/x-python-code',
'.qt': 'video/quicktime',
'.ra': 'audio/x-pn-realaudio',
'.ram': 'application/x-pn-realaudio',
'.ras': 'image/x-cmu-raster',
'.rdf': 'application/xml',
'.rgb': 'image/x-rgb',
'.roff': 'application/x-troff',
'.rtx': 'text/richtext',
'.sgm': 'text/x-sgml',
'.sgml': 'text/x-sgml',
'.sh': 'application/x-sh',
'.shar': 'application/x-shar',
'.snd': 'audio/basic',
'.so': 'application/octet-stream',
'.src': 'application/x-wais-source',
'.stl': 'model/stl',
'.sv4cpio': 'application/x-sv4cpio',
'.sv4crc': 'application/x-sv4crc',
'.svg': 'image/svg+xml',
'.swf': 'application/x-shockwave-flash',
'.t': 'application/x-troff',
'.tar': 'application/x-tar',
'.tcl': 'application/x-tcl',
'.tex': 'application/x-tex',
'.texi': 'application/x-texinfo',
'.texinfo': 'application/x-texinfo',
'.tif': 'image/tiff',
'.tiff': 'image/tiff',
'.tr': 'application/x-troff',
'.tsv': 'text/tab-separated-values',
'.txt': 'text/plain',
'.ustar': 'application/x-ustar',
'.vcf': 'text/x-vcard',
'.wav': 'audio/x-wav',
'.webm': 'video/webm',
'.wiz': 'application/msword',
'.wsdl': 'application/xml',
'.xbm': 'image/x-xbitmap',
'.xlb': 'application/vnd.ms-excel',
'.xls': 'application/vnd.ms-excel',
'.xml': 'text/xml',
'.xpdl': 'application/xml',
'.xpm': 'image/x-xpixmap',
'.xsl': 'application/xml',
'.xwd': 'image/x-xwindowdump',
'.zip': 'application/zip',
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
'.a': ('application/octet-stream', 'binary'),
'.ai': ('application/postscript', 'image'),
'.aif': ('audio/x-aiff', 'audio'),
'.aifc': ('audio/x-aiff', 'audio'),
'.aiff': ('audio/x-aiff', 'audio'),
'.au': ('audio/basic', 'audio'),
'.avi': ('video/x-msvideo', 'video'),
'.bat': ('text/plain', 'document'),
'.bcpio': ('application/x-bcpio', 'binary'),
'.bin': ('application/octet-stream', 'binary'),
'.bmp': ('image/bmp', 'image'),
'.c': ('text/plain', 'document'),
'.cdf': ('application/x-netcdf', 'binary'),
'.cpio': ('application/x-cpio', 'binary'),
'.csh': ('application/x-csh', 'binary'),
'.css': ('text/css', 'document'),
'.csv': ('text/csv', 'document'),
'.dll': ('application/octet-stream', 'binary'),
'.doc': ('application/msword', 'document'),
'.dot': ('application/msword', 'document'),
'.dvi': ('application/x-dvi', 'binary'),
'.eml': ('message/rfc822', 'document'),
'.eps': ('application/postscript', 'document'),
'.epub': ('application/epub+zip', 'document'),
'.etx': ('text/x-setext', 'document'),
'.exe': ('application/octet-stream', 'binary'),
'.gif': ('image/gif', 'image'),
'.gtar': ('application/x-gtar', 'binary'),
'.h': ('text/plain', 'document'),
'.hdf': ('application/x-hdf', 'binary'),
'.htm': ('text/html', 'document'),
'.html': ('text/html', 'document'),
'.ico': ('image/vnd.microsoft.icon', 'image'),
'.ief': ('image/ief', 'image'),
'.iges': ('model/iges', 'model'),
'.jpe': ('image/jpeg', 'image'),
'.jpeg': ('image/jpeg', 'image'),
'.jpg': ('image/jpeg', 'image'),
'.js': ('application/javascript', 'document'),
'.json': ('application/json', 'document'),
'.ksh': ('text/plain', 'document'),
'.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
'.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'),
'.me': ('application/x-troff-me', 'binary'),
'.mht': ('message/rfc822', 'document'),
'.mhtml': ('message/rfc822', 'document'),
'.mif': ('application/x-mif', 'binary'),
'.mov': ('video/quicktime', 'video'),
'.movie': ('video/x-sgi-movie', 'video'),
'.mp2': ('audio/mpeg', 'audio'),
'.mp3': ('audio/mpeg', 'audio'),
'.mp4': ('video/mp4', 'video'),
'.mpa': ('video/mpeg', 'video'),
'.mpe': ('video/mpeg', 'video'),
'.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'),
'.ms': ('application/x-troff-ms', 'binary'),
'.nc': ('application/x-netcdf', 'binary'),
'.nws': ('message/rfc822', 'document'),
'.o': ('application/octet-stream', 'binary'),
'.obj': ('application/octet-stream', 'model'),
'.oda': ('application/oda', 'binary'),
'.p12': ('application/x-pkcs12', 'binary'),
'.p7c': ('application/pkcs7-mime', 'binary'),
'.pbm': ('image/x-portable-bitmap', 'image'),
'.pdf': ('application/pdf', 'document'),
'.pfx': ('application/x-pkcs12', 'binary'),
'.pgm': ('image/x-portable-graymap', 'image'),
'.pl': ('text/plain', 'document'),
'.png': ('image/png', 'image'),
'.pnm': ('image/x-portable-anymap', 'image'),
'.pot': ('application/vnd.ms-powerpoint', 'document'),
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
'.ppm': ('image/x-portable-pixmap', 'image'),
'.pps': ('application/vnd.ms-powerpoint', 'document'),
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
'.ps': ('application/postscript', 'document'),
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
'.py': ('text/x-python', 'document'),
'.pyc': ('application/x-python-code', 'binary'),
'.pyo': ('application/x-python-code', 'binary'),
'.qt': ('video/quicktime', 'video'),
'.ra': ('audio/x-pn-realaudio', 'audio'),
'.ram': ('application/x-pn-realaudio', 'audio'),
'.ras': ('image/x-cmu-raster', 'image'),
'.rdf': ('application/xml', 'binary'),
'.rgb': ('image/x-rgb', 'image'),
'.roff': ('application/x-troff', 'binary'),
'.rtx': ('text/richtext', 'document'),
'.sgm': ('text/x-sgml', 'document'),
'.sgml': ('text/x-sgml', 'document'),
'.sh': ('application/x-sh', 'document'),
'.shar': ('application/x-shar', 'binary'),
'.snd': ('audio/basic', 'audio'),
'.so': ('application/octet-stream', 'binary'),
'.src': ('application/x-wais-source', 'binary'),
'.stl': ('model/stl', 'model'),
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
'.sv4crc': ('application/x-sv4crc', 'binary'),
'.svg': ('image/svg+xml', 'image'),
'.swf': ('application/x-shockwave-flash', 'binary'),
'.t': ('application/x-troff', 'binary'),
'.tar': ('application/x-tar', 'binary'),
'.tcl': ('application/x-tcl', 'binary'),
'.tex': ('application/x-tex', 'binary'),
'.texi': ('application/x-texinfo', 'binary'),
'.texinfo': ('application/x-texinfo', 'binary'),
'.tif': ('image/tiff', 'image'),
'.tiff': ('image/tiff', 'image'),
'.tr': ('application/x-troff', 'binary'),
'.tsv': ('text/tab-separated-values', 'document'),
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
'.wsdl': ('application/xml', 'document'),
'.xbm': ('image/x-xbitmap', 'image'),
'.xlb': ('application/vnd.ms-excel', 'document'),
'.xls': ('application/vnd.ms-excel', 'document'),
'.xml': ('text/xml', 'document'),
'.xpdl': ('application/xml', 'document'),
'.xpm': ('image/x-xpixmap', 'image'),
'.xsl': ('application/xml', 'document'),
'.xwd': ('image/x-xwindowdump', 'image'),
'.zip': ('application/zip', 'binary'),
# These are non-standard types, commonly found in the wild.
'.mid': 'audio/midi',
'.midi': 'audio/midi',
'.pct': 'image/pict',
'.pic': 'image/pict',
'.pict': 'image/pict',
'.rtf': 'application/rtf',
'.xul': 'text/xul',
'.m4v': 'video/m4v',
'.cbr': ('application/vnd.comicbook-rar', 'document'),
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'),
'.prc': ('application/x-mobipocket-ebook', 'document'),
'.rtf': ('application/rtf', 'document'),
'.xul': ('text/xul', 'document'),
'.m4v': ('video/m4v', 'video'),
# microsoft is special and has its own 'standard'
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
'.wmv': 'video/x-ms-wmv'
'.wmv': ('video/x-ms-wmv', 'video')
}
def guess_media_type(path):
_, ext = os.path.splitext(path)
return types_map.get(
ext.strip().lower(),
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
)
extension = ext.strip().lower()
if extension[1:]:
if extension in types_map:
return types_map[extension]
return f'application/x-ext-{extension[1:]}', 'binary'
return 'application/octet-stream', 'binary'
def guess_stream_type(media_type):
for media, stream in types_map.values():
if media == media_type:
return stream
return 'binary'
+460 -252
View File
@@ -19,11 +19,33 @@ DESCRIPTOR = _descriptor.FileDescriptor(
package='pb',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"M\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x42\x06\n\x04type\"\xe8\x01\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x15\n\rcontact_email\x18\x08 \x01(\t\x12\x14\n\x0chomepage_url\x18\t \x01(\t\x12\x11\n\tcover_url\x18\n \x01(\t\"\x8e\x03\n\x06Stream\x12\x0f\n\x07sd_hash\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x0e\n\x06\x61uthor\x18\x08 \x01(\t\x12\x0f\n\x07license\x18\t \x01(\t\x12\x13\n\x0blicense_url\x18\n \x01(\t\x12\x14\n\x0crelease_time\x18\x0b \x01(\x03\x12\x12\n\nmedia_type\x18\r \x01(\t\x12\x16\n\x04\x66ile\x18\x0e \x01(\x0b\x32\x08.pb.File\x12\x14\n\x03\x66\x65\x65\x18\x0f \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\x10 \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x11 \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x12 \x01(\x0b\x32\t.pb.AudioH\x00\x42\x06\n\x04type\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"0\n\x04\x46ile\x12\x0c\n\x04hash\x18\x03 \x01(\x0c\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"8\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\nLine truncated
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"\xab\x02\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x12#\n\ncollection\x18\x03 \x01(\x0b\x32\r.pb.ClaimListH\x00\x12$\n\x06repost\x18\x04 \x01(\x0b\x32\x12.pb.ClaimReferenceH\x00\x12\r\n\x05title\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x1d\n\tthumbnail\x18\n \x01(\x0b\x32\n.pb.Source\x12\x0c\n\x04tags\x18\x0b \x03(\t\x12\x1f\n\tlanguages\x18\x0c \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\r \x03(\x0b\x32\x0c.pb.LocationB\x06\n\x04type\"\x84\x02\n\x06Stream\x12\x1a\n\x06source\x18\x01 \x01(\x0b\x32\n.pb.Source\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\x0f\n\x07license\x18\x03 \x01(\t\x12\x13\n\x0blicense_url\x18\x04 \x01(\t\x12\x14\n\x0crelease_time\x18\x05 \x01(\x03\x12\x14\n\x03\x66\x65\x65\x18\x06 \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\n \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x0b \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x0c \x01(\x0b\x32\t.pb.AudioH\x00\x12 \n\x08software\x18\r \x01(\x0b\x32\x0c.pb.SoftwareH\x00\x42\x06\n\x04type\"}\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x13\n\x0bwebsite_url\x18\x03 \x01(\t\x12\x19\n\x05\x63over\x18\x04 \x01(\x0b\x32\n.pb.Source\x12\x1f\n\x08\x66\x65\x61tured\x18\x05 \x01(\x0b\x32\r.pb.ClaimList\"$\n\x0e\x43laimReference\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\"\x90\x01\n\tClaimList\x12)\n\tlist_type\x18\x01 \x01(\x0e\x32\x16.pb.ClaimList.ListType\x12,\n\x10\x63laim_references\x18\x02 \x03(\x0b\x32\x12.pb.ClaimReference\"*\n\x08ListType\x12\x0e\n\nCOLLECTION\x10\x00\x12\x0e\n\nDERIVATION\x10\x02\"d\n\x06Source\x12\x0c\n\x04hash\x18\x01 \x01(\x0c\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x12\n\nmedia_type\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07sd_hash\x18\x06 \x01(\x0c\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"R\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\x12\x18\n\x05\x61udio\x18\x0f \x01(\x0b\x32\t.pb.Audio\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"l\n\x08Software\x12\n\n\x02os\x18\x01 \x01(\t\"T\n\x02OS\x12\x0e\n\nUNKNOWN_OS\x10\x00\x12\x07\n\x03\x41NY\x10\x01\x12\t\n\x05LINUX\x10\x02\x12\x0b\n\x07WINDOWS\x10\x03\x12\x07\n\x03MAC\x10\x04\x12\x0b\n\x07\x41NDROID\x10\x05\x12\x07\n\x03IOS\x10\x06\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x0Line truncated
)
_CLAIMLIST_LISTTYPE = _descriptor.EnumDescriptor(
name='ListType',
full_name='pb.ClaimList.ListType',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='COLLECTION', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='DERIVATION', index=1, number=2,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=852,
serialized_end=894,
)
_sym_db.RegisterEnumDescriptor(_CLAIMLIST_LISTTYPE)
_FEE_CURRENCY = _descriptor.EnumDescriptor(
name='Currency',
full_name='pb.Fee.Currency',
@@ -49,11 +71,53 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=811,
serialized_end=870,
serialized_start=1075,
serialized_end=1134,
)
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
_SOFTWARE_OS = _descriptor.EnumDescriptor(
name='OS',
full_name='pb.Software.OS',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='UNKNOWN_OS', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='ANY', index=1, number=1,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='LINUX', index=2, number=2,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='WINDOWS', index=3, number=3,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='MAC', index=4, number=4,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='ANDROID', index=5, number=5,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='IOS', index=6, number=6,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=1311,
serialized_end=1395,
)
_sym_db.RegisterEnumDescriptor(_SOFTWARE_OS)
_LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
name='Language',
full_name='pb.Language.Language',
@@ -803,8 +867,8 @@ _LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=1177,
serialized_end=2738,
serialized_start=1527,
serialized_end=3088,
)
_sym_db.RegisterEnumDescriptor(_LANGUAGE_LANGUAGE)
@@ -1617,8 +1681,8 @@ _LANGUAGE_SCRIPT = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=2741,
serialized_end=4831,
serialized_start=3091,
serialized_end=5181,
)
_sym_db.RegisterEnumDescriptor(_LANGUAGE_SCRIPT)
@@ -2631,8 +2695,8 @@ _LOCATION_COUNTRY = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=4966,
serialized_end=7110,
serialized_start=5316,
serialized_end=7460,
)
_sym_db.RegisterEnumDescriptor(_LOCATION_COUNTRY)
@@ -2658,6 +2722,62 @@ _CLAIM = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='collection', full_name='pb.Claim.collection', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='repost', full_name='pb.Claim.repost', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Claim.title', index=4,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Claim.description', index=5,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail', full_name='pb.Claim.thumbnail', index=6,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Claim.tags', index=7,
number=11, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Claim.languages', index=8,
number=12, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Claim.locations', index=9,
number=13, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -2673,102 +2793,8 @@ _CLAIM = _descriptor.Descriptor(
name='type', full_name='pb.Claim.type',
index=0, containing_type=None, fields=[]),
],
serialized_start=19,
serialized_end=96,
)
_CHANNEL = _descriptor.Descriptor(
name='Channel',
full_name='pb.Channel',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='public_key', full_name='pb.Channel.public_key', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Channel.title', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Channel.description', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail_url', full_name='pb.Channel.thumbnail_url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Channel.tags', index=4,
number=5, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Channel.languages', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Channel.locations', index=6,
number=7, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='contact_email', full_name='pb.Channel.contact_email', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='homepage_url', full_name='pb.Channel.homepage_url', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cover_url', full_name='pb.Channel.cover_url', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=99,
serialized_end=331,
serialized_start=20,
serialized_end=319,
)
@@ -2780,120 +2806,71 @@ _STREAM = _descriptor.Descriptor(
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='sd_hash', full_name='pb.Stream.sd_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
name='source', full_name='pb.Stream.source', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='pb.Stream.title', index=1,
name='author', full_name='pb.Stream.author', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='pb.Stream.description', index=2,
name='license', full_name='pb.Stream.license', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail_url', full_name='pb.Stream.thumbnail_url', index=3,
name='license_url', full_name='pb.Stream.license_url', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tags', full_name='pb.Stream.tags', index=4,
number=5, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='languages', full_name='pb.Stream.languages', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='locations', full_name='pb.Stream.locations', index=6,
number=7, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='author', full_name='pb.Stream.author', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license', full_name='pb.Stream.license', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license_url', full_name='pb.Stream.license_url', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='release_time', full_name='pb.Stream.release_time', index=10,
number=11, type=3, cpp_type=2, label=1,
name='release_time', full_name='pb.Stream.release_time', index=4,
number=5, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='media_type', full_name='pb.Stream.media_type', index=11,
number=13, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='file', full_name='pb.Stream.file', index=12,
number=14, type=11, cpp_type=10, label=1,
name='fee', full_name='pb.Stream.fee', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='fee', full_name='pb.Stream.fee', index=13,
number=15, type=11, cpp_type=10, label=1,
name='image', full_name='pb.Stream.image', index=6,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='image', full_name='pb.Stream.image', index=14,
number=16, type=11, cpp_type=10, label=1,
name='video', full_name='pb.Stream.video', index=7,
number=11, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='video', full_name='pb.Stream.video', index=15,
number=17, type=11, cpp_type=10, label=1,
name='audio', full_name='pb.Stream.audio', index=8,
number=12, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='audio', full_name='pb.Stream.audio', index=16,
number=18, type=11, cpp_type=10, label=1,
name='software', full_name='pb.Stream.software', index=9,
number=13, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@@ -2913,8 +2890,203 @@ _STREAM = _descriptor.Descriptor(
name='type', full_name='pb.Stream.type',
index=0, containing_type=None, fields=[]),
],
serialized_start=334,
serialized_end=732,
serialized_start=322,
serialized_end=582,
)
_CHANNEL = _descriptor.Descriptor(
name='Channel',
full_name='pb.Channel',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='public_key', full_name='pb.Channel.public_key', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='email', full_name='pb.Channel.email', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='website_url', full_name='pb.Channel.website_url', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cover', full_name='pb.Channel.cover', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='featured', full_name='pb.Channel.featured', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=584,
serialized_end=709,
)
_CLAIMREFERENCE = _descriptor.Descriptor(
name='ClaimReference',
full_name='pb.ClaimReference',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='claim_hash', full_name='pb.ClaimReference.claim_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=711,
serialized_end=747,
)
_CLAIMLIST = _descriptor.Descriptor(
name='ClaimList',
full_name='pb.ClaimList',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='list_type', full_name='pb.ClaimList.list_type', index=0,
number=1, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claim_references', full_name='pb.ClaimList.claim_references', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_CLAIMLIST_LISTTYPE,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=750,
serialized_end=894,
)
_SOURCE = _descriptor.Descriptor(
name='Source',
full_name='pb.Source',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hash', full_name='pb.Source.hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='pb.Source.name', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='size', full_name='pb.Source.size', index=2,
number=3, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='media_type', full_name='pb.Source.media_type', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='url', full_name='pb.Source.url', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sd_hash', full_name='pb.Source.sd_hash', index=5,
number=6, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=896,
serialized_end=996,
)
@@ -2959,53 +3131,8 @@ _FEE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=735,
serialized_end=870,
)
_FILE = _descriptor.Descriptor(
name='File',
full_name='pb.File',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hash', full_name='pb.File.hash', index=0,
number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='pb.File.name', index=1,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='size', full_name='pb.File.size', index=2,
number=2, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=872,
serialized_end=920,
serialized_start=999,
serialized_end=1134,
)
@@ -3042,8 +3169,8 @@ _IMAGE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=922,
serialized_end=960,
serialized_start=1136,
serialized_end=1174,
)
@@ -3075,6 +3202,13 @@ _VIDEO = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='audio', full_name='pb.Video.audio', index=3,
number=15, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -3087,8 +3221,8 @@ _VIDEO = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=962,
serialized_end=1018,
serialized_start=1176,
serialized_end=1258,
)
@@ -3118,8 +3252,40 @@ _AUDIO = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1020,
serialized_end=1045,
serialized_start=1260,
serialized_end=1285,
)
_SOFTWARE = _descriptor.Descriptor(
name='Software',
full_name='pb.Software',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='os', full_name='pb.Software.os', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_SOFTWARE_OS,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=1287,
serialized_end=1395,
)
@@ -3165,8 +3331,8 @@ _LANGUAGE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=1048,
serialized_end=4831,
serialized_start=1398,
serialized_end=5181,
)
@@ -3232,27 +3398,35 @@ _LOCATION = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=4834,
serialized_end=7110,
serialized_start=5184,
serialized_end=7460,
)
_CLAIM.fields_by_name['stream'].message_type = _STREAM
_CLAIM.fields_by_name['channel'].message_type = _CHANNEL
_CLAIM.fields_by_name['collection'].message_type = _CLAIMLIST
_CLAIM.fields_by_name['repost'].message_type = _CLAIMREFERENCE
_CLAIM.fields_by_name['thumbnail'].message_type = _SOURCE
_CLAIM.fields_by_name['languages'].message_type = _LANGUAGE
_CLAIM.fields_by_name['locations'].message_type = _LOCATION
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['stream'])
_CLAIM.fields_by_name['stream'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['channel'])
_CLAIM.fields_by_name['channel'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CHANNEL.fields_by_name['languages'].message_type = _LANGUAGE
_CHANNEL.fields_by_name['locations'].message_type = _LOCATION
_STREAM.fields_by_name['languages'].message_type = _LANGUAGE
_STREAM.fields_by_name['locations'].message_type = _LOCATION
_STREAM.fields_by_name['file'].message_type = _FILE
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['collection'])
_CLAIM.fields_by_name['collection'].containing_oneof = _CLAIM.oneofs_by_name['type']
_CLAIM.oneofs_by_name['type'].fields.append(
_CLAIM.fields_by_name['repost'])
_CLAIM.fields_by_name['repost'].containing_oneof = _CLAIM.oneofs_by_name['type']
_STREAM.fields_by_name['source'].message_type = _SOURCE
_STREAM.fields_by_name['fee'].message_type = _FEE
_STREAM.fields_by_name['image'].message_type = _IMAGE
_STREAM.fields_by_name['video'].message_type = _VIDEO
_STREAM.fields_by_name['audio'].message_type = _AUDIO
_STREAM.fields_by_name['software'].message_type = _SOFTWARE
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['image'])
_STREAM.fields_by_name['image'].containing_oneof = _STREAM.oneofs_by_name['type']
@@ -3262,8 +3436,18 @@ _STREAM.fields_by_name['video'].containing_oneof = _STREAM.oneofs_by_name['type'
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['audio'])
_STREAM.fields_by_name['audio'].containing_oneof = _STREAM.oneofs_by_name['type']
_STREAM.oneofs_by_name['type'].fields.append(
_STREAM.fields_by_name['software'])
_STREAM.fields_by_name['software'].containing_oneof = _STREAM.oneofs_by_name['type']
_CHANNEL.fields_by_name['cover'].message_type = _SOURCE
_CHANNEL.fields_by_name['featured'].message_type = _CLAIMLIST
_CLAIMLIST.fields_by_name['list_type'].enum_type = _CLAIMLIST_LISTTYPE
_CLAIMLIST.fields_by_name['claim_references'].message_type = _CLAIMREFERENCE
_CLAIMLIST_LISTTYPE.containing_type = _CLAIMLIST
_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
_FEE_CURRENCY.containing_type = _FEE
_VIDEO.fields_by_name['audio'].message_type = _AUDIO
_SOFTWARE_OS.containing_type = _SOFTWARE
_LANGUAGE.fields_by_name['language'].enum_type = _LANGUAGE_LANGUAGE
_LANGUAGE.fields_by_name['script'].enum_type = _LANGUAGE_SCRIPT
_LANGUAGE.fields_by_name['region'].enum_type = _LOCATION_COUNTRY
@@ -3272,13 +3456,16 @@ _LANGUAGE_SCRIPT.containing_type = _LANGUAGE
_LOCATION.fields_by_name['country'].enum_type = _LOCATION_COUNTRY
_LOCATION_COUNTRY.containing_type = _LOCATION
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
DESCRIPTOR.message_types_by_name['ClaimReference'] = _CLAIMREFERENCE
DESCRIPTOR.message_types_by_name['ClaimList'] = _CLAIMLIST
DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
DESCRIPTOR.message_types_by_name['Fee'] = _FEE
DESCRIPTOR.message_types_by_name['File'] = _FILE
DESCRIPTOR.message_types_by_name['Image'] = _IMAGE
DESCRIPTOR.message_types_by_name['Video'] = _VIDEO
DESCRIPTOR.message_types_by_name['Audio'] = _AUDIO
DESCRIPTOR.message_types_by_name['Software'] = _SOFTWARE
DESCRIPTOR.message_types_by_name['Language'] = _LANGUAGE
DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -3290,13 +3477,6 @@ Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), d
))
_sym_db.RegisterMessage(Claim)
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
DESCRIPTOR = _CHANNEL,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Channel)
))
_sym_db.RegisterMessage(Channel)
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
DESCRIPTOR = _STREAM,
__module__ = 'claim_pb2'
@@ -3304,6 +3484,34 @@ Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,),
))
_sym_db.RegisterMessage(Stream)
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
DESCRIPTOR = _CHANNEL,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Channel)
))
_sym_db.RegisterMessage(Channel)
ClaimReference = _reflection.GeneratedProtocolMessageType('ClaimReference', (_message.Message,), dict(
DESCRIPTOR = _CLAIMREFERENCE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimReference)
))
_sym_db.RegisterMessage(ClaimReference)
ClaimList = _reflection.GeneratedProtocolMessageType('ClaimList', (_message.Message,), dict(
DESCRIPTOR = _CLAIMLIST,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimList)
))
_sym_db.RegisterMessage(ClaimList)
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
DESCRIPTOR = _SOURCE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Source)
))
_sym_db.RegisterMessage(Source)
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
DESCRIPTOR = _FEE,
__module__ = 'claim_pb2'
@@ -3311,13 +3519,6 @@ Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
))
_sym_db.RegisterMessage(Fee)
File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dict(
DESCRIPTOR = _FILE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.File)
))
_sym_db.RegisterMessage(File)
Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(
DESCRIPTOR = _IMAGE,
__module__ = 'claim_pb2'
@@ -3339,6 +3540,13 @@ Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), d
))
_sym_db.RegisterMessage(Audio)
Software = _reflection.GeneratedProtocolMessageType('Software', (_message.Message,), dict(
DESCRIPTOR = _SOFTWARE,
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Software)
))
_sym_db.RegisterMessage(Software)
Language = _reflection.GeneratedProtocolMessageType('Language', (_message.Message,), dict(
DESCRIPTOR = _LANGUAGE,
__module__ = 'claim_pb2'
-136
View File
@@ -1,136 +0,0 @@
import os
import binascii
import logging
import typing
import asyncio
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.blob_file import BlobFile
log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class StreamAssembler:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
output_file_name: typing.Optional[str] = None):
self.output_file_name = output_file_name
self.loop = loop
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.sd_blob: 'BlobFile' = None
self.descriptor: StreamDescriptor = None
self.got_descriptor = asyncio.Event(loop=self.loop)
self.wrote_bytes_event = asyncio.Event(loop=self.loop)
self.stream_finished_event = asyncio.Event(loop=self.loop)
self.output_path = ''
self.stream_handle = None
self.written_bytes: int = 0
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
if not blob or not self.stream_handle or self.stream_handle.closed:
return False
def _decrypt_and_write():
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
self.stream_handle.seek(offset)
_decrypted = blob.decrypt(
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
)
self.stream_handle.write(_decrypted)
self.stream_handle.flush()
self.written_bytes += len(_decrypted)
log.debug("decrypted %s", blob.blob_hash[:8])
await self.loop.run_in_executor(None, _decrypt_and_write)
return True
async def setup(self):
pass
async def after_got_descriptor(self):
pass
async def after_finished(self):
pass
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
if not os.path.isdir(output_dir):
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
await self.setup()
self.sd_blob = await self.get_blob(self.sd_hash)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
self.sd_blob)
await self.after_got_descriptor()
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
self.output_path = os.path.join(output_dir, self.output_file_name)
if not self.got_descriptor.is_set():
self.got_descriptor.set()
await self.blob_manager.storage.store_stream(
self.sd_blob, self.descriptor
)
await self.blob_manager.blob_completed(self.sd_blob)
written_blobs = None
save_tasks = []
try:
with open(self.output_path, 'wb') as stream_handle:
self.stream_handle = stream_handle
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
if blob_info.blob_num != i:
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
return
while self.stream_handle and not self.stream_handle.closed:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
if blob and blob.length != blob_info.length:
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
await self.blob_manager.delete_blobs([blob_info.blob_hash])
continue
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
written_blobs = i
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
break
except FileNotFoundError:
log.debug("stream assembler stopped")
return
except (ValueError, IOError, OSError):
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
self.descriptor.sd_hash)
continue
finally:
if written_blobs == len(self.descriptor.blobs) - 2:
log.debug("finished decrypting and assembling stream")
if save_tasks:
await asyncio.wait(save_tasks)
await self.after_finished()
self.stream_finished_event.set()
else:
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
len(self.descriptor.blobs) - 2)
if self.output_path and os.path.isfile(self.output_path):
log.debug("erasing incomplete file assembly: %s", self.output_path)
os.unlink(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return self.blob_manager.get_blob(blob_hash, length)
+34 -19
View File
@@ -8,7 +8,7 @@ from collections import OrderedDict
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import AbstractBlob, BlobFile
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.error import InvalidStreamDescriptorError
@@ -47,6 +47,17 @@ def file_reader(file_path: str):
class StreamDescriptor:
__slots__ = [
'loop',
'blob_dir',
'stream_name',
'key',
'suggested_file_name',
'blobs',
'stream_hash',
'sd_hash'
]
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, stream_name: str, key: str,
suggested_file_name: str, blobs: typing.List[BlobInfo], stream_hash: typing.Optional[str] = None,
sd_hash: typing.Optional[str] = None):
@@ -108,29 +119,30 @@ class StreamDescriptor:
h.update(self.old_sort_json())
return h.hexdigest()
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
old_sort: typing.Optional[bool] = False):
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
old_sort: typing.Optional[bool] = False,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
if not old_sort:
sd_data = self.as_json()
else:
sd_data = self.old_sort_json()
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
if blob_file_obj:
blob_file_obj.set_length(len(sd_data))
if not sd_blob.get_is_verified():
writer = sd_blob.open_for_writing()
writer = sd_blob.get_blob_writer()
writer.write(sd_data)
await sd_blob.verified.wait()
sd_blob.close()
return sd_blob
@classmethod
def _from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
blob: BlobFile) -> 'StreamDescriptor':
assert os.path.isfile(blob.file_path)
with open(blob.file_path, 'rb') as f:
json_bytes = f.read()
blob: AbstractBlob) -> 'StreamDescriptor':
with blob.reader_context() as blob_reader:
json_bytes = blob_reader.read()
try:
decoded = json.loads(json_bytes.decode())
except json.JSONDecodeError:
@@ -160,8 +172,10 @@ class StreamDescriptor:
@classmethod
async def from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
blob: BlobFile) -> 'StreamDescriptor':
return await loop.run_in_executor(None, lambda: cls._from_stream_descriptor_blob(loop, blob_dir, blob))
blob: AbstractBlob) -> 'StreamDescriptor':
if not blob.is_readable():
raise InvalidStreamDescriptorError(f"unreadable/missing blob: {blob.blob_hash}")
return await loop.run_in_executor(None, cls._from_stream_descriptor_blob, loop, blob_dir, blob)
@staticmethod
def get_blob_hashsum(b: typing.Dict):
@@ -194,11 +208,12 @@ class StreamDescriptor:
return h.hexdigest()
@classmethod
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False) -> 'StreamDescriptor':
async def create_stream(
cls, loop: asyncio.BaseEventLoop, blob_dir: str, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> 'StreamDescriptor':
blobs: typing.List[BlobInfo] = []
iv_generator = iv_generator or random_iv_generator()
@@ -207,7 +222,7 @@ class StreamDescriptor:
for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
)
blobs.append(blob_info)
blobs.append(
@@ -216,7 +231,7 @@ class StreamDescriptor:
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
blobs
)
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
descriptor.sd_hash = sd_blob.blob_hash
return descriptor
@@ -228,7 +243,7 @@ class StreamDescriptor:
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
@classmethod
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
async def recover(cls, blob_dir: str, sd_blob: 'AbstractBlob', stream_hash: str, stream_name: str,
suggested_file_name: str, key: str,
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
+103 -79
View File
@@ -1,108 +1,132 @@
import os
import asyncio
import typing
import logging
import binascii
from lbrynet.error import DownloadSDTimeout
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.blob_info import BlobInfo
log = logging.getLogger(__name__)
def drain_into(a: list, b: list):
while a:
b.append(a.pop())
class StreamDownloader(StreamAssembler):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
super().__init__(loop, blob_manager, sd_hash, output_file_name)
class StreamDownloader:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager', sd_hash: str,
descriptor: typing.Optional[StreamDescriptor] = None):
self.loop = loop
self.config = config
self.output_dir = output_dir or self.config.download_dir
self.output_file_name = output_file_name
self.blob_downloader: typing.Optional[BlobDownloader] = None
self.search_queue = asyncio.Queue(loop=loop)
self.peer_queue = asyncio.Queue(loop=loop)
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.descriptor: typing.Optional[StreamDescriptor]
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.search_queue = asyncio.Queue(loop=loop) # blob hashes to feed into the iterative finder
self.peer_queue = asyncio.Queue(loop=loop) # new peers to try
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.descriptor: typing.Optional[StreamDescriptor] = descriptor
self.node: typing.Optional['Node'] = None
self.assemble_task: typing.Optional[asyncio.Task] = None
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
self.fixed_peers_delay: typing.Optional[float] = None
self.added_fixed_peers = False
self.time_to_descriptor: typing.Optional[float] = None
self.time_to_first_bytes: typing.Optional[float] = None
async def setup(self): # start the peer accumulator and initialize the downloader
if self.blob_downloader:
raise Exception("downloader is already set up")
if self.node:
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.search_queue.put_nowait(self.sd_hash)
async def add_fixed_peers(self):
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
async def after_got_descriptor(self):
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to search")
async def after_finished(self):
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
self.blob_downloader.close()
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.assemble_task:
self.assemble_task.cancel()
self.assemble_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader = None
if self.stream_handle:
if not self.stream_handle.closed:
self.stream_handle.close()
self.stream_handle = None
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
self.output_path and os.path.isfile(self.output_path):
os.remove(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return await self.blob_downloader.download_blob(blob_hash, length)
def add_fixed_peers(self):
async def _add_fixed_peers():
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
if not self.config.reflector_servers:
return
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()):
self.fixed_peers_delay = 0.0
else:
self.fixed_peers_delay = self.config.fixed_peer_delay
self.loop.create_task(_add_fixed_peers())
def download(self, node: typing.Optional['Node'] = None):
self.node = node
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
self.add_fixed_peers()
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
async def load_descriptor(self, connection_id: int = 0):
# download or get the sd blob
sd_blob = self.blob_manager.get_blob(self.sd_hash)
if not sd_blob.get_is_verified():
try:
now = self.loop.time()
sd_blob = await asyncio.wait_for(
self.blob_downloader.download_blob(self.sd_hash, connection_id),
self.config.blob_download_timeout, loop=self.loop
)
log.info("downloaded sd blob %s", self.sd_hash)
self.time_to_descriptor = self.loop.time() - now
except asyncio.TimeoutError:
raise DownloadSDTimeout(self.sd_hash)
# parse the descriptor
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, sd_blob
)
log.info("loaded stream manifest %s", self.sd_hash)
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
# set up peer accumulation
if node:
self.node = node
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
await self.add_fixed_peers()
# start searching for peers for the sd hash
self.search_queue.put_nowait(self.sd_hash)
log.info("searching for peers for stream %s", self.sd_hash)
if not self.descriptor:
await self.load_descriptor(connection_id)
# add the head blob to the peer search
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to peer search for stream %s", self.sd_hash)
if not await self.blob_manager.storage.stream_exists(self.sd_hash):
await self.blob_manager.storage.store_stream(
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
)
async def download_stream_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> 'AbstractBlob':
if not filter(lambda blob: blob.blob_hash == blob_info.blob_hash, self.descriptor.blobs[:-1]):
raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
blob = await self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id)
return blob
def decrypt_blob(self, blob_info: 'BlobInfo', blob: 'AbstractBlob') -> bytes:
return blob.decrypt(
binascii.unhexlify(self.descriptor.key.encode()), binascii.unhexlify(blob_info.iv.encode())
)
async def read_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> bytes:
start = None
if self.time_to_first_bytes is None:
start = self.loop.time()
blob = await self.download_stream_blob(blob_info, connection_id)
decrypted = self.decrypt_blob(blob_info, blob)
if start:
self.time_to_first_bytes = self.loop.time() - start
return decrypted
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader.close()
+323 -50
View File
@@ -3,56 +3,124 @@ import asyncio
import typing
import logging
import binascii
from aiohttp.web import Request, StreamResponse
from lbrynet.utils import generate_id
from lbrynet.error import DownloadSDTimeout
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
from lbrynet.extras.daemon.storage import StoredStreamClaim
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.schema.claim import Claim
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.wallet.transaction import Transaction
log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class ManagedStream:
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
downloader: typing.Optional[StreamDownloader] = None,
__slots__ = [
'loop',
'config',
'blob_manager',
'sd_hash',
'download_directory',
'_file_name',
'_status',
'stream_claim_info',
'download_id',
'rowid',
'written_bytes',
'content_fee',
'downloader',
'analytics_manager',
'fully_reflected',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
'streaming',
'_running',
'saving',
'finished_writing',
'started_writing',
]
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None):
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
descriptor: typing.Optional[StreamDescriptor] = None,
content_fee: typing.Optional['Transaction'] = None,
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.rowid = rowid
self.sd_hash = sd_hash
self.download_directory = download_directory
self._file_name = file_name
self.descriptor = descriptor
self.downloader = downloader
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.written_bytes = 0
self.content_fee = content_fee
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.tx = None
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
self.streaming = asyncio.Event(loop=self.loop)
self._running = asyncio.Event(loop=self.loop)
self.saving = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop)
@property
def descriptor(self) -> StreamDescriptor:
return self.downloader.descriptor
@property
def stream_hash(self) -> str:
return self.descriptor.stream_hash
@property
def file_name(self) -> typing.Optional[str]:
return self.downloader.output_file_name if self.downloader else self._file_name
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
@property
def status(self) -> str:
return self._status
def update_status(self, status: str):
async def update_status(self, status: str):
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
self._status = status
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
@property
def finished(self) -> bool:
@@ -95,57 +163,67 @@ class ManagedStream:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) ->typing.Optional[typing.Dict]:
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property
def blobs_completed(self) -> int:
return sum([1 if self.blob_manager.get_blob(b.blob_hash).get_is_verified() else 0
return sum([1 if self.blob_manager.is_blob_verified(b.blob_hash) else 0
for b in self.descriptor.blobs[:-1]])
@property
def blobs_in_stream(self) -> int:
return len(self.descriptor.blobs) - 1
@property
def sd_hash(self):
return self.descriptor.sd_hash
@property
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
def as_dict(self) -> typing.Dict:
full_path = self.full_path if self.output_file_exists else None
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
@property
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
if self.downloader and self.downloader.written_bytes:
written_bytes = self.downloader.written_bytes
elif full_path:
written_bytes = os.stat(full_path).st_size
def as_dict(self) -> typing.Dict:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if self.full_path and self.output_file_exists:
if self.written_bytes:
written_bytes = self.written_bytes
else:
written_bytes = os.stat(self.full_path).st_size
else:
full_path = None
file_name = None
download_directory = None
written_bytes = None
return {
'completed': self.finished,
'file_name': self.file_name,
'download_directory': self.download_directory,
'completed': (self.output_file_exists and self.status in ('stopped', 'finished')) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
'tx': self.tx,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
'suggested_file_name': self.descriptor.suggested_file_name,
'sd_hash': self.descriptor.sd_hash,
'download_path': full_path,
'mime_type': mime_type,
'mime_type': self.mime_type,
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
@@ -159,38 +237,180 @@ class ManagedStream:
'nout': self.nout,
'outpoint': self.outpoint,
'metadata': self.metadata,
'protobuf': self.metadata_protobuf,
'channel_claim_id': self.channel_claim_id,
'channel_name': self.channel_name,
'claim_name': self.claim_name
'claim_name': self.claim_name,
'content_fee': self.content_fee
}
@classmethod
async def create(cls, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager',
async def create(cls, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=blob_manager.blob_completed
)
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
await blob_manager.blob_completed(sd_blob)
for blob in descriptor.blobs[:-1]:
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
status=cls.STATUS_FINISHED)
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
def start_download(self, node: typing.Optional['Node']):
self.downloader.download(node)
self.update_status(self.STATUS_RUNNING)
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
save_now: bool = False):
timeout = timeout or self.config.download_timeout
if self._running.is_set():
return
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
self._running.set()
try:
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
except asyncio.TimeoutError:
self._running.clear()
raise DownloadSDTimeout(self.sd_hash)
def stop_download(self):
if self.downloader:
self.downloader.stop()
self.downloader = None
if self.delayed_stop_task and not self.delayed_stop_task.done():
self.delayed_stop_task.cancel()
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
if not await self.blob_manager.storage.file_exists(self.sd_hash):
if save_now:
file_name, download_dir = self._file_name, self.download_directory
else:
file_name, download_dir = None, None
self.rowid = await self.blob_manager.storage.save_downloaded_file(
self.stream_hash, file_name, download_dir, 0.0
)
if self.status != self.STATUS_RUNNING:
await self.update_status(self.STATUS_RUNNING)
async def stop(self, finished: bool = False):
"""
Stop any running save/stream tasks as well as the downloader and update the status in the database
"""
self.stop_tasks()
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
if start_blob_num >= len(self.descriptor.blobs[:-1]):
raise IndexError(start_blob_num)
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
assert i + start_blob_num == blob_info.blob_num
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
await self.start(node)
headers, size, skip_blobs = self._prepare_range_response_headers(request.headers.get('range', 'bytes=0-'))
response = StreamResponse(
status=206,
headers=headers
)
await response.prepare(request)
self.streaming_responses.append((request, response))
self.streaming.set()
try:
wrote = 0
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=2):
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
await response.write_eof(decrypted)
else:
await response.write(decrypted)
wrote += len(decrypted)
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
if response._eof_sent:
break
return response
finally:
response.force_close()
if (request, response) in self.streaming_responses:
self.streaming_responses.remove((request, response))
if not self.streaming_responses:
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
output_path)
self.saving.set()
self.finished_writing.clear()
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=1):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
self.download_id, self.claim_name, self.sd_hash
))
self.finished_writing.set()
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
self.sd_hash[:6], self.full_path)
await self.blob_manager.storage.set_saved_file(self.stream_hash)
except Exception as err:
if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path)
if not isinstance(err, asyncio.CancelledError):
log.exception("unexpected error encountered writing file for stream %s", self.sd_hash)
raise err
finally:
self.saving.clear()
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
node: typing.Optional['Node'] = None):
await self.start(node)
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
self.file_output_task.cancel()
self.download_directory = download_directory or self.download_directory or self.config.download_dir
if not self.download_directory:
raise ValueError("no directory to download to")
if not (file_name or self._file_name or self.descriptor.suggested_file_name):
raise ValueError("no file name to download to")
if not os.path.isdir(self.download_directory):
log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
os.mkdir(self.download_directory)
self._file_name = await get_next_available_file_name(
self.loop, self.download_directory,
file_name or self.descriptor.suggested_file_name
)
await self.blob_manager.storage.change_file_download_dir_and_file_name(
self.stream_hash, self.download_directory, self.file_name
)
await self.update_status(ManagedStream.STATUS_RUNNING)
self.written_bytes = 0
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
await self.started_writing.wait()
def stop_tasks(self):
if self.file_output_task and not self.file_output_task.done():
self.file_output_task.cancel()
self.file_output_task = None
while self.streaming_responses:
req, response = self.streaming_responses.pop()
response.force_close()
req.transport.close()
self.downloader.stop()
self._running.clear()
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
sent = []
@@ -206,7 +426,9 @@ class ManagedStream:
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return []
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
we_have = [
blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes
]
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
@@ -229,3 +451,54 @@ class ManagedStream:
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
if not claim_info:
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
self.set_claim(claim_info, claim_info['value'])
async def _delayed_stop(self):
stalled_count = 0
while self._running.is_set():
if self.saving.is_set() or self.streaming.is_set():
stalled_count = 0
else:
stalled_count += 1
if stalled_count > 1:
log.info("stopping inactive download for lbry://%s#%s (%s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
await self.stop()
return
await asyncio.sleep(1, loop=self.loop)
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
if '=' in get_range:
get_range = get_range.split('=')[1]
start, end = get_range.split('-')
size = 0
for blob in self.descriptor.blobs[:-1]:
size += blob.length - 1
if self.stream_claim_info and self.stream_claim_info.claim.stream.source.size:
size_from_claim = int(self.stream_claim_info.claim.stream.source.size)
if not size_from_claim <= size <= size_from_claim + 16:
raise ValueError("claim contains implausible stream size")
log.debug("using stream size from claim")
size = size_from_claim
elif self.stream_claim_info:
log.debug("estimating stream size")
start = int(start)
end = int(end) if end else size - 1
skip_blobs = start // 2097150
skip = skip_blobs * 2097151
start = skip
final_size = end - start + 1
headers = {
'Accept-Ranges': 'bytes',
'Content-Range': f'bytes {start}-{end}/{size}',
'Content-Length': str(final_size),
'Content-Type': self.mime_type
}
return headers, size, skip_blobs
+5 -5
View File
@@ -4,7 +4,7 @@ import logging
import typing
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.descriptor import StreamDescriptor
REFLECTOR_V1 = 0
@@ -14,7 +14,7 @@ log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
@@ -64,7 +64,7 @@ class StreamReflectorClient(asyncio.Protocol):
async def send_descriptor(self) -> typing.Tuple[bool, typing.List[str]]: # returns a list of needed blob hashes
sd_blob = self.blob_manager.get_blob(self.descriptor.sd_hash)
assert sd_blob.get_is_verified(), "need to have a sd blob to send at this point"
assert self.blob_manager.is_blob_verified(self.descriptor.sd_hash), "need to have sd blob to send at this point"
response = await self.send_request({
'sd_blob_hash': sd_blob.blob_hash,
'sd_blob_size': sd_blob.length
@@ -80,7 +80,7 @@ class StreamReflectorClient(asyncio.Protocol):
sent_sd = True
if not needed:
for blob in self.descriptor.blobs[:-1]:
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
if self.blob_manager.is_blob_verified(blob.blob_hash, blob.length):
needed.append(blob.blob_hash)
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
self.reflected_blobs.append(sd_blob.blob_hash)
@@ -91,8 +91,8 @@ class StreamReflectorClient(asyncio.Protocol):
return sent_sd, needed
async def send_blob(self, blob_hash: str):
assert self.blob_manager.is_blob_verified(blob_hash), "need to have a blob to send at this point"
blob = self.blob_manager.get_blob(blob_hash)
assert blob.get_is_verified(), "need to have a blob to send at this point"
response = await self.send_request({
'blob_hash': blob.blob_hash,
'blob_size': blob.length
+7 -7
View File
@@ -7,7 +7,7 @@ from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.writer import HashBlobWriter
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager'):
def __init__(self, blob_manager: 'BlobManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -63,11 +63,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
return
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.open_for_writing()
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.incoming.set()
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
@@ -102,11 +102,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
return
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified():
self.writer = blob.open_for_writing()
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.incoming.set()
self.send_response({"send_blob": True})
try:
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop)
self.send_response({"received_blob": True})
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
@@ -121,7 +121,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobFileManager'):
def __init__(self, blob_manager: 'BlobManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
+226 -303
View File
@@ -5,22 +5,23 @@ import binascii
import logging
import random
from decimal import Decimal
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
DownloadDataTimeout, DownloadSDTimeout
from lbrynet.utils import generate_id
from aiohttp.web import Request
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError
from lbrynet.error import ResolveTimeout, DownloadDataTimeout
from lbrynet.utils import cache_concurrent
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import Claim
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.storage import lbc_to_dewies
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
from lbrynet.wallet import LbryWalletManager
from lbrynet.wallet.transaction import Transaction
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
@@ -54,8 +55,14 @@ comparison_operators = {
}
def path_or_none(p) -> typing.Optional[str]:
if not p:
return
return binascii.unhexlify(p).decode()
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
@@ -65,79 +72,23 @@ class StreamManager:
self.storage = storage
self.node = node
self.analytics_manager = analytics_manager
self.streams: typing.Set[ManagedStream] = set()
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
self.resume_downloading_task: asyncio.Task = None
self.re_reflect_task: asyncio.Task = None
self.streams: typing.Dict[str, ManagedStream] = {}
self.resume_saving_task: typing.Optional[asyncio.Task] = None
self.re_reflect_task: typing.Optional[asyncio.Task] = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.running_reflector_uploads: typing.List[asyncio.Task] = []
self.started = asyncio.Event(loop=self.loop)
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, claim_info['value'])
async def start_stream(self, stream: ManagedStream) -> bool:
"""
Resume or rebuild a partial or completed stream
"""
if not stream.running and not stream.output_file_exists:
if stream.downloader:
stream.downloader.stop()
stream.downloader = None
# the directory is gone, can happen when the folder that contains a published file is deleted
# reset the download directory to the default and update the file name
if not os.path.isdir(stream.download_directory):
stream.download_directory = self.config.download_dir
stream.downloader = self.make_downloader(
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
)
if stream.status != ManagedStream.STATUS_FINISHED:
await self.storage.change_file_status(stream.stream_hash, 'running')
stream.update_status('running')
stream.start_download(self.node)
try:
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
self.config.download_timeout)
except asyncio.TimeoutError:
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
return False
file_name = os.path.basename(stream.downloader.output_path)
output_dir = os.path.dirname(stream.downloader.output_path)
await self.storage.change_file_download_dir_and_file_name(
stream.stream_hash, output_dir, file_name
)
stream._file_name = file_name
stream.download_directory = output_dir
self.wait_for_stream_finished(stream)
return True
return True
async def stop_stream(self, stream: ManagedStream):
stream.stop_download()
if not stream.finished and stream.output_file_exists:
try:
os.remove(stream.full_path)
except OSError as err:
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
str(err))
if stream.running:
stream.update_status(ManagedStream.STATUS_STOPPED)
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
return StreamDownloader(
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
)
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = []
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
suggested_file_name: str, key: str,
content_fee: typing.Optional['Transaction']) -> typing.Optional[StreamDescriptor]:
sd_blob = self.blob_manager.get_blob(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash)
descriptor = await StreamDescriptor.recover(
@@ -145,116 +96,126 @@ class StreamManager:
)
if not descriptor:
return
to_restore.append((descriptor, sd_blob))
to_restore.append((descriptor, sd_blob, content_fee))
await asyncio.gather(*[
recover_stream(
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key'],
file_info['content_fee']
) for file_info in file_infos
])
if to_restore:
await self.storage.recover_streams(to_restore, self.config.download_dir)
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
claim: typing.Optional['StoredStreamClaim']):
sd_blob = self.blob_manager.get_blob(sd_hash)
if not sd_blob.get_is_verified():
return
# if self.blob_manager._save_blobs:
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: typing.Optional[str],
download_directory: typing.Optional[str], status: str,
claim: typing.Optional['StoredStreamClaim'], content_fee: typing.Optional['Transaction']):
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err:
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
return
if status == ManagedStream.STATUS_RUNNING:
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
else:
downloader = None
stream = ManagedStream(
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
self.loop, self.config, self.blob_manager, descriptor.sd_hash, download_directory, file_name, status,
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
analytics_manager=self.analytics_manager
)
self.streams.add(stream)
self.streams[sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
async def load_streams_from_database(self):
async def load_and_resume_streams_from_database(self):
to_recover = []
for file_info in await self.storage.get_all_lbry_files():
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_recover.append(file_info)
to_start = []
await self.storage.update_manually_removed_files_since_last_run()
for file_info in await self.storage.get_all_lbry_files():
# if the sd blob is not verified, try to reconstruct it from the database
# this could either be because the blob files were deleted manually or save_blobs was not true when
# the stream was downloaded
if not self.blob_manager.is_blob_verified(file_info['sd_hash']):
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Attempting to recover %i streams", len(to_recover))
await self.recover_streams(to_recover)
to_start = []
for file_info in await self.storage.get_all_lbry_files():
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_start.append(file_info)
log.info("Initializing %i files", len(to_start))
await asyncio.gather(*[
self.add_stream(
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
file_info['claim']
) for file_info in to_start
])
to_resume_saving = []
add_stream_tasks = []
for file_info in to_start:
file_name = path_or_none(file_info['file_name'])
download_directory = path_or_none(file_info['download_directory'])
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
add_stream_tasks.append(self.loop.create_task(self.add_stream(
file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'],
file_info['claim'], file_info['content_fee']
)))
if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop)
log.info("Started stream manager with %i files", len(self.streams))
async def resume(self):
if self.node:
await self.node.joined.wait()
else:
if not self.node:
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
t = [
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
]
if t:
log.info("resuming %i downloads", len(t))
if to_resume_saving:
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
async def resume(self, to_resume_saving):
log.info("Resuming saving %i files", len(to_resume_saving))
await asyncio.gather(
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
for (file_name, download_directory, sd_hash) in to_resume_saving),
loop=self.loop
)
async def reflect_streams(self):
while True:
if self.config.reflect_streams and self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
while sd_hashes:
stream = self.streams[sd_hashes.pop()]
if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed:
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch)
await asyncio.gather(*batch, loop=self.loop)
batch = []
if batch:
await asyncio.gather(*batch)
await asyncio.gather(*batch, loop=self.loop)
await asyncio.sleep(300, loop=self.loop)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
await self.load_and_resume_streams_from_database()
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
self.started.set()
def stop(self):
if self.resume_downloading_task and not self.resume_downloading_task.done():
self.resume_downloading_task.cancel()
if self.resume_saving_task and not self.resume_saving_task.done():
self.resume_saving_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
stream = self.streams.pop()
stream.stop_download()
_, stream = self.streams.popitem()
stream.stop_tasks()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads:
self.running_reflector_uploads.pop().cancel()
self.started.clear()
log.info("finished stopping the stream manager")
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers:
host, port = random.choice(self.config.reflector_servers)
@@ -267,9 +228,9 @@ class StreamManager:
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
stream.stop_tasks()
if stream.sd_hash in self.streams:
del self.streams[stream.sd_hash]
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor)
@@ -277,7 +238,7 @@ class StreamManager:
os.remove(stream.full_path)
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
if streams:
return streams[0]
@@ -304,66 +265,33 @@ class StreamManager:
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams:
for stream in self.streams.values():
for search, val in search_by.items():
if comparison_operators[comparison](getattr(stream, search), val):
streams.append(stream)
break
else:
streams = list(self.streams)
streams = list(self.streams.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
stream.download_id, stream.claim_name, stream.sd_hash
))
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _store_stream(self, downloader: StreamDownloader) -> int:
file_name = os.path.basename(downloader.output_path)
download_directory = os.path.dirname(downloader.output_path)
if not await self.storage.stream_exists(downloader.sd_hash):
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.storage.file_exists(downloader.sd_hash):
return await self.storage.save_downloaded_file(
downloader.descriptor.stream_hash, file_name, download_directory,
0.0
)
else:
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim) -> typing.Tuple[
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
await self.start_stream(existing[0])
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
f"download {claim_id}")
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
await self.start_stream(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
@@ -372,142 +300,137 @@ class StreamManager:
return None, existing_for_claim_id[0]
return None, None
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
download_id: str, outpoint: str, claim: Claim, resolved: typing.Dict,
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
downloader.download(self.node)
await downloader.got_descriptor.wait()
got_descriptor_time.set_result(self.loop.time() - start_time)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, outpoint
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
stream.set_claim(resolved, claim)
await stream.downloader.wrote_bytes_event.wait()
self.streams.add(stream)
return stream
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['hex']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
file_name)
), timeout
)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
claim.stream.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
time_to_first_bytes, None if not error else error.__class__.__name__
)
)
if error:
raise error
return stream
@cache_concurrent
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: typing.Optional[float] = None,
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> ManagedStream:
download_directory: typing.Optional[str] = None,
save_file: typing.Optional[bool] = None,
resolve_timeout: float = 3.0) -> ManagedStream:
timeout = timeout or self.config.download_timeout
if uri in self.starting_streams:
return await self.starting_streams[uri]
fut = asyncio.Future(loop=self.loop)
self.starting_streams[uri] = fut
start_time = self.loop.time()
resolved_time = None
stream = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
try:
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
fut.set_result(stream)
except Exception as err:
fut.set_exception(err)
try:
return await fut
# resolve the claim
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(self.wallet.ledger.resolve(0, 1, uri), resolve_timeout)
except asyncio.TimeoutError:
raise ResolveTimeout(uri)
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values() if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory, node=self.node
)
return updated_stream
content_fee = None
fee_amount, fee_address = None, None
# check that the fee is payable
if not to_replace and claim.stream.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
stream = ManagedStream(
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
file_name, ManagedStream.STATUS_RUNNING, content_fee=content_fee,
analytics_manager=self.analytics_manager
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
elif fee_address:
stream.content_fee = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1')
)
log.info("paid fee of %s for %s", fee_amount, uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeout(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, dont delete stream
error = err
raise
finally:
del self.starting_streams[uri]
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__
)
)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.streams[sd_hash].stream_file(request, self.node)
@@ -7,7 +7,6 @@ from binascii import unhexlify
from torba.testcase import IntegrationTestCase
import lbrynet.wallet
from lbrynet.schema.claim import Claim
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
@@ -19,7 +18,7 @@ from lbrynet.extras.daemon.Components import (
)
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.reflector.server import ReflectorServer
from lbrynet.blob_exchange.server import BlobServer
@@ -108,9 +107,11 @@ class CommandTestCase(IntegrationTestCase):
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_storage = SQLiteStorage(Config(), ':memory:')
self.server_config = Config()
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
await self.server_storage.open()
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.server.start_server(5567, '127.0.0.1')
await self.server.started_listening.wait()
@@ -123,7 +124,7 @@ class CommandTestCase(IntegrationTestCase):
async def asyncTearDown(self):
await super().asyncTearDown()
self.wallet_component._running = False
await self.daemon.stop()
await self.daemon.stop(shutdown_runner=False)
async def confirm_tx(self, txid):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
@@ -150,33 +151,64 @@ class CommandTestCase(IntegrationTestCase):
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
if confirm:
await self.generate(1)
return txid
async def blockchain_update_name(self, txid: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('updateclaim', txid, value, amount)
if confirm:
await self.generate(1)
return txid
async def out(self, awaitable):
""" Converts Daemon API call results (dictionary)
to JSON and then back to a dictionary. """
""" Serializes lbrynet API results to JSON then loads and returns it as dictionary. """
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
def sout(self, value):
""" Synchronous version of `out` method. """
return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
async def stream_create(self, name='hovercraft', bid='1.0', data=b'hi!', confirm=True, **kwargs):
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
)
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
await self.generate(1)
await self.on_transaction_dict(claim)
return claim
file = tempfile.NamedTemporaryFile()
def cleanup():
try:
file.close()
except FileNotFoundError:
pass
self.addCleanup(cleanup)
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
)
self.assertEqual(claim['outputs'][0]['name'], name)
if confirm:
await self.on_transaction_dict(claim)
await self.generate(1)
await self.on_transaction_dict(claim)
return claim
async def stream_update(self, claim_id, data=None, confirm=True, **kwargs):
if data:
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
)
file = tempfile.NamedTemporaryFile()
file.write(data)
file.flush()
def cleanup():
try:
file.close()
except FileNotFoundError:
pass
self.addCleanup(cleanup)
claim = await self.out(
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
)
else:
claim = await self.out(self.daemon.jsonrpc_stream_update(claim_id, **kwargs))
self.assertIsNotNone(claim['outputs'][0]['name'])
+40 -1
View File
@@ -14,6 +14,7 @@ import pkg_resources
import contextlib
import certifi
import aiohttp
import functools
from lbrynet.schema.claim import Claim
from lbrynet.cryptoutils import get_lbry_hash_obj
@@ -116,7 +117,7 @@ def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, Claim):
return stream_info.stream.sd_hash
return stream_info.stream.source.sd_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\
@@ -146,6 +147,44 @@ def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
cancel_task(tasks.pop())
def async_timed_cache(duration: int):
def wrapper(fn):
cache: typing.Dict[typing.Tuple,
typing.Tuple[typing.Any, float]] = {}
@functools.wraps(fn)
async def _inner(*args, **kwargs) -> typing.Any:
loop = asyncio.get_running_loop()
now = loop.time()
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
if key in cache and (now - cache[key][1] < duration):
return cache[key][0]
to_cache = await fn(*args, **kwargs)
cache[key] = to_cache, now
return to_cache
return _inner
return wrapper
def cache_concurrent(async_fn):
"""
When the decorated function has concurrent calls made to it with the same arguments, only run it once
"""
cache: typing.Dict = {}
@functools.wraps(async_fn)
async def wrapper(*args, **kwargs):
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
try:
return await cache[key]
finally:
cache.pop(key, None)
return wrapper
@async_timed_cache(300)
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
+6 -5
View File
@@ -4,7 +4,7 @@ import binascii
from hashlib import sha256
from string import hexdigits
from torba.client.baseaccount import BaseAccount
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
from torba.client.basetransaction import TXORef
@@ -167,10 +167,11 @@ class Account(BaseAccount):
))
async def save_max_gap(self):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
if issubclass(self.address_generator, HierarchicalDeterministic):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
def get_balance(self, confirmations=0, include_claims=False, **constraints):
if not include_claims:
+6 -26
View File
@@ -3,10 +3,9 @@ import logging
from binascii import unhexlify
from torba.client.baseledger import BaseLedger
from lbrynet.schema.uri import parse_lbry_uri, URIParseError
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.wallet.resolve import Resolver
from lbrynet.wallet.account import Account, validate_claim_id
from lbrynet.wallet.account import Account
from lbrynet.wallet.network import Network
from lbrynet.wallet.database import WalletDatabase
from lbrynet.wallet.transaction import Transaction
@@ -48,39 +47,20 @@ class MainNetLedger(BaseLedger):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self.resolver = Resolver(self)
@property
def resolver(self):
return Resolver(self.headers.claim_trie_root, self.headers.height, self.transaction_class,
hash160_to_address=self.hash160_to_address, network=self.network, ledger=self)
async def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.claim_id:
validate_claim_id(parsed_uri.claim_id)
except URIParseError as err:
return {'error': err.args[0]}
except Exception as e:
return {'error': str(e)}
try:
resolver = self.resolver
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
except Exception as e:
log.exception(e)
return {'error': str(e)}
def resolve(self, page, page_size, *uris):
return self.resolver.resolve(page, page_size, *uris)
async def get_claim_by_claim_id(self, claim_id):
result = (await self.network.get_claims_by_ids(claim_id)).pop(claim_id, {})
return await self.resolver.get_certificate_and_validate_result(result)
return await self.resolver.parse_and_validate_claim_result(result)
async def get_claim_by_outpoint(self, txid, nout):
claims = (await self.network.get_claims_in_tx(txid)) or []
for claim in claims:
if claim['nout'] == nout:
return await self.resolver.get_certificate_and_validate_result(claim)
return await self.resolver.parse_and_validate_claim_result(claim)
return 'claim not found'
async def start(self):
-7
View File
@@ -308,13 +308,6 @@ class LbryWalletManager(BaseWalletManager):
for wallet in self.wallets:
wallet.save()
def get_block(self, block_hash=None, height=None):
if height is None:
height = self.ledger.headers.height
if block_hash is None:
block_hash = self.ledger.headers.hash(height).decode()
return self.ledger.network.get_block(block_hash)
def get_claim_by_claim_id(self, claim_id):
return self.ledger.get_claim_by_claim_id(claim_id)
-3
View File
@@ -3,9 +3,6 @@ from torba.client.basenetwork import BaseNetwork
class Network(BaseNetwork):
def get_block(self, block_hash):
return self.rpc('blockchain.block.get_block', block_hash)
def get_server_height(self):
return self.rpc('blockchain.block.get_server_height')
+142 -217
View File
@@ -1,36 +1,71 @@
import logging
import asyncio
from functools import lru_cache
from cryptography.exceptions import InvalidSignature
from binascii import unhexlify, hexlify
from lbrynet.wallet.account import validate_claim_id
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
from lbrynet.schema.claim import Claim
from google.protobuf.message import DecodeError
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.uri import parse_lbry_uri, URIParseError
from lbrynet.wallet.claim_proofs import verify_proof, InvalidProofError
from lbrynet.wallet.transaction import Transaction
log = logging.getLogger(__name__)
class Resolver:
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network, ledger):
self.claim_trie_root = claim_trie_root
self.height = height
self.transaction_class = transaction_class
self.hash160_to_address = hash160_to_address
self.network = network
def __init__(self, ledger):
self.transaction_class = ledger.transaction_class
self.network = ledger.network
self.ledger = ledger
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
async def resolve(self, page, page_size, *uris):
uris = set(uris)
try:
for uri in uris:
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.claim_id:
validate_claim_id(parsed_uri.claim_id)
claim_trie_root = self.ledger.headers.claim_trie_root
resolutions = await self.network.get_values_for_uris(self.ledger.headers.hash().decode(), *uris)
if len(uris) > 1:
return await self._batch_handle(resolutions, uris, page, page_size, claim_trie_root)
return await self._handle_resolutions(resolutions, uris, page, page_size, claim_trie_root)
except URIParseError as err:
return {'error': err.args[0]}
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def _batch_handle(self, resolutions, uris, page, page_size, claim_trie_root):
futs = []
for uri in uris:
futs.append(
asyncio.ensure_future(self._handle_resolutions(resolutions, [uri], page, page_size, claim_trie_root))
)
results = await asyncio.gather(*futs)
return dict(list(map(lambda result: list(result.items())[0], results)))
@lru_cache(256)
def _fetch_tx(self, txid):
async def __fetch_parse(txid):
return self.transaction_class(unhexlify(await self.network.get_transaction(txid)))
return asyncio.ensure_future(__fetch_parse(txid))
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size, claim_trie_root):
results = {}
for uri in requested_uris:
resolution = (resolutions or {}).get(uri, {})
if resolution:
try:
results[uri] = _handle_claim_result(
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
await self._handle_resolve_uri_response(uri, resolution, claim_trie_root, page, page_size),
uri
)
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
log.exception(err)
@@ -39,50 +74,24 @@ class Resolver:
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
return results
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
async def _handle_resolve_uri_response(self, uri, resolution, claim_trie_root, page=0, page_size=10):
result = {}
claim_trie_root = self.claim_trie_root
parsed_uri = parse_lbry_uri(uri)
certificate = None
certificate_response = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate']['resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
height = certificate_response['height']
depth = self.height - height
certificate_result = _verify_proof(parsed_uri.name,
certificate_response = _verify_proof(parsed_uri.name,
claim_trie_root,
certificate_response,
height, depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
raw=raw)
elif certificate_resolution_type == "claim_id":
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
raw=raw)
elif certificate_resolution_type == "sequence":
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
raw=raw)
else:
log.error("unknown response type: %s", certificate_resolution_type)
if 'certificate' in result:
certificate = result['certificate']
if 'unverified_claims_in_channel' in resolution:
max_results = len(resolution['unverified_claims_in_channel'])
result['claims_in_channel'] = max_results
else:
result['claims_in_channel'] = 0
else:
result['error'] = "claim not found"
result['success'] = False
result['uri'] = str(parsed_uri)
else:
certificate = None
ledger=self.ledger)
elif certificate_resolution_type not in ['winning', 'claim_id', 'sequence']:
raise Exception(f"unknown response type: {certificate_resolution_type}")
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response)
result['claims_in_channel'] = len(resolution.get('unverified_claims_in_channel', []))
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
@@ -90,27 +99,14 @@ class Resolver:
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
height = claim_response['height']
depth = self.height - height
claim_result = _verify_proof(parsed_uri.name,
claim_trie_root,
claim_response,
height, depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
certificate,
raw)
elif claim_resolution_type == "claim_id":
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate,
raw)
elif claim_resolution_type == "sequence":
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate,
raw)
else:
log.error("unknown response type: %s", claim_resolution_type)
claim_response = _verify_proof(parsed_uri.name,
claim_trie_root,
claim_response,
ledger=self.ledger)
elif claim_resolution_type not in ["sequence", "winning", "claim_id"]:
raise Exception(f"unknown response type: {claim_resolution_type}")
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate_response)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
@@ -152,58 +148,51 @@ class Resolver:
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
return result
async def get_certificate_and_validate_result(self, claim_result):
async def parse_and_validate_claim_result(self, claim_result, certificate=None):
if not claim_result or 'value' not in claim_result:
return claim_result
certificate = None
certificate_id = Claim.from_bytes(unhexlify(claim_result['value'])).signing_channel_id
if certificate_id:
certificate = await self.network.get_claims_by_ids(certificate_id)
certificate = certificate.pop(certificate_id) if certificate else None
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
claim_result = _decode_claim_result(claim_result)
if claim_result.get('height'):
claim_result['timestamp'] = self.ledger.headers[claim_result['height']]['timestamp']
if claim_result.get('depth'):
claim_result['confirmations'] = claim_result.pop('depth')
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
if not claim_result or 'value' not in claim_result:
return claim_result
claim_result['decoded_claim'] = False
decoded = None
if not raw:
claim_value = claim_result['value']
try:
decoded = claim_result['value'] = Claim.from_bytes(unhexlify(claim_value))
claim_result['decoded_claim'] = True
except DecodeError:
pass
if decoded:
if claim_result['value']:
claim_result['has_signature'] = False
if decoded.is_signed:
claim_tx = await self.network.get_transaction(claim_result['txid'])
if claim_result['value'].is_signed:
claim_result['has_signature'] = True
claim_tx = await self._fetch_tx(claim_result['txid'])
if certificate is None:
log.info("fetching certificate to check claim signature")
certificate = await self.network.get_claims_by_ids(decoded.signing_channel_id)
channel_id = claim_result['value'].signing_channel_id
certificate = (await self.network.get_claims_by_ids(channel_id)).get(channel_id)
if not certificate:
log.warning('Certificate %s not found', decoded.signing_channel_id)
cert_tx = await self.network.get_transaction(certificate['txid']) if certificate else None
claim_result['has_signature'] = True
claim_result['signature_is_valid'] = False
validated, channel_name = validate_claim_signature_and_get_channel_name(
log.warning('Certificate %s not found', channel_id)
claim_result['channel_name'] = certificate['name'] if certificate else None
cert_tx = await self._fetch_tx(certificate['txid']) if certificate else None
claim_result['signature_is_valid'] = validate_claim_signature_and_get_channel_name(
claim_result, certificate, self.ledger, claim_tx=claim_tx, cert_tx=cert_tx
)
claim_result['channel_name'] = channel_name
if validated:
claim_result['signature_is_valid'] = True
if 'height' in claim_result and claim_result['height'] is None:
claim_result['height'] = -1
# fixme: workaround while json encoder isnt used here
if cert_tx:
channel_txo = cert_tx.outputs[certificate['nout']]
claim_result['signing_channel'] = {
'name': channel_txo.claim_name,
'claim_id': channel_txo.claim_id,
'value': channel_txo.claim
}
claim_result['is_channel_signature_valid'] = claim_result['signature_is_valid']
if 'amount' in claim_result:
claim_result = format_amount_value(claim_result)
claim_result['amount'] = dewies_to_lbc(claim_result['amount'])
claim_result['effective_amount'] = dewies_to_lbc(claim_result['effective_amount'])
claim_result['supports'] = [
{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
for (txid, nout, amount) in claim_result['supports']
]
claim_result['permanent_url'] = _get_permanent_url(
claim_result, decoded.signing_channel_id if decoded else None)
claim_result['height'] = claim_result.get('height', -1) or -1
claim_result['permanent_url'] = f"lbry://{claim_result['name']}#{claim_result['claim_id']}"
return claim_result
@@ -284,87 +273,51 @@ class Resolver:
start_position = (page - 1) * page_size
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
channel_claim_infos)
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
certificate, page_size=page_size)
upper_bound = len(claim_positions)
if not page:
return None, upper_bound
if start_position > upper_bound:
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
certificate, page_size=page_size)
return page_generator, upper_bound
# Format amount to be decimal encoded string
# Format value to be hex encoded string
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
def format_amount_value(obj):
if isinstance(obj, dict):
for k, v in obj.items():
if k in ('amount', 'effective_amount'):
if not isinstance(obj[k], float):
obj[k] = dewies_to_lbc(obj[k])
elif k == 'supports' and isinstance(v, list):
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
for (txid, nout, amount) in v]
elif isinstance(v, (list, dict)):
obj[k] = format_amount_value(v)
elif isinstance(obj, list):
obj = [format_amount_value(o) for o in obj]
return obj
def _get_permanent_url(claim_result, certificate_id):
if certificate_id:
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
else:
return f"{claim_result['name']}#{claim_result['claim_id']}"
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
def _verify_proof(name, claim_trie_root, result, ledger):
"""
Verify proof for name claim
"""
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
claim_sequence, claim_address, supports):
def _build_response(name, tx, nOut):
output = tx.outputs[nOut]
r = {
'name': name,
'value': hexlify(value),
'claim_id': claim_id,
'txid': txid,
'nout': n,
'amount': amount,
'effective_amount': effective_amount,
'height': height,
'depth': depth,
'claim_sequence': claim_sequence,
'address': claim_address,
'supports': supports
'value': hexlify(output.script.values['claim']),
'claim_id': output.claim_id,
'txid': tx.id,
'nout': nOut,
'amount': output.amount,
'effective_amount': output.amount + support_amount,
'height': result['height'],
'confirmations': result['depth'],
'claim_sequence': result['claim_sequence'],
'address': output.get_address(ledger),
'valid_at_height': result['valid_at_height'],
'timestamp': ledger.headers[result['height']]['timestamp'],
'supports': result['supports']
}
return r
def _parse_proof_result(name, result):
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
supports = result['supports']
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
if 'transaction' in result:
tx = transaction_class(raw=unhexlify(result['transaction']))
tx = ledger.transaction_class(raw=unhexlify(result['transaction']))
nOut = result['proof']['nOut']
if result['proof']['txhash'] == tx.id:
if 0 <= nOut < len(tx.outputs):
claim_output = tx.outputs[nOut]
effective_amount = claim_output.amount + support_amount
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
claim_id = result['claim_id']
claim_sequence = result['claim_sequence']
claim_script = claim_output.script
decoded_name = claim_script.values['claim_name'].decode()
decoded_value = claim_script.values['claim']
if decoded_name == name:
return _build_response(name, decoded_value, claim_id,
tx.id, nOut, claim_output.amount,
effective_amount, claim_sequence,
claim_address, supports)
if tx.outputs[nOut].claim_name == name:
return _build_response(name, tx, nOut)
return {'error': 'name in proof did not match requested name'}
outputs = len(tx['outputs'])
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
@@ -375,12 +328,8 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
return {'error': 'name is not claimed'}
if 'proof' in result:
proof_name = name
if 'name' in result:
proof_name = result['name']
name = result['name']
if 'normalized_name' in result:
proof_name = result['normalized_name']
name = result.get('name', name)
proof_name = result.get('normalized_name', name)
try:
verify_proof(result['proof'], claim_trie_root, proof_name)
except InvalidProofError:
@@ -392,48 +341,46 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
def validate_claim_signature_and_get_channel_name(claim_result, certificate_claim, ledger,
claim_tx=None, cert_tx=None):
valid_signature = False
if cert_tx and certificate_claim and claim_tx and claim_result:
tx = Transaction(unhexlify(claim_tx))
cert_tx = Transaction(unhexlify(cert_tx))
try:
is_signed = tx.outputs[claim_result['nout']].is_signed_by(
valid_signature = claim_tx.outputs[claim_result['nout']].is_signed_by(
cert_tx.outputs[certificate_claim['nout']], ledger
)
except InvalidSignature:
return False, None
return is_signed, certificate_claim['name']
return False, None
pass
if not valid_signature:
log.warning("lbry://%s#%s has an invalid signature",
claim_result['name'], claim_result['claim_id'])
return valid_signature
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
def _decode_claim_result(claim):
if 'has_signature' in claim and claim['has_signature']:
if not claim['signature_is_valid']:
log.warning("lbry://%s#%s has an invalid signature",
claim['name'], claim['claim_id'])
if 'decoded_claim' in claim:
return claim
if 'value' not in claim:
log.warning('Got an invalid claim while parsing, please report: %s', claim)
claim['hex'] = None
claim['protobuf'] = None
claim['value'] = None
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
backend_message = ' SDK message: ' + claim.get('error', '')
claim['error'] = "Failed to parse: missing value." + backend_message
return claim
try:
if not isinstance(claim['value'], Claim):
claim['value'] = Claim.from_bytes(claim['value'])
claim['hex'] = hexlify(claim['value'].to_bytes())
claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
claim['protobuf'] = hexlify(claim['value'].to_bytes())
claim['decoded_claim'] = True
except DecodeError:
claim['hex'] = claim['value']
claim['decoded_claim'] = False
claim['protobuf'] = claim['value']
claim['value'] = None
claim['error'] = "Failed to decode value"
return claim
def _handle_claim_result(results):
def _handle_claim_result(results, uri):
if not results:
#TODO: cannot determine what name we searched for here
# we should fix lbryum commands that return None
raise UnknownNameError("")
raise UnknownURI(uri)
if 'error' in results:
if results['error'] in ['name is not claimed', 'claim not found']:
@@ -446,26 +393,8 @@ def _handle_claim_result(results):
if 'outpoint' in results:
raise UnknownOutpoint(results['outpoint'])
raise Exception(results['error'])
# case where return value is {'certificate':{'txid', 'value',...},...}
if 'certificate' in results:
results['certificate'] = _decode_claim_result(results['certificate'])
# case where return value is {'claim':{'txid','value',...},...}
if 'claim' in results:
results['claim'] = _decode_claim_result(results['claim'])
# case where return value is {'txid','value',...}
# returned by queries that are not name resolve related
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
elif 'value' in results:
results = _decode_claim_result(results)
# case where there is no 'certificate', 'value', or 'claim' key
elif 'certificate' not in results:
msg = f'result in unexpected format:{results}'
assert False, msg
if not {'value', 'claim', 'certificate'}.intersection(results.keys()):
raise Exception(f'result in unexpected format:{results}')
return results
@@ -474,13 +403,9 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
# potentially triggering that.
winner = None
winner = {}
for claim in claims_in_channel:
if not claim['signature_is_valid']:
continue
if winner is None:
winner = claim
elif claim['height'] < winner['height'] or \
if not winner or claim['height'] < winner['height'] or \
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
winner = claim
return winner
winner = claim if claim['signature_is_valid'] else winner
return winner or None
+1 -5
View File
@@ -41,7 +41,6 @@ class LBRYElectrumX(ElectrumX):
'blockchain.claimtrie.getvaluesforuris': self.claimtrie_getvalueforuris,
'blockchain.claimtrie.getclaimssignedbyid': self.claimtrie_getclaimssignedbyid,
'blockchain.block.get_server_height': self.get_server_height,
'blockchain.block.get_block': self.get_block,
}
# fixme: methods we use but shouldnt be using anymore. To be removed when torba goes out
handlers.update({
@@ -104,9 +103,6 @@ class LBRYElectrumX(ElectrumX):
verbose = False
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def get_block(self, block_hash):
return await self.daemon.deserialised_block(block_hash)
async def get_server_height(self):
return self.bp.height
@@ -241,7 +237,7 @@ class LBRYElectrumX(ElectrumX):
"txid": claim['txid'],
"nout": claim['n'],
"amount": amount,
"depth": self.db.db_height - height,
"depth": self.db.db_height - height + 1,
"height": height,
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
"address": address, # from index
+12 -12
View File
@@ -80,13 +80,7 @@ class Output(BaseOutput):
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
if self.channel is not None:
return "{}#{}/{}".format(
self.channel.claim_name,
self.channel.claim_id,
self.claim_name
)
return f"{self.claim_name}#{self.claim_id}"
return f"lbry://{self.claim_name}#{self.claim_id}"
raise ValueError('No claim associated.')
@property
@@ -98,11 +92,11 @@ class Output(BaseOutput):
pieces = [
Base58.decode(self.get_address(ledger)),
self.claim.unsigned_payload,
self.claim.signing_channel_hash
self.claim.signing_channel_hash[::-1]
]
else:
pieces = [
self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
self.tx_ref.tx.inputs[0].txo_ref.hash,
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]
@@ -120,7 +114,7 @@ class Output(BaseOutput):
self.channel = channel
self.claim.signing_channel_hash = channel.claim_hash
digest = sha256(b''.join([
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
self.claim.signing_channel_hash,
self.claim.to_message_bytes()
]))
@@ -128,6 +122,10 @@ class Output(BaseOutput):
self.claim.signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.script.generate()
def clear_signature(self):
self.channel = None
self.claim.clear_signature()
def generate_channel_private_key(self):
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
self.private_key = private_key.to_pem().decode()
@@ -194,15 +192,17 @@ class Transaction(BaseTransaction):
@classmethod
def claim_update(
cls, previous_claim: Output, amount: int, holding_address: str,
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
updated_claim = Output.pay_update_claim_pubkey_hash(
amount, previous_claim.claim_name, previous_claim.claim_id,
previous_claim.claim, ledger.address_to_hash160(holding_address)
claim, ledger.address_to_hash160(holding_address)
)
if signing_channel is not None:
updated_claim.sign(signing_channel, b'placeholder txid:nout')
else:
updated_claim.clear_signature()
return cls.create(
[Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account, sign=False
)
+4
View File
@@ -25,6 +25,9 @@ RUN python3.7 -m pip install --upgrade pip setuptools
# get torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
# get uvloop
RUN python3.7 -m pip install --user uvloop
# copy lbrynet
RUN mkdir projects/
COPY . projects/lbry
@@ -46,4 +49,5 @@ ENV DB_DIRECTORY=$db_dir
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
ENV MAX_SESSIONS=1000000000
ENV MAX_SEND=1000000000000000000
ENV EVENT_LOOP_POLICY=uvloop
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
+2 -2
View File
@@ -5,7 +5,7 @@ import socket
import ipaddress
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
import logging
@@ -32,7 +32,7 @@ async def main(blob_hash: str, url: str):
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
await storage.open()
await blob_manager.setup()
+418 -7
View File
@@ -2,7 +2,380 @@ import os
import re
import json
import inspect
from lbrynet.extras.daemon.Daemon import Daemon
import tempfile
import asyncio
import time
from docopt import docopt
from binascii import unhexlify
from textwrap import indent
from lbrynet.extras.cli import set_kwargs, get_argument_parser
from lbrynet.extras.daemon.Daemon import (
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
)
from lbrynet.extras.daemon.json_response_encoder import (
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
)
from lbrynet.testcase import CommandTestCase
RETURN_DOCS = {
'Account': encode_account_doc(),
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
'Address': 'an address in base58'
}
class ExampleRecorder:
def __init__(self, test):
self.test = test
self.examples = {}
async def __call__(self, title, *command):
parser = get_argument_parser()
args, command_args = parser.parse_known_args(command)
api_method_name = args.api_method_name
parsed = docopt(args.doc, command_args)
kwargs = set_kwargs(parsed)
for k, v in kwargs.items():
if v and isinstance(v, str) and (v[0], v[-1]) == ('"', '"'):
kwargs[k] = v[1:-1]
params = json.dumps({"method": api_method_name, "params": kwargs})
method = getattr(self.test.daemon, f'jsonrpc_{api_method_name}')
result = method(**kwargs)
if asyncio.iscoroutine(result):
result = await result
output = jsonrpc_dumps_pretty(result, ledger=self.test.daemon.ledger)
self.examples.setdefault(api_method_name, []).append({
'title': title,
'curl': f"curl -d'{params}' http://localhost:5279/",
'lbrynet': 'lbrynet ' + ' '.join(command),
'python': f'requests.post("http://localhost:5279", json={params}).json()',
'output': output.strip()
})
return json.loads(output)['result']
class Examples(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.recorder = ExampleRecorder(self)
async def play(self):
r = self.recorder
# general sdk
await r(
'Get status',
'status'
)
await r(
'Get version',
'version'
)
await r(
'Get settings',
'settings', 'get'
)
# accounts
await r(
'List your accounts',
'account', 'list'
)
account = await r(
'Create an account',
'account', 'create', '"generated account"'
)
await r(
'Remove an account',
'account', 'remove', account['id']
)
await r(
'Add an account from seed',
'account', 'add', '"new account"', f"--seed=\"{account['seed']}\""
)
await r(
'Modify maximum number of times a change address can be reused',
'account', 'set', account['id'], '--change_max_uses=10'
)
# addresses
await r(
'List addresses in default account',
'address', 'list'
)
an_address = await r(
'Get an unused address',
'address', 'unused'
)
address_list_by_id = await r(
'List addresses in specified account',
'address', 'list', f"--account_id=\"{account['id']}\""
)
await r(
'Check if address is mine',
'address', 'is_mine', an_address
)
# sends/funds
transfer = await r(
'Transfer 2 LBC from default account to specific account',
'account', 'fund', f"--to_account=\"{account['id']}\"", "--amount=2.0", "--broadcast"
)
await self.on_transaction_dict(transfer)
await self.generate(1)
await self.on_transaction_dict(transfer)
await r(
'Get default account balance',
'account', 'balance'
)
txlist = await r(
'List your transactions',
'transaction', 'list'
)
await r(
'Get balance for specific account by id',
'account', 'balance', f"\"{account['id']}\""
)
spread_transaction = await r(
'Spread LBC between multiple addresses',
'account', 'fund', f"--to_account=\"{account['id']}\"", f"--from_account=\"{account['id']}\"", '--amount=1.5', '--outputs=2', '--broadcast'
)
await self.on_transaction_dict(spread_transaction)
await self.generate(1)
await self.on_transaction_dict(spread_transaction)
await r(
'Transfer all LBC to a specified account',
'account', 'fund', f"--from_account=\"{account['id']}\"", "--everything", "--broadcast"
)
# channels
channel = await r(
'Create a channel claim without metadata',
'channel', 'create', '@channel', '1.0'
)
channel_id = channel['outputs'][0]['claim_id']
await self.on_transaction_dict(channel)
await self.generate(1)
await self.on_transaction_dict(channel)
await r(
'List your channel claims',
'channel', 'list'
)
await r(
'Paginate your channel claims',
'channel', 'list', '--page=1', '--page_size=20'
)
channel = await r(
'Update a channel claim',
'channel', 'update', channel['outputs'][0]['claim_id'], '--title="New Channel"'
)
await self.on_transaction_dict(channel)
await self.generate(1)
await self.on_transaction_dict(channel)
big_channel = await r(
'Create a channel claim with all metadata',
'channel', 'create', '@bigchannel', '1.0',
'--title="Big Channel"', '--description="A channel with lots of videos."',
'--email="creator@smallmedia.com"', '--tags=music', '--tags=art',
'--languages=pt-BR', '--languages=uk', '--locations=BR', '--locations=UA::Kiyv',
'--website_url="http://smallmedia.com"', '--thumbnail_url="http://smallmedia.com/logo.jpg"',
'--cover_url="http://smallmedia.com/logo.jpg"'
)
await self.on_transaction_dict(big_channel)
await self.generate(1)
await self.on_transaction_dict(big_channel)
await self.daemon.jsonrpc_channel_abandon(big_channel['outputs'][0]['claim_id'])
await self.generate(1)
# stream claims
with tempfile.NamedTemporaryFile() as file:
file.write(b'hello world')
file.flush()
stream = await r(
'Create a stream claim without metadata',
'stream', 'create', 'astream', '1.0', file.name
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
stream_id = stream['outputs'][0]['claim_id']
stream_name = stream['outputs'][0]['name']
stream = await r(
'Update a stream claim to add channel',
'stream', 'update', stream_id,
f'--channel_id="{channel_id}"'
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
await r(
'List all your claims',
'claim', 'list'
)
await r(
'Paginate your claims',
'claim', 'list', '--page=1', '--page_size=20'
)
await r(
'List all your stream claims',
'stream', 'list'
)
await r(
'Paginate your stream claims',
'stream', 'list', '--page=1', '--page_size=20'
)
await r(
'Search for all claims in channel',
'claim', 'search', f'--channel_id="{channel_id}"'
)
await r(
'Search for claims matching a name',
'claim', 'search', f'--name="{stream_name}"'
)
with tempfile.NamedTemporaryFile(suffix='.png') as file:
file.write(unhexlify(
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
b'9454e44ae426082'
))
file.flush()
big_stream = await r(
'Create an image stream claim with all metadata and fee',
'stream', 'create', 'blank-image', '1.0', file.name,
'--tags=blank', '--tags=art', '--languages=en', '--locations=US:NH:Manchester',
'--fee_currency=LBC', '--fee_amount=0.3',
'--title="Blank Image"', '--description="A blank PNG that is 5x7."', '--author=Picaso',
'--license="Public Domain"', '--license_url=http://public-domain.org',
'--thumbnail_url="http://smallmedia.com/thumbnail.jpg"', f'--release_time={int(time.time())}',
f'--channel_id="{channel_id}"'
)
await self.on_transaction_dict(big_stream)
await self.generate(1)
await self.on_transaction_dict(big_stream)
await self.daemon.jsonrpc_channel_abandon(big_stream['outputs'][0]['claim_id'])
await self.generate(1)
# files
file_list_result = await r(
'List local files',
'file', 'list'
)
file_uri = f"{file_list_result[0]['claim_name']}#{file_list_result[0]['claim_id']}"
await r(
'Resolve a claim',
'resolve', file_uri
)
await r(
'List files matching a parameter',
'file', 'list', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
)
await r(
'Delete a file',
'file', 'delete', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
)
await r(
'Get a file',
'get', file_uri
)
await r(
'Save a file to the downloads directory',
'file', 'save', f"--sd_hash=\"{file_list_result[0]['sd_hash']}\""
)
# blobs
bloblist = await r(
'List your local blobs',
'blob', 'list'
)
await r(
'Delete a blob',
'blob', 'delete', f"{bloblist[0]}"
)
# abandon all the things
abandon_stream = await r(
'Abandon a stream claim',
'stream', 'abandon', stream_id
)
await self.on_transaction_dict(abandon_stream)
await self.generate(1)
await self.on_transaction_dict(abandon_stream)
abandon_channel = await r(
'Abandon a channel claim',
'channel', 'abandon', channel_id
)
await self.on_transaction_dict(abandon_channel)
await self.generate(1)
await self.on_transaction_dict(abandon_channel)
with tempfile.NamedTemporaryFile() as file:
file.write(b'hello world')
file.flush()
stream = await r(
'Publish a file',
'publish', 'a-new-stream', '--bid=1.0', f'--file_path={file.name}'
)
await self.on_transaction_dict(stream)
await self.generate(1)
await self.on_transaction_dict(stream)
def get_examples():
player = Examples('play')
result = player.run()
if result.errors:
for error in result.errors:
print(error[1])
raise Exception('See above for errors while running the examples.')
return player.recorder.examples
SECTIONS = re.compile("(.*?)Usage:(.*?)Options:(.*?)Returns:(.*)", re.DOTALL)
@@ -11,7 +384,27 @@ ARGUMENT_NAME = re.compile("--([^=]+)")
ARGUMENT_TYPE = re.compile("\s*\((.*?)\)(.*)")
def get_api(obj):
def get_return_def(returns):
result = returns.strip()
if (result[0], result[-1]) == ('{', '}'):
obj_type = result[1:-1]
if '[' in obj_type:
sub_type = obj_type[obj_type.index('[')+1:-1]
obj_type = obj_type[:obj_type.index('[')]
if obj_type == 'Paginated':
obj_def = encode_pagination_doc(RETURN_DOCS[sub_type])
elif obj_type == 'List':
obj_def = [RETURN_DOCS[sub_type]]
else:
raise NameError(f'Unknown return type: {obj_type}')
else:
obj_def = RETURN_DOCS[obj_type]
return indent(json.dumps(obj_def, indent=4), ' '*12)
return result
def get_api(name, examples):
obj = Daemon.callable_methods[name]
docstr = inspect.getdoc(obj).strip()
try:
@@ -45,17 +438,35 @@ def get_api(obj):
arg['description'] = ' '.join(arg['description'])
return {
'name': obj.__name__[len('jsonrpc_'):],
'name': name,
'description': description.strip(),
'arguments': arguments,
'returns': returns.strip()
'returns': get_return_def(returns),
'examples': examples
}
def write_api(f):
apis = []
for method_name in sorted(Daemon.callable_methods.keys()):
apis.append(get_api(Daemon.callable_methods[method_name]))
examples = get_examples()
api_definitions = Daemon.get_api_definitions()
apis = {
'main': {
'doc': 'Ungrouped commands.',
'commands': []
}
}
for group_name, group_doc in api_definitions['groups'].items():
apis[group_name] = {
'doc': group_doc,
'commands': []
}
for method_name, command in api_definitions['commands'].items():
if 'replaced_by' in command:
continue
apis[command['group'] or 'main']['commands'].append(get_api(
method_name,
examples.get(method_name, [])
))
json.dump(apis, f, indent=4)
+2 -2
View File
@@ -1,7 +1,7 @@
import sys
import os
import asyncio
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.schema.address import decode_address
from lbrynet.extras.daemon.storage import SQLiteStorage
@@ -17,7 +17,7 @@ async def main(address: str):
storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite"))
await storage.open()
blob_manager = BlobFileManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
await blob_manager.setup()
server = await loop.create_server(
+42 -32
View File
@@ -11,7 +11,7 @@ from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.conf import Config
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.client import daemon_rpc
from lbrynet.extras import system_info, cli
from lbrynet.extras import system_info
def extract_uris(response):
@@ -58,7 +58,7 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, uri):
async def wait_for_done(conf, uri, timeout):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
@@ -73,11 +73,11 @@ async def wait_for_done(conf, uri):
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > 10:
if hang_count > timeout:
return False, file['blobs_completed'], file['blobs_in_stream']
async def main(uris=None, allow_fees=False):
async def main(uris=None, cmd_args=None):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
@@ -90,15 +90,16 @@ async def main(uris=None, allow_fees=False):
print("**********************************************")
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', name)
async def __resolve(name):
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
if 'error' not in resolved.get(name, {}):
if ("fee" not in resolved[name]['claim']['value']['stream']['metadata']) or allow_fees:
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
await asyncio.gather(*(__resolve(name) for name in uris))
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
first_byte_times = []
@@ -113,24 +114,29 @@ async def main(uris=None, allow_fees=False):
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri)
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except:
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, uri, cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except Exception as e:
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
failed_to_start.append(uri)
return
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
if cmd_args.exit_on_error:
return
if cmd_args.delete_after_download or cmd_args.head_blob_only:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
@@ -139,12 +145,13 @@ async def main(uris=None, allow_fees=False):
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
f"Variance: {variance(first_byte_times)}\n"
if not cmd_args.head_blob_only:
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
@@ -159,9 +166,12 @@ async def main(uris=None, allow_fees=False):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir")
parser.add_argument("--wallet_dir")
parser.add_argument("--download_directory")
#parser.add_argument("--data_dir")
#parser.add_argument("--wallet_dir")
#parser.add_argument("--download_directory")
parser.add_argument("--allow_fees", action='store_true')
args = parser.parse_args()
asyncio.run(main(allow_fees=args.allow_fees))
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=10, type=int)
parser.add_argument("--delete_after_download", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
asyncio.run(main(cmd_args=parser.parse_args()))
+1 -1
View File
@@ -23,6 +23,7 @@ setup(
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
},
install_requires=[
'torba',
'aiohttp==3.5.4',
'aioupnp',
'appdirs==1.4.3',
@@ -35,7 +36,6 @@ setup(
'protobuf==3.6.1',
'msgpack==0.6.1',
'ecdsa==0.13',
'torba',
'pyyaml==4.2b1',
'docopt==0.6.2',
'hachoir',
+1
View File
@@ -61,6 +61,7 @@ def mock_network_loop(loop: asyncio.BaseEventLoop):
protocol = proto_lam()
transport = asyncio.DatagramTransport(extra={'socket': mock_sock})
transport.is_closing = lambda: False
transport.close = lambda: mock_sock.close()
mock_sock.sendto = sendto
transport.sendto = mock_sock.sendto
+1 -1
View File
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class AccountManagement(CommandTestCase):
+4 -4
View File
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class EpicAdventuresOfChris45(CommandTestCase):
@@ -92,12 +92,12 @@ class EpicAdventuresOfChris45(CommandTestCase):
# After abandoning he just waits for his LBCs to be returned to his account
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.969379')
self.assertEqual(result, '8.9693455')
# Amidst all this Chris receives a call from his friend Ramsey
# who says that it is of utmost urgency that Chris transfer him
# 1 LBC to which Chris readily obliges
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id']
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address))
self.assertIn("txid", result)
@@ -108,7 +108,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
# Chris' balance was correct
self.assertEqual(result, '7.969255')
self.assertEqual(result, '7.9692215')
# Ramsey too assured him that he had received the 1 LBC and thanks him
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
+349 -116
View File
@@ -1,6 +1,9 @@
import os.path
import hashlib
import tempfile
import logging
from binascii import unhexlify
from urllib.request import urlopen
import ecdsa
@@ -8,10 +11,13 @@ from lbrynet.wallet.transaction import Transaction, Output
from torba.client.errors import InsufficientFundsError
from lbrynet.schema.compat import OldClaimMessage
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
from torba.client.hash import sha256, Base58
log = logging.getLogger(__name__)
class ChannelCommands(CommandTestCase):
async def test_create_channel_names(self):
@@ -69,75 +75,65 @@ class ChannelCommands(CommandTestCase):
async def test_setting_channel_fields(self):
values = {
'tags': ["cool", "awesome"],
'title': "Cool Channel",
'description': "Best channel on LBRY.",
'thumbnail_url': "https://co.ol/thumbnail.png",
'tags': ["cool", "awesome"],
'languages': ["en-US"],
'locations': ['US::Manchester'],
'contact_email': "human@email.com",
'homepage_url': "https://co.ol",
'email': "human@email.com",
'website_url': "https://co.ol",
'cover_url': "https://co.ol/cover.png",
'featured': ['cafe']
}
fixed_values = values.copy()
fixed_values['languages'] = ['en-US']
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
fixed_values['locations'] = [{'country': 'US', 'city': 'Manchester'}]
fixed_values['cover'] = {'url': fixed_values.pop('cover_url')}
# create new channel with all fields set
tx = await self.out(self.channel_create('@bigchannel', **values))
txo = tx['outputs'][0]
self.assertEqual(
txo['value']['channel'],
{'public_key': txo['value']['channel']['public_key'], **fixed_values}
)
channel = tx['outputs'][0]['value']
self.assertEqual(channel, {'public_key': channel['public_key'], **fixed_values})
# create channel with nothing set
tx = await self.out(self.channel_create('@lightchannel'))
txo = tx['outputs'][0]
self.assertEqual(
txo['value']['channel'],
{'public_key': txo['value']['channel']['public_key']}
)
channel = tx['outputs'][0]['value']
self.assertEqual(channel, {'public_key': channel['public_key']})
# create channel with just some tags
tx = await self.out(self.channel_create('@updatedchannel', tags='blah'))
# create channel with just a featured claim
tx = await self.out(self.channel_create('@featurechannel', featured='beef'))
txo = tx['outputs'][0]
claim_id = txo['claim_id']
public_key = txo['value']['channel']['public_key']
self.assertEqual(
txo['value']['channel'],
{'public_key': public_key, 'tags': ['blah']}
)
claim_id, channel = txo['claim_id'], txo['value']
fixed_values['public_key'] = channel['public_key']
self.assertEqual(channel, {'public_key': fixed_values['public_key'], 'featured': ['beef']})
# update channel setting all fields
tx = await self.out(self.channel_update(claim_id, **values))
txo = tx['outputs'][0]
fixed_values['public_key'] = public_key
fixed_values['tags'].insert(0, 'blah') # existing tag
self.assertEqual(
txo['value']['channel'],
fixed_values
)
channel = tx['outputs'][0]['value']
fixed_values['featured'].insert(0, 'beef') # existing featured claim
self.assertEqual(channel, fixed_values)
# clearing and settings tags
tx = await self.out(self.channel_update(claim_id, tags='single', clear_tags=True))
txo = tx['outputs'][0]
fixed_values['tags'] = ['single']
self.assertEqual(
txo['value']['channel'],
fixed_values
)
# clearing and settings featured content
tx = await self.out(self.channel_update(claim_id, featured='beefcafe', clear_featured=True))
channel = tx['outputs'][0]['value']
fixed_values['featured'] = ['beefcafe']
self.assertEqual(channel, fixed_values)
# reset signing key
tx = await self.out(self.channel_update(claim_id, new_signing_key=True))
txo = tx['outputs'][0]
self.assertNotEqual(
txo['value']['channel']['public_key'],
fixed_values['public_key']
channel = tx['outputs'][0]['value']
self.assertNotEqual(channel['public_key'], fixed_values['public_key'])
# replace mode (clears everything except public_key)
tx = await self.out(self.channel_update(claim_id, replace=True, title='foo', email='new@email.com'))
self.assertEqual(
tx['outputs'][0]['value'],
{'public_key': channel['public_key'], 'title': 'foo', 'email': 'new@email.com'}
)
# send channel to someone else
new_account = await self.daemon.jsonrpc_account_create('second account')
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
# before sending
@@ -166,6 +162,19 @@ class ChannelCommands(CommandTestCase):
class StreamCommands(CommandTestCase):
files_directory = os.path.join(os.path.dirname(__file__), 'files')
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
def setUp(self):
if not os.path.exists(self.video_file_name):
if not os.path.exists(self.files_directory):
os.mkdir(self.files_directory)
log.info(f'downloading test video from {self.video_file_name}')
with urlopen(self.video_file_url) as response,\
open(self.video_file_name, 'wb') as video_file:
video_file.write(response.read())
async def test_create_stream_names(self):
# claim new name
await self.stream_create('foo')
@@ -205,24 +214,24 @@ class StreamCommands(CommandTestCase):
tx = await self.stream_update(claim_id, bid='3.0')
self.assertEqual(tx['outputs'][0]['amount'], '3.0')
await self.assertBalance(self.account, '6.993384')
await self.assertBalance(self.account, '6.993319')
# not enough funds
with self.assertRaisesRegex(
InsufficientFundsError, "Not enough funds to cover this transaction."):
await self.stream_create('foo2', '9.0')
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
await self.assertBalance(self.account, '6.993384')
await self.assertBalance(self.account, '6.993319')
# spend exactly amount available, no change
tx = await self.stream_create('foo3', '6.98527700')
tx = await self.stream_create('foo3', '6.98523')
await self.assertBalance(self.account, '0.0')
self.assertEqual(len(tx['outputs']), 1) # no change
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
async def test_publishing_checks_all_accounts_for_channel(self):
account1_id, account1 = self.account.id, self.account
new_account = await self.daemon.jsonrpc_account_create('second account')
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
await self.out(self.channel_create('@spam', '1.0'))
@@ -249,25 +258,66 @@ class StreamCommands(CommandTestCase):
self.assertEqual(channels[0]['name'], '@baz')
# defaults to using all accounts to lookup channel
await self.stream_create('hovercraft1', channel_id=baz_id)
await self.stream_create('hovercraft1', '0.1', channel_id=baz_id)
self.assertEqual((await self.claim_search('hovercraft1'))[0]['channel_name'], '@baz')
# uses only the specific accounts which contains the channel
await self.stream_create('hovercraft2', channel_id=baz_id, channel_account_id=[account2_id])
# lookup by channel_name in all accounts
await self.stream_create('hovercraft2', '0.1', channel_name='@baz')
self.assertEqual((await self.claim_search('hovercraft2'))[0]['channel_name'], '@baz')
# uses only the specific accounts which contains the channel
await self.stream_create('hovercraft3', '0.1', channel_id=baz_id, channel_account_id=[account2_id])
self.assertEqual((await self.claim_search('hovercraft3'))[0]['channel_name'], '@baz')
# lookup by channel_name in specific account
await self.stream_create('hovercraft4', '0.1', channel_name='@baz', channel_account_id=[account2_id])
self.assertEqual((await self.claim_search('hovercraft4'))[0]['channel_name'], '@baz')
# fails when specifying account which does not contain channel
with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_id"):
await self.stream_create(
'hovercraft3', channel_id=baz_id, channel_account_id=[account1_id]
'hovercraft5', '0.1', channel_id=baz_id, channel_account_id=[account1_id]
)
# fail with channel_name
with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_name '@baz'"):
await self.stream_create(
'hovercraft5', '0.1', channel_name='@baz', channel_account_id=[account1_id]
)
async def test_preview_works_with_signed_streams(self):
await self.out(self.channel_create('@spam', '1.0'))
signed = await self.out(self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False))
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
async def test_publish_updates_file_list(self):
tx = await self.out(self.stream_create(title='created'))
txo = tx['outputs'][0]
claim_id, expected = txo['claim_id'], txo['value']
files = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(1, len(files))
self.assertEqual(tx['txid'], files[0]['txid'])
self.assertEqual(expected, files[0]['metadata'])
# update with metadata-only changes
tx = await self.out(self.stream_update(claim_id, title='update 1'))
files = self.sout(self.daemon.jsonrpc_file_list())
expected['title'] = 'update 1'
self.assertEqual(1, len(files))
self.assertEqual(tx['txid'], files[0]['txid'])
self.assertEqual(expected, files[0]['metadata'])
# update with new data
tx = await self.out(self.stream_update(claim_id, title='update 2', data=b'updated data'))
expected = tx['outputs'][0]['value']
files = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(1, len(files))
self.assertEqual(tx['txid'], files[0]['txid'])
self.assertEqual(expected, files[0]['metadata'])
async def test_setting_stream_fields(self):
values = {
'tags': ["cool", "awesome"],
'title': "Cool Content",
'description': "Best content on LBRY.",
'thumbnail_url': "https://co.ol/thumbnail.png",
'tags': ["cool", "awesome"],
'languages': ["en"],
'locations': ['{"country": "UA"}'],
'locations': ['US:NH:Manchester:03101:42.990605:-71.460989'],
'author': "Jules Verne",
'license': 'Public Domain',
@@ -277,86 +327,253 @@ class StreamCommands(CommandTestCase):
'fee_currency': 'usd',
'fee_amount': '2.99',
'fee_address': 'mmCsWAiXMUVecFQ3fVzUwvpT9XFMXno2Ca',
'video_width': 800,
'video_height': 600
}
fixed_values = values.copy()
fixed_values['languages'] = ['en']
fixed_values['locations'] = [{'country': 'UA'}]
# create new channel with all fields set
tx = await self.out(self.stream_create('big', **values))
txo = tx['outputs'][0]
stream = txo['value']['stream']
fixed_values['sd_hash'] = stream['sd_hash']
fixed_values['file'] = stream['file']
fixed_values['media_type'] = 'application/octet-stream'
fixed_values['locations'] = [{
'country': 'US',
'state': 'NH',
'city': 'Manchester',
'code': '03101',
'latitude': '42.990605',
'longitude': '-71.460989'
}]
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
fixed_values['release_time'] = str(values['release_time'])
fixed_values['source'] = {
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
'media_type': 'application/octet-stream',
'size': '3'
}
fixed_values['fee'] = {
'address': fixed_values.pop('fee_address'),
'amount': float(fixed_values.pop('fee_amount')),
'amount': fixed_values.pop('fee_amount'),
'currency': fixed_values.pop('fee_currency').upper()
}
fixed_values['video'] = {
'height': fixed_values.pop('video_height'),
'width': fixed_values.pop('video_width')
}
# create new stream with all fields set
tx = await self.out(self.stream_create('big', **values))
stream = tx['outputs'][0]['value']
fixed_values['source']['name'] = stream['source']['name']
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
self.assertEqual(stream, fixed_values)
# create channel with nothing set
# create stream with nothing set
tx = await self.out(self.stream_create('light'))
txo = tx['outputs'][0]
stream = tx['outputs'][0]['value']
self.assertEqual(
txo['value']['stream'], {
'file': {'size': '3'},
'media_type': 'application/octet-stream',
'sd_hash': txo['value']['stream']['sd_hash']
stream, {
'source': {
'size': '3',
'media_type': 'application/octet-stream',
'name': stream['source']['name'],
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
'sd_hash': stream['source']['sd_hash']
},
}
)
# create channel with just some tags
tx = await self.out(self.stream_create('updated', tags='blah'))
# create stream with just some tags, langs and locations
tx = await self.out(self.stream_create('updated', tags='blah', languages='uk', locations='UA::Kyiv'))
txo = tx['outputs'][0]
claim_id = txo['claim_id']
fixed_values['sd_hash'] = txo['value']['stream']['sd_hash']
claim_id, stream = txo['claim_id'], txo['value']
fixed_values['source']['name'] = stream['source']['name']
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
self.assertEqual(
txo['value']['stream'], {
'file': {'size': '3'},
'media_type': 'application/octet-stream',
'sd_hash': fixed_values['sd_hash'],
'tags': ['blah']
stream, {
'source': {
'size': '3',
'media_type': 'application/octet-stream',
'name': fixed_values['source']['name'],
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
'sd_hash': fixed_values['source']['sd_hash'],
},
'tags': ['blah'],
'languages': ['uk'],
'locations': [{'country': 'UA', 'city': 'Kyiv'}]
}
)
# update channel setting all fields
# update stream setting all fields, 'source' doesn't change
tx = await self.out(self.stream_update(claim_id, **values))
txo = tx['outputs'][0]
stream = tx['outputs'][0]['value']
fixed_values['tags'].insert(0, 'blah') # existing tag
self.assertEqual(txo['value']['stream'], fixed_values)
fixed_values['languages'].insert(0, 'uk') # existing language
fixed_values['locations'].insert(0, {'country': 'UA', 'city': 'Kyiv'}) # existing location
self.assertEqual(stream, fixed_values)
# clearing and settings tags
tx = await self.out(self.stream_update(claim_id, tags='single', clear_tags=True))
# clearing and settings tags, languages and locations
tx = await self.out(self.stream_update(
claim_id, tags='single', clear_tags=True,
languages='pt', clear_languages=True,
locations='BR', clear_locations=True,
))
txo = tx['outputs'][0]
fixed_values['tags'] = ['single']
self.assertEqual(txo['value']['stream'], fixed_values)
fixed_values['languages'] = ['pt']
fixed_values['locations'] = [{'country': 'BR'}]
self.assertEqual(txo['value'], fixed_values)
# clearing fee
tx = await self.out(self.stream_update(claim_id, clear_fee=True))
txo = tx['outputs'][0]
del fixed_values['fee']
self.assertEqual(txo['value'], fixed_values)
# modifying hash/size/name
fixed_values['source']['name'] = 'changed_name'
fixed_values['source']['hash'] = 'cafebeef'
fixed_values['source']['size'] = '42'
tx = await self.out(self.stream_update(
claim_id, file_name='changed_name', file_hash='cafebeef', file_size=42
))
self.assertEqual(tx['outputs'][0]['value'], fixed_values)
# stream_update re-signs with the same channel
channel_id = (await self.channel_create('@chan'))['outputs'][0]['claim_id']
tx = await self.stream_update(claim_id, channel_id=channel_id)
self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan')
tx = await self.stream_update(claim_id, title='channel re-signs')
self.assertEqual(tx['outputs'][0]['value']['title'], 'channel re-signs')
self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan')
# send claim to someone else
new_account = await self.daemon.jsonrpc_account_create('second account')
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
# before sending
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 0)
other_address = await account2.receiving.get_or_create_usable_address()
tx = await self.out(self.stream_update(claim_id, claim_address=other_address))
# after sending
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 3)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
async def test_create_update_and_abandon_claim(self):
async def test_automatic_type_and_metadata_detection_for_image(self):
with tempfile.NamedTemporaryFile(suffix='.png') as file:
file.write(unhexlify(
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
b'9454e44ae426082'
))
file.flush()
tx = await self.out(
self.daemon.jsonrpc_stream_create(
'blank-image', '1.0', file_path=file.name
)
)
txo = tx['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {
'size': '99',
'name': os.path.basename(file.name),
'media_type': 'image/png',
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
'sd_hash': txo['value']['source']['sd_hash'],
},
'stream_type': 'image',
'image': {
'width': 5,
'height': 7
}
}
)
async def test_automatic_type_and_metadata_detection_for_video(self):
tx = await self.out(
self.daemon.jsonrpc_stream_create(
'chrome', '1.0', file_path=self.video_file_name
)
)
txo = tx['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {
'size': '2299653',
'name': 'ForBiggerEscapes.mp4',
'media_type': 'video/mp4',
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
'sd_hash': txo['value']['source']['sd_hash'],
},
'stream_type': 'video',
'video': {
'width': 1280,
'height': 720,
'duration': 15
}
}
)
async def test_overriding_automatic_metadata_detection(self):
tx = await self.out(
self.daemon.jsonrpc_stream_create(
'chrome', '1.0', file_path=self.video_file_name, width=99, height=88, duration=9
)
)
txo = tx['outputs'][0]
self.assertEqual(
txo['value'], {
'source': {
'size': '2299653',
'name': 'ForBiggerEscapes.mp4',
'media_type': 'video/mp4',
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
'sd_hash': txo['value']['source']['sd_hash'],
},
'stream_type': 'video',
'video': {
'width': 99,
'height': 88,
'duration': 9
}
}
)
async def test_replace_mode_preserves_source_and_type(self):
expected = {
'tags': ['blah'],
'languages': ['uk'],
'locations': [{'country': 'UA', 'city': 'Kyiv'}],
'source': {
'size': '2299653',
'name': 'ForBiggerEscapes.mp4',
'media_type': 'video/mp4',
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
},
'stream_type': 'video',
'video': {
'width': 1280,
'height': 720,
'duration': 15
}
}
channel = await self.channel_create('@chan')
tx = await self.out(self.daemon.jsonrpc_stream_create(
'chrome', '1.0', file_path=self.video_file_name,
tags='blah', languages='uk', locations='UA::Kyiv',
channel_id=channel['outputs'][0]['claim_id']
))
await self.on_transaction_dict(tx)
txo = tx['outputs'][0]
expected['source']['sd_hash'] = txo['value']['source']['sd_hash']
self.assertEqual(txo['value'], expected)
self.assertEqual(txo['signing_channel']['name'], '@chan')
tx = await self.out(self.daemon.jsonrpc_stream_update(
txo['claim_id'], title='new title', replace=True
))
txo = tx['outputs'][0]
expected['title'] = 'new title'
del expected['tags']
del expected['languages']
del expected['locations']
self.assertEqual(txo['value'], expected)
self.assertNotIn('signing_channel', txo)
async def test_create_update_and_abandon_stream(self):
await self.assertBalance(self.account, '10.0')
tx = await self.stream_create(bid='2.5') # creates new claim
@@ -369,6 +586,10 @@ class StreamCommands(CommandTestCase):
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
await self.assertBalance(self.account, '7.479893')
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
await self.daemon.jsonrpc_file_delete(delete_all=True)
self.assertEqual(0, len(self.daemon.jsonrpc_file_list()))
await self.stream_update(claim_id, bid='1.0') # updates previous claim
txs = await self.out(self.daemon.jsonrpc_transaction_list())
@@ -376,8 +597,8 @@ class StreamCommands(CommandTestCase):
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000184')
await self.assertBalance(self.account, '8.979709')
self.assertEqual(txs[0]['fee'], '-0.0002165')
await self.assertBalance(self.account, '8.9796765')
await self.stream_abandon(claim_id)
txs = await self.out(self.daemon.jsonrpc_transaction_list())
@@ -386,9 +607,9 @@ class StreamCommands(CommandTestCase):
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
await self.assertBalance(self.account, '9.979602')
await self.assertBalance(self.account, '9.9795695')
async def test_abandoning_claim_at_loss(self):
async def test_abandoning_stream_at_loss(self):
await self.assertBalance(self.account, '10.0')
tx = await self.stream_create(bid='0.0001')
await self.assertBalance(self.account, '9.979793')
@@ -410,8 +631,12 @@ class StreamCommands(CommandTestCase):
file.flush()
tx1 = await self.publish('foo', bid='1.0', file_path=file.name)
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
# doesn't error on missing arguments when doing an update stream
tx2 = await self.publish('foo', tags='updated')
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
self.assertEqual(
tx1['outputs'][0]['claim_id'],
tx2['outputs'][0]['claim_id']
@@ -422,26 +647,28 @@ class StreamCommands(CommandTestCase):
with self.assertRaisesRegex(Exception, "There are 2 claims for 'foo'"):
await self.daemon.jsonrpc_publish('foo')
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
# abandon duplicate stream
await self.stream_abandon(tx3['outputs'][0]['claim_id'])
# publish to a channel
await self.channel_create('@abc')
tx3 = await self.publish('foo', channel_name='@abc')
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
r = await self.resolve('lbry://@abc/foo')
self.assertEqual(
r['lbry://@abc/foo']['claim']['claim_id'],
tx3['outputs'][0]['claim_id']
)
# publishing again re-signs with the same channel
# publishing again clears channel
tx4 = await self.publish('foo', languages='uk-UA')
r = await self.resolve('lbry://@abc/foo')
claim = r['lbry://@abc/foo']['claim']
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
r = await self.resolve('lbry://foo')
claim = r['lbry://foo']['claim']
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
self.assertEqual(claim['channel_name'], '@abc')
self.assertEqual(claim['signature_is_valid'], True)
self.assertEqual(claim['value']['stream']['languages'], ['uk-UA'])
self.assertNotIn('channel_name', claim)
self.assertEqual(claim['value']['languages'], ['uk-UA'])
async def test_claim_search(self):
# search for channel claim
@@ -480,6 +707,13 @@ class StreamCommands(CommandTestCase):
claims = await self.claim_search(channel_id=channel_id)
self.assertEqual(len(claims), 3)
# same is expected using name or name#claim_id urls
claims = await self.claim_search(channel_name="@abc")
self.assertEqual(len(claims), 3)
claims = await self.claim_search(channel_name="@abc", channel_id=channel_id)
self.assertEqual(len(claims), 3)
claims = await self.claim_search(channel_name=f"@abc#{channel_id}")
self.assertEqual(len(claims), 3)
await self.stream_abandon(claim_id=claims[0]['claim_id'])
await self.stream_abandon(claim_id=claims[1]['claim_id'])
@@ -527,10 +761,10 @@ class StreamCommands(CommandTestCase):
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
response = await self.resolve('lbry://on-channel-claim')
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
self.assertIs(False, response['lbry://on-channel-claim']['claim']['signature_is_valid'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = await self.resolve(direct_uri)
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
self.assertIs(False, response[direct_uri]['claim']['signature_is_valid'])
await self.stream_abandon(claim_id=orphan_claim_id)
uri = 'lbry://@abc/on-channel-claim'
@@ -571,8 +805,7 @@ class StreamCommands(CommandTestCase):
self.assertNotEqual(r1c, r2c)
await self.generate(50)
head = await self.daemon.jsonrpc_block_show()
self.assertTrue(head['height'] > 250)
self.assertTrue(self.ledger.headers.height > 250)
r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
@@ -591,7 +824,7 @@ class StreamCommands(CommandTestCase):
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
await self.confirm_tx(channel.id)
address = channel.outputs[0].get_address(self.account.ledger)
claim = generate_signed_legacy('example', address, channel.outputs[0])
claim = generate_signed_legacy(address, channel.outputs[0])
tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
await tx.sign([self.account])
await self.broadcast(tx)
@@ -609,12 +842,12 @@ class StreamCommands(CommandTestCase):
await self.confirm_tx(tx.id)
response = await self.daemon.jsonrpc_resolve(urls='bad_example')
self.assertFalse(response['bad_example']['claim']['signature_is_valid'], response)
self.assertIs(False, response['bad_example']['claim']['signature_is_valid'], response)
response = await self.daemon.jsonrpc_resolve(urls='@olds/bad_example')
self.assertEqual('URI lbry://@olds/bad_example cannot be resolved', response['@olds/bad_example']['error'])
def generate_signed_legacy(name: str, address: bytes, output: Output):
def generate_signed_legacy(address: bytes, output: Output):
decoded_address = Base58.decode(address)
claim = OldClaimMessage()
claim.ParseFromString(unhexlify(
@@ -640,14 +873,14 @@ def generate_signed_legacy(name: str, address: bytes, output: Output):
digest = sha256(b''.join([
decoded_address,
claim.SerializeToString(),
output.claim_hash
output.claim_hash[::-1]
]))
private_key = ecdsa.SigningKey.from_pem(output.private_key, hashfunc=hashlib.sha256)
signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
claim.publisherSignature.version = 1
claim.publisherSignature.signatureType = 1
claim.publisherSignature.signature = signature
claim.publisherSignature.certificateId = output.claim_hash
claim.publisherSignature.certificateId = output.claim_hash[::-1]
return claim
@@ -655,7 +888,7 @@ class SupportCommands(CommandTestCase):
async def test_regular_supports_and_tip_supports(self):
# account2 will be used to send tips and supports to account1
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
account2 = self.daemon.get_account_or_error(account2_id)
# send account2 5 LBC out of the 10 LBC in account1
+1 -1
View File
@@ -29,7 +29,7 @@ class CLIIntegrationTest(AsyncioTestCase):
await self.daemon.start()
async def asyncTearDown(self):
await self.daemon.stop()
await self.daemon.stop(shutdown_runner=False)
def test_cli_status_command_with_auth(self):
actual_output = StringIO()
+203
View File
@@ -0,0 +1,203 @@
import logging
import typing
import random
import asyncio
from aiohttp import web
from lbrynet.testcase import CommandTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
class FakedCommentServer:
ERRORS = {
'INVALID_URI': {'code': 1, 'message': 'Invalid claim URI'},
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
self.app.add_routes([web.post('/api', self.api)])
self.runner = None
self.server = None
def get_claim_comments(self, uri: str, better_keys: bool) -> typing.Union[dict, list, None]:
if not uri.startswith('lbry://'): # Very basic error case
return {'error': self.ERRORS['INVALID_URI']}
return [self.get_comment(i) for i in range(75)]
def get_comment(self, comment_id: int, parent_id: int = None) -> dict:
return {
'comment_id': comment_id,
'parent_id': parent_id,
'author': f'Person{comment_id}',
'message': f'comment {comment_id}',
'claim_id': random.randint(1, 2**16),
'time_posted': random.randint(2**16, 2**32 - 1),
'upvotes': random.randint(0, 9999), 'downvotes': random.randint(0, 9999)
}
def comment(self, uri: str, poster: str, message: str) -> typing.Union[int, dict, None]:
if not uri.startswith('lbry://'):
return {'error': self.ERRORS['INVALID_URI']}
return random.randint(1, 9999)
def reply(self, parent_id: int, poster: str, message: str) -> dict:
if 2 <= len(message) <= 2000 and 2 <= len(poster) <= 127 and parent_id > 0:
return random.randint(parent_id + 1, 2**32 - 1)
return {'error': self.ERRORS['INVALID_PARAMS']}
def get_comment_data(self, comm_index: int, better_keys: bool = False) -> typing.Union[dict, None]:
return self.get_comment(comm_index)
def get_comment_replies(self, comm_index: int) -> typing.Union[list, None]:
return [random.randint(comm_index, comm_index+250) for _ in range(75)]
methods = {
'get_claim_comments': get_claim_comments,
'get_comment_data': get_comment_data,
'get_comment_replies': get_comment_replies,
'comment': comment,
'reply': reply
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
if body['method'] in self.methods:
params = body.get('params', {})
result = self.methods[body['method']](self, **params)
if type(result) is dict and 'error' in result:
response['error'] = result['error']
else:
response['result'] = result
else:
response['error'] = self.ERRORS['UNKNOWN']
return response
async def _start(self):
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.server = web.TCPSite(self.runner, 'localhost', self.port)
await self.server.start()
async def _stop(self):
await self.runner.cleanup()
async def run(self, max_timeout=3600):
try:
await self._start()
await asyncio.sleep(max_timeout)
except asyncio.CancelledError:
pass
finally:
await self._stop()
async def api(self, request):
body = await request.json()
if type(body) is list or type(body) is dict:
if type(body) is list:
response = [self.process_json(part) for part in body]
else:
response = self.process_json(body)
return web.json_response(response)
else:
return web.json_response({'error': self.ERRORS['UNKNOWN']})
class CommentCommands(CommandTestCase):
VERBOSITY = logging.WARN
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon.conf.comment_server = 'http://localhost:2903/api'
self.server = FakedCommentServer(2903)
self.server_task = asyncio.create_task(self.server.run(self.timeout))
async def asyncTearDown(self):
await super().asyncTearDown()
self.server_task.cancel()
if not self.server_task.cancelled():
await self.server_task
async def test_comment_create(self):
claim = await self.stream_create(name='doge', bid='0.001', data=b'loool')
self.assertIn('outputs', claim)
comment = await self.daemon.jsonrpc_comment_create(
claim_id=claim['outputs'][0]['claim_id'],
channel_id='Jimmy Buffett',
message="It's 5 O'Clock Somewhere"
)
self.assertIs(type(comment), dict, msg=f"Response type ({type(comment)})is not dict: {comment}")
self.assertIn('message', comment, msg=f"Response {comment} doesn't contain message")
self.assertIn('author', comment)
async def test_comment_create_reply(self):
claim = await self.stream_create(name='doge', bid='0.001')
self.assertIn('outputs', claim)
reply = await self.daemon.jsonrpc_comment_create(
claim_id=claim['outputs'][0]['claim_id'],
channel_id='Jimmy Buffett',
message='Let\'s all go to Margaritaville',
parent_comment_id=42
)
self.assertIs(type(reply), dict, msg=f'Response {type(reply)} is not dict\nResponse: {reply}')
self.assertIn('author', reply)
async def test_comment_list_root_level(self):
claim = await self.stream_create(name='doge', bid='0.001')
self.assertIn('outputs', claim)
claim_id = claim['outputs'][0]['claim_id']
comments = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertIsNotNone(type(comments))
self.assertIs(type(comments), dict)
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
self.assertIs(type(comments['comments']), list, msg=f'comment_list: {comments}')
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50)
self.assertIsNotNone(comments)
self.assertIs(type(comments), dict)
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
comment_list = comments['comments']
self.assertEqual(len(comment_list), 50, msg=f'comment_list incorrect size {len(comment_list)}: {comment_list}')
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=2)
self.assertEqual(len(comments['comments']), 25, msg=f'comment list page 2: {comments["comments"]}')
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=3)
self.assertEqual(len(comments['comments']), 0, msg=f'comment list is non-zero: {comments["comments"]}')
async def test_comment_list_replies(self):
claim = await self.stream_create(name='doge', bid='0.001')
self.assertIn('outputs', claim)
claim_id = claim['outputs'][0]['claim_id']
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23)
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, page_size=50))['comments']
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
page_size=23, page=5))['comments']
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
page_size=60, page=2))['comments']
self.assertEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
async def test_comment_list_flatness_flatness_LA(self):
claim = await self.stream_create(name='doge', bid='0.001')
self.assertIn('outputs', claim)
claim_id = claim['outputs'][0]['claim_id']
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23, flat=True)
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, flat=True,
max_replies_shown=0, page_size=50))['comments']
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
flat=True, page_size=23, page=5))['comments']
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
page_size=60, page=2))['comments']
self.assertGreaterEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
+107 -38
View File
@@ -1,8 +1,10 @@
import asyncio
import logging
import os
from binascii import unhexlify, hexlify
from integration.testcase import CommandTestCase
from lbrynet.schema import Claim
from lbrynet.testcase import CommandTestCase
from lbrynet.blob_exchange.downloader import BlobDownloader
@@ -14,7 +16,7 @@ class FileCommands(CommandTestCase):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
file1, file2 = self.sout(self.daemon.jsonrpc_file_list('claim_name'))
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
@@ -26,30 +28,91 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_get('lbry://foo')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
async def test_announces(self):
# announces on publish
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
await self.stream_create('foo', '0.01')
stream = self.daemon.jsonrpc_file_list()[0]
self.assertSetEqual(
set(await self.daemon.storage.get_blobs_to_announce()),
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
)
self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True))
# announces on download
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
stream = await self.daemon.jsonrpc_get('foo')
self.assertSetEqual(
set(await self.daemon.storage.get_blobs_to_announce()),
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
)
async def test_file_list_fields(self):
await self.stream_create('foo', '0.01')
file_list = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(
file_list[0]['timestamp'],
None
)
self.assertEqual(file_list[0]['confirmations'], -1)
await self.daemon.jsonrpc_resolve('foo')
file_list = self.sout(self.daemon.jsonrpc_file_list())
self.assertEqual(
file_list[0]['timestamp'],
self.ledger.headers[file_list[0]['height']]['timestamp']
)
self.assertEqual(file_list[0]['confirmations'], 1)
async def test_get_doesnt_touch_user_written_files_between_calls(self):
await self.stream_create('foo', '0.01', data=bytes([0] * (2 << 23)))
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
first_path = (await self.daemon.jsonrpc_get('lbry://foo', save_file=True)).full_path
await self.wait_files_to_complete()
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
with open(first_path, 'wb') as f:
f.write(b' ')
f.flush()
second_path = await self.daemon.jsonrpc_get('lbry://foo', save_file=True)
await self.wait_files_to_complete()
self.assertNotEquals(first_path, second_path)
async def test_file_list_updated_metadata_on_resolve(self):
await self.stream_create('foo', '0.01')
claim = await self.daemon.resolve('lbry://foo')
claim = claim['lbry://foo']['claim']['protobuf'].decode()
await self.daemon.jsonrpc_file_delete(claim_name='foo')
txid = await self.blockchain_claim_name('bar', claim, '0.01')
await self.daemon.jsonrpc_get('lbry://bar')
claim = Claim.from_bytes(unhexlify(claim))
claim.stream.description = "fix typos, fix the world"
await self.blockchain_update_name(txid, hexlify(claim.to_bytes()).decode(), '0.01')
await self.daemon.jsonrpc_resolve('lbry://bar')
file_list = self.daemon.jsonrpc_file_list()
self.assertEqual(file_list[0].stream_claim_info.claim.stream.description, claim.stream.description)
async def test_download_different_timeouts(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
all_except_sd = [
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
]
await self.server.blob_manager.delete_blobs(all_except_sd)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didnt create a file")
await self.server.blob_manager.delete_blobs([sd_hash])
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
self.assertIn('error', resp)
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
self.assertEqual('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
async def wait_files_to_complete(self):
while self.daemon.jsonrpc_file_list(status='running'):
while self.sout(self.daemon.jsonrpc_file_list(status='running')):
await asyncio.sleep(0.01)
async def test_filename_conflicts_management_on_resume_download(self):
await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
file_info = self.daemon.jsonrpc_file_list()[0]
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.daemon.jsonrpc_get('lbry://foo')
@@ -59,43 +122,42 @@ class FileCommands(CommandTestCase):
await self.daemon.stream_manager.start()
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
# check that internal state got through up to the file list API
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
file_info = self.daemon.jsonrpc_file_list()[0]
self.assertEqual(downloader.output_file_name, file_info['file_name'])
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
file_info = self.sout(self.daemon.jsonrpc_file_list()[0])
self.assertEqual(stream.file_name, file_info['file_name'])
# checks if what the API shows is what he have at the very internal level.
self.assertEqual(downloader.output_path, file_info['download_path'])
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
self.assertEqual(stream.full_path, file_info['download_path'])
async def test_incomplete_downloads_erases_output_file_on_stop(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
file_info = self.daemon.jsonrpc_file_list()[0]
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
blob.blob_hash for blob in blobs[1:-1]
]
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
self.assertFalse(os.path.isfile(path))
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
self.assertNotIn('error', resp)
self.assertTrue(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
self.assertTrue(os.path.isfile(path))
self.daemon.stream_manager.stop()
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
self.assertFalse(os.path.isfile(path))
async def test_incomplete_downloads_retry(self):
tx = await self.stream_create('foo', '0.01')
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
blob.blob_hash for blob in blobs[1:-1]
]
# backup server blobs
@@ -107,10 +169,10 @@ class FileCommands(CommandTestCase):
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
# start the download
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
self.assertNotIn('error', resp)
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
self.assertEqual('running', self.daemon.jsonrpc_file_list()[0]['status'])
self.assertEqual('running', self.sout(self.daemon.jsonrpc_file_list())[0]['status'])
await self.daemon.jsonrpc_file_set_status('stop', claim_name='foo')
# recover blobs
@@ -122,14 +184,14 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_file_set_status('start', claim_name='foo')
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
file_info = self.daemon.jsonrpc_file_list()[0]
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
self.assertEqual('finished', file_info['status'])
async def test_unban_recovers_stream(self):
BlobDownloader.BAN_TIME = .5 # fixme: temporary field, will move to connection manager or a conf
BlobDownloader.BAN_FACTOR = .5 # fixme: temporary field, will move to connection manager or a conf
tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
# backup blob
@@ -143,7 +205,7 @@ class FileCommands(CommandTestCase):
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
self.server_blob_manager.blobs.clear()
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
await self.server_blob_manager.blob_completed(missing_blob)
self.server_blob_manager.blob_completed(missing_blob)
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
async def test_paid_download(self):
@@ -155,7 +217,7 @@ class FileCommands(CommandTestCase):
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
response = await self.daemon.jsonrpc_get('lbry://expensive')
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
self.assertEqual(response['error'], 'fee of 11.00000 exceeds max available balance')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
@@ -165,7 +227,7 @@ class FileCommands(CommandTestCase):
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
response = await self.daemon.jsonrpc_get('lbry://maxkey')
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
self.assertEqual(response['error'], 'fee of 111.00000 exceeds max configured to allow of 50.00000')
@@ -177,8 +239,8 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
await self.assertBalance(self.account, '9.925679')
response = await self.daemon.jsonrpc_get('lbry://icanpay')
self.assertNotIn('error', response)
await self.ledger.wait(response['tx'])
raw_content_fee = response.content_fee.raw
await self.ledger.wait(response.content_fee)
await self.assertBalance(self.account, '8.925555')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
@@ -191,3 +253,10 @@ class FileCommands(CommandTestCase):
self.assertEqual(
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
)
# restart the daemon and make sure the fee is still there
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
self.assertEqual(self.daemon.jsonrpc_file_list()[0].content_fee.raw, raw_content_fee)
@@ -36,7 +36,7 @@ class BasicTransactionTest(IntegrationTestCase):
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
stream = Claim()
stream.stream.media_type = "video/mp4"
stream.stream.source.media_type = "video/mp4"
stream_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
)
+31 -1
View File
@@ -1,4 +1,6 @@
from integration.testcase import CommandTestCase
import json
from binascii import hexlify
from lbrynet.testcase import CommandTestCase
class ResolveCommand(CommandTestCase):
@@ -36,6 +38,14 @@ class ResolveCommand(CommandTestCase):
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
self.assertEqual(
claim['claim']['timestamp'],
self.ledger.headers[claim['claim']['height']]['timestamp']
)
self.assertEqual(
claim['certificate']['timestamp'],
self.ledger.headers[claim['certificate']['height']]['timestamp']
)
# resolving claim foo by itself
response = await self.resolve('lbry://foo')
@@ -48,6 +58,10 @@ class ResolveCommand(CommandTestCase):
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
# resolving from the given permanent url
new_response = await self.resolve(claim['claim']['permanent_url'])
self.assertEqual(new_response[claim['claim']['permanent_url']], claim)
# resolving multiple at once
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
@@ -58,3 +72,19 @@ class ResolveCommand(CommandTestCase):
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
# resolve has correct confirmations
tx_details = await self.blockchain.get_raw_transaction(claim['claim']['txid'])
self.assertEqual(claim['claim']['confirmations'], json.loads(tx_details)['confirmations'])
# resolve handles invalid data
txid = await self.blockchain_claim_name(
"gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
response = await self.resolve("lbry://gibberish")
self.assertSetEqual({'lbry://gibberish'}, set(response))
claim = response['lbry://gibberish']['claim']
self.assertEqual(claim['name'], 'gibberish')
self.assertEqual(claim['protobuf'], hexlify(b"{'invalid':'json'}").decode())
self.assertFalse(claim['decoded_claim'])
self.assertEqual(claim['txid'], txid)
self.assertEqual(claim['effective_amount'], "0.1")
+366
View File
@@ -0,0 +1,366 @@
import os
import hashlib
import aiohttp
import aiohttp.web
from lbrynet.utils import aiohttp_request
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.testcase import CommandTestCase
def get_random_bytes(n: int) -> bytes:
result = b''.join(hashlib.sha256(os.urandom(4)).digest() for _ in range(n // 16))
if len(result) < n:
result += os.urandom(n - len(result))
elif len(result) > n:
result = result[:-(len(result) - n)]
assert len(result) == n, (n, len(result))
return result
class RangeRequests(CommandTestCase):
async def _restart_stream_manager(self):
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
return
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
self.daemon.conf.save_blobs = save_blobs
self.daemon.conf.save_files = save_files
self.data = data
await self.stream_create('foo', '0.01', data=self.data, file_size=file_size)
if save_blobs:
self.assertTrue(len(os.listdir(self.daemon.blob_manager.blob_dir)) > 1)
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
# await self._restart_stream_manager()
await self.daemon.runner.setup()
site = aiohttp.web.TCPSite(self.daemon.runner, self.daemon.conf.api_host, self.daemon.conf.api_port)
await site.start()
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
async def _test_range_requests(self):
name = 'foo'
url = f'http://{self.daemon.conf.api_host}:{self.daemon.conf.api_port}/get/{name}'
async with aiohttp_request('get', url) as req:
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
self.assertEqual(content_length, len(streamed_bytes))
return streamed_bytes, content_range, content_length
async def test_range_requests_2_byte(self):
self.data = b'hi'
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(15, content_length)
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_15_byte(self):
self.data = b'123456789abcdef'
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(15, content_length)
self.assertEqual(15, len(streamed))
self.assertEqual(self.data, streamed)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_0_padded_bytes(self, size: int = (MAX_BLOB_SIZE - 1) * 4,
expected_range: str = 'bytes 0-8388603/8388604', padding=b'',
file_size=0):
self.data = get_random_bytes(size)
await self._setup_stream(self.data, file_size=file_size)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(len(self.data + padding), content_length)
self.assertEqual(streamed, self.data + padding)
self.assertEqual(expected_range, content_range)
async def test_range_requests_1_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 1, padding=b'\x00'
)
async def test_range_requests_2_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 2, padding=b'\x00' * 2
)
async def test_range_requests_14_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 14, padding=b'\x00' * 14
)
async def test_range_requests_no_padding_size_from_claim(self):
size = ((MAX_BLOB_SIZE - 1) * 4) - 14
await self.test_range_requests_0_padded_bytes(size, padding=b'', file_size=size,
expected_range=f"bytes 0-{size-1}/{size}")
async def test_range_requests_15_padded_bytes(self):
await self.test_range_requests_0_padded_bytes(
((MAX_BLOB_SIZE - 1) * 4) - 15, padding=b'\x00' * 15
)
async def test_range_requests_last_block_of_last_blob_padding(self):
self.data = get_random_bytes(((MAX_BLOB_SIZE - 1) * 4) - 16)
await self._setup_stream(self.data)
streamed, content_range, content_length = await self._test_range_requests()
self.assertEqual(len(self.data), content_length)
self.assertEqual(streamed, self.data)
self.assertEqual('bytes 0-8388587/8388588', content_range)
async def test_streaming_only_with_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
# test that repeated range requests do not create duplicate files
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
# test that a range request after restart does not create a duplicate file
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
async def test_streaming_only_without_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_blobs=False)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
# test that repeated range requests do not create duplicate files
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
# test that a range request after restart does not create a duplicate file
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.download_directory)
self.assertIsNone(stream.full_path)
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
async def test_stream_and_save_file_with_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_files=True)
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
full_path = stream.full_path
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
for _ in range(3):
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
async def test_stream_and_save_file_without_blobs(self):
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
await self._setup_stream(self.data, save_files=True)
self.daemon.conf.save_blobs = False
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
full_path = stream.full_path
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
for _ in range(3):
await self._test_range_requests()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
await self._restart_stream_manager()
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
await self._test_range_requests()
streams = self.daemon.jsonrpc_file_list()
self.assertEqual(1, len(streams))
stream = streams[0]
self.assertTrue(os.path.isdir(stream.download_directory))
self.assertTrue(os.path.isfile(stream.full_path))
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
self.assertEqual(
len(files_in_download_dir), len(current_files_in_download_dir)
)
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
async def test_switch_save_blobs_while_running(self):
await self.test_streaming_only_without_blobs()
self.daemon.conf.save_blobs = True
blobs_in_stream = self.daemon.jsonrpc_file_list()[0].blobs_in_stream
sd_hash = self.daemon.jsonrpc_file_list()[0].sd_hash
start_file_count = len(os.listdir(self.daemon.blob_manager.blob_dir))
await self._test_range_requests()
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
# switch back
self.daemon.conf.save_blobs = False
await self._test_range_requests()
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=sd_hash)
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
await self._test_range_requests()
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
self.assertEqual(blobs_in_stream, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
async def test_file_save_streaming_only_save_blobs(self):
await self.test_streaming_only_with_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
self.server.stop_server()
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNotNone(stream.full_path)
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=stream.sd_hash)
async def test_file_save_stop_before_finished_streaming_only(self, wait_for_start_writing=False):
await self.test_streaming_only_with_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
self.server.stop_server()
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
path = stream.full_path
self.assertIsNotNone(path)
if wait_for_start_writing:
await stream.started_writing.wait()
self.assertTrue(os.path.isfile(path))
await self._restart_stream_manager()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNotNone(stream.full_path)
self.assertFalse(os.path.isfile(path))
if wait_for_start_writing:
await stream.started_writing.wait()
self.assertTrue(os.path.isfile(path))
async def test_file_save_stop_before_finished_streaming_only_wait_for_start(self):
return await self.test_file_save_stop_before_finished_streaming_only(wait_for_start_writing=True)
async def test_file_save_streaming_only_dont_save_blobs(self):
await self.test_streaming_only_without_blobs()
stream = self.daemon.jsonrpc_file_list()[0]
self.assertIsNone(stream.full_path)
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
stream = self.daemon.jsonrpc_file_list()[0]
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
+1 -1
View File
@@ -54,7 +54,7 @@ class AccountSynchronization(AsyncioTestCase):
async def asyncTearDown(self):
self.wallet_component._running = False
await self.daemon.stop()
await self.daemon.stop(shutdown_runner=False)
@mock.patch('time.time', mock.Mock(return_value=12345))
def test_sync(self):
@@ -1,4 +1,4 @@
from integration.testcase import CommandTestCase
from lbrynet.testcase import CommandTestCase
class TransactionCommandsTestCase(CommandTestCase):
+209 -20
View File
@@ -3,34 +3,223 @@ import tempfile
import shutil
import os
from torba.testcase import AsyncioTestCase
from lbrynet.error import InvalidDataError, InvalidBlobHashError
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import BlobFile, BlobBuffer, AbstractBlob
class TestBlobfile(AsyncioTestCase):
async def test_create_blob(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
class TestBlob(AsyncioTestCase):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
async def asyncSetUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
self.loop = asyncio.get_running_loop()
self.config = Config()
self.storage = SQLiteStorage(self.config, ":memory:", self.loop)
self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.config)
await self.storage.open()
def _get_blob(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(self.loop, self.blob_hash, len(self.blob_bytes), self.blob_manager.blob_completed,
blob_directory=blob_directory)
self.assertFalse(blob.get_is_verified())
self.addCleanup(blob.close)
return blob
async def _test_create_blob(self, blob_class=AbstractBlob, blob_directory=None):
blob = self._get_blob(blob_class, blob_directory)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
await asyncio.sleep(0, loop=self.loop) # wait for the db save task
return blob
async def _test_close_writers_on_finished(self, blob_class=AbstractBlob, blob_directory=None):
blob = self._get_blob(blob_class, blob_directory=blob_directory)
writers = [blob.get_blob_writer('1.2.3.4', port) for port in range(5)]
self.assertEqual(5, len(blob.writers))
# test that writing too much causes the writer to fail with InvalidDataError and to be removed
with self.assertRaises(InvalidDataError):
writers[1].write(self.blob_bytes * 2)
await writers[1].finished
await asyncio.sleep(0, loop=self.loop)
self.assertEqual(4, len(blob.writers))
# write the blob
other = writers[2]
writers[3].write(self.blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertEqual(0, len(blob.writers))
with self.assertRaises(IOError):
other.write(self.blob_bytes)
def _test_ioerror_if_length_not_set(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(
self.loop, self.blob_hash, blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=blob_directory
)
self.addCleanup(blob.close)
writer = blob.get_blob_writer()
with self.assertRaises(IOError):
writer.write(b'')
async def _test_invalid_blob_bytes(self, blob_class=AbstractBlob, blob_directory=None):
blob = blob_class(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=blob_directory
)
self.addCleanup(blob.close)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes[:-4] + b'fake')
with self.assertRaises(InvalidBlobHashError):
await writer.finished
async def test_add_blob_buffer_to_db(self):
blob = await self._test_create_blob(BlobBuffer)
db_status = await self.storage.get_blob_status(blob.blob_hash)
self.assertEqual(db_status, 'pending')
async def test_add_blob_file_to_db(self):
blob = await self._test_create_blob(BlobFile, self.tmp_dir)
db_status = await self.storage.get_blob_status(blob.blob_hash)
self.assertEqual(db_status, 'finished')
async def test_invalid_blob_bytes(self):
await self._test_invalid_blob_bytes(BlobBuffer)
await self._test_invalid_blob_bytes(BlobFile, self.tmp_dir)
def test_ioerror_if_length_not_set(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self._test_ioerror_if_length_not_set(BlobBuffer)
self._test_ioerror_if_length_not_set(BlobFile, tmp_dir)
async def test_create_blob_file(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
blob = await self._test_create_blob(BlobFile, tmp_dir)
self.assertIsInstance(blob, BlobFile)
self.assertTrue(os.path.isfile(blob.file_path))
for _ in range(2):
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
async def test_create_blob_buffer(self):
blob = await self._test_create_blob(BlobBuffer)
self.assertIsInstance(blob, BlobBuffer)
self.assertIsNotNone(blob._verified_bytes)
# check we can only read the bytes once, and that the buffer is torn down
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
self.assertIsNone(blob._verified_bytes)
with self.assertRaises(OSError):
with blob.reader_context() as reader:
self.assertEqual(self.blob_bytes, reader.read())
self.assertIsNone(blob._verified_bytes)
async def test_close_writers_on_finished(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
await self._test_close_writers_on_finished(BlobBuffer)
await self._test_close_writers_on_finished(BlobFile, tmp_dir)
async def test_concurrency_and_premature_closes(self):
blob_directory = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(blob_directory))
blob = self._get_blob(BlobBuffer, blob_directory=blob_directory)
writer = blob.get_blob_writer('1.1.1.1', 1337)
self.assertEqual(1, len(blob.writers))
with self.assertRaises(OSError):
blob.get_blob_writer('1.1.1.1', 1337)
writer.close_handle()
self.assertTrue(blob.writers[('1.1.1.1', 1337)].closed())
writer = blob.get_blob_writer('1.1.1.1', 1337)
self.assertEqual(blob.writers[('1.1.1.1', 1337)], writer)
writer.close_handle()
await asyncio.sleep(0.000000001) # flush callbacks
self.assertEqual(0, len(blob.writers))
async def test_delete(self):
blob_buffer = await self._test_create_blob(BlobBuffer)
self.assertIsInstance(blob_buffer, BlobBuffer)
self.assertIsNotNone(blob_buffer._verified_bytes)
self.assertTrue(blob_buffer.get_is_verified())
blob_buffer.delete()
self.assertIsNone(blob_buffer._verified_bytes)
self.assertFalse(blob_buffer.get_is_verified())
loop = asyncio.get_event_loop()
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, tmp_dir, storage)
blob_file = await self._test_create_blob(BlobFile, tmp_dir)
self.assertIsInstance(blob_file, BlobFile)
self.assertTrue(os.path.isfile(blob_file.file_path))
self.assertTrue(blob_file.get_is_verified())
blob_file.delete()
self.assertFalse(os.path.isfile(blob_file.file_path))
self.assertFalse(blob_file.get_is_verified())
await storage.open()
await blob_manager.setup()
async def test_delete_corrupt(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
writer = blob.get_blob_writer()
writer.write(self.blob_bytes)
await blob.verified.wait()
blob.close()
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
self.assertTrue(blob.get_is_verified())
# add the blob on the server
blob = blob_manager.get_blob(blob_hash, len(blob_bytes))
self.assertEqual(blob.get_is_verified(), False)
self.assertNotIn(blob_hash, blob_manager.completed_blob_hashes)
with open(blob.file_path, 'wb+') as f:
f.write(b'\x00')
blob = BlobFile(
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
blob_directory=tmp_dir
)
self.assertFalse(blob.get_is_verified())
self.assertFalse(os.path.isfile(blob.file_path))
writer = blob.open_for_writing()
writer.write(blob_bytes)
await blob.finished_writing.wait()
self.assertTrue(os.path.isfile(blob.file_path), True)
self.assertEqual(blob.get_is_verified(), True)
self.assertIn(blob_hash, blob_manager.completed_blob_hashes)
def test_invalid_blob_hash(self):
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, '', len(self.blob_bytes))
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'x' * 96, len(self.blob_bytes))
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'a' * 97, len(self.blob_bytes))
async def _test_close_reader(self, blob_class=AbstractBlob, blob_directory=None):
blob = await self._test_create_blob(blob_class, blob_directory)
reader = blob.reader_context()
self.assertEqual(0, len(blob.readers))
async def read_blob_buffer():
with reader as read_handle:
self.assertEqual(1, len(blob.readers))
await asyncio.sleep(2, loop=self.loop)
self.assertEqual(0, len(blob.readers))
return read_handle.read()
self.loop.call_later(1, blob.close)
with self.assertRaises(ValueError) as err:
read_task = self.loop.create_task(read_blob_buffer())
await read_task
self.assertEqual(err.exception, ValueError("I/O operation on closed file"))
async def test_close_reader(self):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
await self._test_close_reader(BlobBuffer)
await self._test_close_reader(BlobFile, tmp_dir)
+23 -20
View File
@@ -5,50 +5,53 @@ import os
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
class TestBlobManager(AsyncioTestCase):
async def test_sync_blob_manager_on_startup(self):
loop = asyncio.get_event_loop()
async def setup_blob_manager(self, save_blobs=True):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.config = Config(save_blobs=save_blobs)
self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite"))
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, tmp_dir, storage)
async def test_sync_blob_file_manager_on_startup(self):
await self.setup_blob_manager(save_blobs=True)
# add a blob file
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
with open(os.path.join(blob_manager.blob_dir, blob_hash), 'wb') as f:
with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f:
f.write(blob_bytes)
# it should not have been added automatically on startup
await storage.open()
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# make sure we can add the blob
await blob_manager.blob_completed(blob_manager.get_blob(blob_hash, len(blob_bytes)))
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes)))
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# stop the blob manager and restart it, make sure the blob is there
blob_manager.stop()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
self.blob_manager.stop()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
# test that the blob is removed upon the next startup after the file being manually deleted
blob_manager.stop()
self.blob_manager.stop()
# manually delete the blob file and restart the blob manager
os.remove(os.path.join(blob_manager.blob_dir, blob_hash))
await blob_manager.setup()
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash))
await self.blob_manager.setup()
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
# check that the deleted blob was updated in the database
self.assertEqual(
'pending', (
await storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
)
)
+83 -17
View File
@@ -9,9 +9,9 @@ from lbrynet.blob_exchange.serialization import BlobRequest
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.blob_exchange.client import request_blob
from lbrynet.dht.peer import KademliaPeer, PeerManager
# import logging
@@ -35,13 +35,13 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
reflector_servers=[])
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
reflector_servers=[])
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
@@ -51,6 +51,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
await self.server_blob_manager.setup()
self.server.start_server(33333, '127.0.0.1')
self.addCleanup(self.server.stop_server)
await self.server.started_listening.wait()
@@ -58,21 +59,25 @@ class TestBlobExchange(BlobExchangeTestBase):
async def _add_blob_to_server(self, blob_hash: str, blob_bytes: bytes):
# add the blob on the server
server_blob = self.server_blob_manager.get_blob(blob_hash, len(blob_bytes))
writer = server_blob.open_for_writing()
writer = server_blob.get_blob_writer()
writer.write(blob_bytes)
await server_blob.finished_writing.wait()
await server_blob.verified.wait()
self.assertTrue(os.path.isfile(server_blob.file_path))
self.assertEqual(server_blob.get_is_verified(), True)
self.assertTrue(writer.closed())
async def _test_transfer_blob(self, blob_hash: str):
client_blob = self.client_blob_manager.get_blob(blob_hash)
# download the blob
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
self.server_from_client.tcp_port, 2, 3)
await client_blob.finished_writing.wait()
downloaded, transport = await request_blob(self.loop, client_blob, self.server_from_client.address,
self.server_from_client.tcp_port, 2, 3)
self.assertIsNotNone(transport)
self.addCleanup(transport.close)
await client_blob.verified.wait()
self.assertEqual(client_blob.get_is_verified(), True)
self.assertTrue(downloaded)
client_blob.close()
async def test_transfer_sd_blob(self):
sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02"
@@ -92,9 +97,11 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
second_client_conf = Config()
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
@@ -102,7 +109,7 @@ class TestBlobExchange(BlobExchangeTestBase):
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
second_client_blob = second_client_blob_manager.get_blob(blob_hash)
# download the blob
await asyncio.gather(
@@ -112,9 +119,65 @@ class TestBlobExchange(BlobExchangeTestBase):
),
self._test_transfer_blob(blob_hash)
)
await second_client_blob.finished_writing.wait()
await second_client_blob.verified.wait()
self.assertEqual(second_client_blob.get_is_verified(), True)
async def test_blob_writers_concurrency(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.server_blob_manager.get_blob(blob_hash)
write_blob = blob._write_blob
write_called_count = 0
def wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
write_blob(blob_bytes)
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
writer2 = blob.get_blob_writer(peer_port=2)
reader1_ctx_before_write = blob.reader_context()
with self.assertRaises(OSError):
blob.get_blob_writer(peer_port=2)
with self.assertRaises(OSError):
with blob.reader_context():
pass
blob.set_length(len(mock_blob_bytes))
results = {}
def check_finished_callback(writer, num):
def inner(writer_future: asyncio.Future):
results[num] = writer_future.result()
writer.finished.add_done_callback(inner)
check_finished_callback(writer1, 1)
check_finished_callback(writer2, 2)
def write_task(writer):
async def _inner():
writer.write(mock_blob_bytes)
return self.loop.create_task(_inner())
await asyncio.gather(write_task(writer1), write_task(writer2), loop=self.loop)
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
with reader1_ctx_before_write as f:
self.assertEqual(mock_blob_bytes, f.read())
with blob.reader_context() as f:
self.assertEqual(mock_blob_bytes, f.read())
with blob.reader_context() as f:
blob.close()
with self.assertRaises(ValueError):
f.read()
self.assertListEqual([], blob.readers)
async def test_host_different_blobs_to_multiple_peers_at_once(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
@@ -124,9 +187,12 @@ class TestBlobExchange(BlobExchangeTestBase):
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_conf = Config()
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobManager(
self.loop, second_client_dir, second_client_storage, second_client_conf
)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
@@ -143,7 +209,7 @@ class TestBlobExchange(BlobExchangeTestBase):
server_from_second_client.tcp_port, 2, 3
),
self._test_transfer_blob(sd_hash),
second_client_blob.finished_writing.wait()
second_client_blob.verified.wait()
)
self.assertEqual(second_client_blob.get_is_verified(), True)
+76 -2
View File
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from lbrynet import utils
import unittest
import asyncio
from lbrynet import utils
from torba.testcase import AsyncioTestCase
class CompareVersionTest(unittest.TestCase):
@@ -61,3 +62,76 @@ class SdHashTests(unittest.TestCase):
}
}
self.assertIsNone(utils.get_sd_hash(claim))
class CacheConcurrentDecoratorTests(AsyncioTestCase):
def setUp(self):
self.called = []
self.finished = []
self.counter = 0
@utils.cache_concurrent
async def foo(self, arg1, arg2=None, delay=1):
self.called.append((arg1, arg2, delay))
await asyncio.sleep(delay, loop=self.loop)
self.counter += 1
self.finished.append((arg1, arg2, delay))
return object()
async def test_gather_duplicates(self):
result = await asyncio.gather(
self.loop.create_task(self.foo(1)), self.loop.create_task(self.foo(1)), loop=self.loop
)
self.assertEqual(1, len(self.called))
self.assertEqual(1, len(self.finished))
self.assertEqual(1, self.counter)
self.assertIs(result[0], result[1])
self.assertEqual(2, len(result))
async def test_one_cancelled_all_cancel(self):
t1 = self.loop.create_task(self.foo(1))
self.loop.call_later(0.1, t1.cancel)
with self.assertRaises(asyncio.CancelledError):
await asyncio.gather(
t1, self.loop.create_task(self.foo(1)), loop=self.loop
)
self.assertEqual(1, len(self.called))
self.assertEqual(0, len(self.finished))
self.assertEqual(0, self.counter)
async def test_error_after_success(self):
def cause_type_error():
self.counter = ""
self.loop.call_later(0.1, cause_type_error)
t1 = self.loop.create_task(self.foo(1))
t2 = self.loop.create_task(self.foo(1))
with self.assertRaises(TypeError):
await t2
self.assertEqual(1, len(self.called))
self.assertEqual(0, len(self.finished))
self.assertTrue(t1.done())
self.assertEqual("", self.counter)
# test that the task is run fresh, it should not error
self.counter = 0
t3 = self.loop.create_task(self.foo(1))
self.assertTrue((await t3))
self.assertEqual(1, self.counter)
# the previously failed call should still raise if awaited
with self.assertRaises(TypeError):
await t1
self.assertEqual(1, self.counter)
async def test_break_it(self):
t1 = self.loop.create_task(self.foo(1))
t2 = self.loop.create_task(self.foo(1))
t3 = self.loop.create_task(self.foo(2, delay=0))
t3.add_done_callback(lambda _: t2.cancel())
with self.assertRaises(asyncio.CancelledError):
await asyncio.gather(t1, t2, t3)
+6 -5
View File
@@ -7,7 +7,7 @@ from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.descriptor import StreamDescriptor
from tests.test_utils import random_lbry_hash
@@ -68,17 +68,18 @@ fake_claim_info = {
class StorageTest(AsyncioTestCase):
async def asyncSetUp(self):
self.storage = SQLiteStorage(Config(), ':memory:')
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ':memory:')
self.blob_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.blob_dir)
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), self.blob_dir, self.storage)
self.blob_manager = BlobManager(asyncio.get_event_loop(), self.blob_dir, self.storage, self.conf)
await self.storage.open()
async def asyncTearDown(self):
await self.storage.close()
async def store_fake_blob(self, blob_hash, length=100):
await self.storage.add_completed_blob(blob_hash, length)
await self.storage.add_blobs((blob_hash, length), finished=True)
async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"):
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", random_lbry_hash())]
@@ -130,7 +131,7 @@ class TestSQLiteStorage(StorageTest):
} for i in range(20)]
expected_supports = {}
for idx, claim_id in enumerate(claim_ids):
await self.storage.save_supports(claim_id, random_supports[idx*2:idx*2+2])
await self.storage.save_supports({claim_id: random_supports[idx*2:idx*2+2]})
for random_support in random_supports[idx*2:idx*2+2]:
random_support['claim_id'] = claim_id
expected_supports.setdefault(claim_id, []).append(random_support)
@@ -17,10 +17,10 @@ class MockAsyncGen:
return self
async def __anext__(self):
await asyncio.sleep(self.delay, loop=self.loop)
if self.count > self.stop_cnt - 1:
raise StopAsyncIteration()
self.count += 1
await asyncio.sleep(self.delay, loop=self.loop)
return self.result
async def aclose(self):
@@ -48,6 +48,18 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
self.assertEqual(fast_gen.called_close, True)
self.assertEqual(slow_gen.called_close, True)
async def test_nothing_to_yield(self):
async def __nothing():
for _ in []:
yield self.fail("nada")
await self._test_junction([], __nothing())
async def test_fast_iteratiors(self):
async def __gotta_go_fast():
for _ in range(10):
yield 0
await self._test_junction([0]*40, __gotta_go_fast(), __gotta_go_fast(), __gotta_go_fast(), __gotta_go_fast())
@unittest.SkipTest
async def test_one_stopped_first(self):
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
+4 -3
View File
@@ -1,12 +1,13 @@
import asyncio
from torba.testcase import AsyncioTestCase
from unittest import mock, TestCase
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.dht.peer import PeerManager
class DataStoreTests(AsyncioTestCase):
class DataStoreTests(TestCase):
def setUp(self):
self.loop = asyncio.get_event_loop()
self.loop = mock.Mock(spec=asyncio.BaseEventLoop)
self.loop.time = lambda: 0.0
self.peer_manager = PeerManager(self.loop)
self.data_store = DictDataStore(self.loop, self.peer_manager)
+3 -4
View File
@@ -78,15 +78,14 @@ class TestBlobAnnouncer(AsyncioTestCase):
blob2 = binascii.hexlify(b'2' * 48).decode()
async with self._test_network_context():
await self.storage.add_completed_blob(blob1, 1024)
await self.storage.add_completed_blob(blob2, 1024)
await self.storage.add_blobs((blob1, 1024), (blob2, 1024), finished=True)
await self.storage.db.execute(
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
(blob1, blob2)
)
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(2, len(to_announce))
self.blob_announcer.start()
self.blob_announcer.start(batch_size=1) # so it covers batching logic
await self.advance(61.0)
to_announce = await self.storage.get_blobs_to_announce()
self.assertEqual(0, len(to_announce))
@@ -106,7 +105,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
_, task = last.accumulate_peers(search_q, peer_q)
found_peers = await peer_q.get()
task.cancel()
await task
self.assertEqual(1, len(found_peers))
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
+6 -6
View File
@@ -4,13 +4,13 @@ from lbrynet.schema import mime_types
class TestMimeTypes(unittest.TestCase):
def test_mp4_video(self):
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4"))
self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4"))
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4")[0])
self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4")[0])
def test_x_ext_(self):
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry"))
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY"))
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry")[0])
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY")[0])
def test_octet_stream(self):
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test."))
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test"))
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test.")[0])
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test")[0])
+12 -12
View File
@@ -21,10 +21,10 @@ class TestOldJSONSchemaCompatibility(TestCase):
self.assertEqual(stream.license, 'None')
self.assertEqual(stream.author, 'root')
self.assertEqual(stream.langtags, ['en'])
self.assertEqual(stream.media_type, 'application/octet-stream')
self.assertEqual(stream.thumbnail_url, '/home/robert/lbry/speed.jpg')
self.assertEqual(stream.source.media_type, 'application/octet-stream')
self.assertEqual(stream.thumbnail.url, '/home/robert/lbry/speed.jpg')
self.assertEqual(
stream.sd_hash,
stream.source.sd_hash,
'bbd1f68374ff9a1044a90d7dd578ce41979211c386caf19e'
'6f496536db5f2c96b58fe2c7a6677b331419a117873b539f'
)
@@ -52,9 +52,9 @@ class TestOldJSONSchemaCompatibility(TestCase):
self.assertEqual(stream.license_url, 'https://creativecommons.org/licenses/by/3.0/us/legalcode')
self.assertEqual(stream.author, 'Mii')
self.assertEqual(stream.langtags, ['en'])
self.assertEqual(stream.media_type, 'application/x-msdownload')
self.assertEqual(stream.source.media_type, 'application/x-msdownload')
self.assertEqual(
stream.sd_hash,
stream.source.sd_hash,
'c5ffee0fa5168e16681b519d9d85446e8d1d818a616bd555'
'40aa7374d2321b51abf2ac3dae1443a03dadcc8f7affaa62'
)
@@ -78,9 +78,9 @@ class TestOldJSONSchemaCompatibility(TestCase):
self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International')
self.assertEqual(stream.author, 'sgb')
self.assertEqual(stream.langtags, ['en'])
self.assertEqual(stream.media_type, 'video/mp4')
self.assertEqual(stream.source.media_type, 'video/mp4')
self.assertEqual(
stream.sd_hash,
stream.source.sd_hash,
'd83db664c6d7d570aa824300f4869e0bfb560e765efa477a'
'ebf566467f8d3a57f4f8c704cab1308eb75ff8b7e84e3caf'
)
@@ -121,10 +121,10 @@ class TestTypesV1Compatibility(TestCase):
self.assertEqual(stream.license, 'Copyrighted (contact author)')
self.assertEqual(stream.author, 'The Linux Gamer')
self.assertEqual(stream.langtags, ['en'])
self.assertEqual(stream.media_type, 'video/mp4')
self.assertEqual(stream.thumbnail_url, 'https://berk.ninja/thumbnails/FrTdBCOS_fc')
self.assertEqual(stream.source.media_type, 'video/mp4')
self.assertEqual(stream.thumbnail.url, 'https://berk.ninja/thumbnails/FrTdBCOS_fc')
self.assertEqual(
stream.sd_hash,
stream.source.sd_hash,
'040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc'
'22f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f3'
)
@@ -158,9 +158,9 @@ class TestTypesV1Compatibility(TestCase):
self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International')
self.assertEqual(stream.author, 'rpg midi')
self.assertEqual(stream.langtags, ['en'])
self.assertEqual(stream.media_type, 'application/x-zip-compressed')
self.assertEqual(stream.source.media_type, 'application/x-zip-compressed')
self.assertEqual(
stream.sd_hash,
stream.source.sd_hash,
'1f41eb0312aa7e8a5ce49349bc77d811da975833719d7515'
'23b19f123fc3d528d6a94e3446ccddb7b9329f27a9cad7e3'
)
+3 -17
View File
@@ -1,7 +1,7 @@
from unittest import TestCase
from decimal import Decimal
from lbrynet.schema.claim import Claim, Channel, Stream
from lbrynet.schema.claim import Claim, Stream
class TestClaimContainerAwareness(TestCase):
@@ -9,27 +9,13 @@ class TestClaimContainerAwareness(TestCase):
def test_stream_claim(self):
stream = Stream()
claim = stream.claim
self.assertTrue(claim.is_stream)
self.assertFalse(claim.is_channel)
self.assertEqual(claim.claim_type, Claim.STREAM)
claim = Claim.from_bytes(claim.to_bytes())
self.assertTrue(claim.is_stream)
self.assertFalse(claim.is_channel)
self.assertEqual(claim.claim_type, Claim.STREAM)
self.assertIsNotNone(claim.stream)
with self.assertRaisesRegex(ValueError, 'Claim is not a channel.'):
print(claim.channel)
def test_channel_claim(self):
channel = Channel()
claim = channel.claim
self.assertFalse(claim.is_stream)
self.assertTrue(claim.is_channel)
claim = Claim.from_bytes(claim.to_bytes())
self.assertFalse(claim.is_stream)
self.assertTrue(claim.is_channel)
self.assertIsNotNone(claim.channel)
with self.assertRaisesRegex(ValueError, 'Claim is not a stream.'):
print(claim.stream)
class TestFee(TestCase):
-117
View File
@@ -1,117 +0,0 @@
import os
import asyncio
import tempfile
import shutil
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.stream_manager import StreamManager
class TestStreamAssembler(AsyncioTestCase):
def setUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
self.cleartext = b'test'
async def test_create_and_decrypt_one_blob_stream(self, corrupt=False):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.storage = SQLiteStorage(Config(), ":memory:")
await self.storage.open()
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
# create the stream
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
sd = await StreamDescriptor.create_stream(self.loop, tmp_dir, file_path, key=self.key)
# copy blob files
sd_hash = sd.calculate_sd_hash()
shutil.copy(os.path.join(tmp_dir, sd_hash), os.path.join(download_dir, sd_hash))
for blob_info in sd.blobs:
if blob_info.blob_hash:
shutil.copy(os.path.join(tmp_dir, blob_info.blob_hash), os.path.join(download_dir, blob_info.blob_hash))
if corrupt and blob_info.length == MAX_BLOB_SIZE:
with open(os.path.join(download_dir, blob_info.blob_hash), "rb+") as handle:
handle.truncate()
handle.flush()
downloader_storage = SQLiteStorage(Config(), os.path.join(download_dir, "lbrynet.sqlite"))
await downloader_storage.open()
# add the blobs to the blob table (this would happen upon a blob download finishing)
downloader_blob_manager = BlobFileManager(self.loop, download_dir, downloader_storage)
descriptor = await downloader_blob_manager.get_stream_descriptor(sd_hash)
# assemble the decrypted file
assembler = StreamAssembler(self.loop, downloader_blob_manager, descriptor.sd_hash)
await assembler.assemble_decrypted_stream(download_dir)
if corrupt:
return self.assertFalse(os.path.isfile(os.path.join(download_dir, "test_file")))
with open(os.path.join(download_dir, "test_file"), "rb") as f:
decrypted = f.read()
self.assertEqual(decrypted, self.cleartext)
self.assertEqual(True, self.blob_manager.get_blob(sd_hash).get_is_verified())
self.assertEqual(True, self.blob_manager.get_blob(descriptor.blobs[0].blob_hash).get_is_verified())
# its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
self.assertEqual(
[descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
)
await downloader_storage.close()
await self.storage.close()
async def test_create_and_decrypt_multi_blob_stream(self):
self.cleartext = b'test\n' * 20000000
await self.test_create_and_decrypt_one_blob_stream()
async def test_create_and_decrypt_padding(self):
for i in range(16):
self.cleartext = os.urandom((MAX_BLOB_SIZE*2) + i)
await self.test_create_and_decrypt_one_blob_stream()
for i in range(16):
self.cleartext = os.urandom((MAX_BLOB_SIZE*2) - i)
await self.test_create_and_decrypt_one_blob_stream()
async def test_create_and_decrypt_random(self):
self.cleartext = os.urandom(20000000)
await self.test_create_and_decrypt_one_blob_stream()
async def test_create_managed_stream_announces(self):
# setup a blob manager
storage = SQLiteStorage(Config(), ":memory:")
await storage.open()
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
blob_manager = BlobFileManager(self.loop, tmp_dir, storage)
stream_manager = StreamManager(self.loop, Config(), blob_manager, None, storage, None)
# create the stream
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
file_path = os.path.join(download_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(b'testtest')
stream = await stream_manager.create_stream(file_path)
self.assertEqual(
[stream.sd_hash, stream.descriptor.blobs[0].blob_hash],
await storage.get_blobs_to_announce())
async def test_create_truncate_and_handle_stream(self):
self.cleartext = b'potato' * 1337 * 5279
# The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
-102
View File
@@ -1,102 +0,0 @@
import os
import time
import unittest
from unittest import mock
import asyncio
from lbrynet.blob_exchange.serialization import BlobResponse
from lbrynet.blob_exchange.server import BlobServerProtocol
from lbrynet.conf import Config
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
class TestStreamDownloader(BlobExchangeTestBase):
async def setup_stream(self, blob_count: int = 10):
self.stream_bytes = b''
for _ in range(blob_count):
self.stream_bytes += os.urandom((MAX_BLOB_SIZE - 1))
# create the stream
file_path = os.path.join(self.server_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.stream_bytes)
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
self.sd_hash = descriptor.calculate_sd_hash()
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
reflector_servers=[])
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
await self.setup_stream(blob_count)
mock_node = mock.Mock(spec=Node)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
q2.put_nowait([self.server_from_client])
return q2, self.loop.create_task(_task())
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
self.downloader.download(mock_node)
await self.downloader.stream_finished_event.wait()
self.assertTrue(self.downloader.stream_handle.closed)
self.assertTrue(os.path.isfile(self.downloader.output_path))
self.downloader.stop()
self.assertIs(self.downloader.stream_handle, None)
self.assertTrue(os.path.isfile(self.downloader.output_path))
with open(self.downloader.output_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
await asyncio.sleep(0.01)
async def test_transfer_stream(self):
await self._test_transfer_stream(10)
@unittest.SkipTest
async def test_transfer_hundred_blob_stream(self):
await self._test_transfer_stream(100)
async def test_transfer_stream_bad_first_peer_good_second(self):
await self.setup_stream(2)
mock_node = mock.Mock(spec=Node)
q = asyncio.Queue()
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
q2.put_nowait([bad_peer])
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
return q2, self.loop.create_task(_task())
mock_node.accumulate_peers = _mock_accumulate_peers
self.downloader.download(mock_node)
await self.downloader.stream_finished_event.wait()
self.assertTrue(os.path.isfile(self.downloader.output_path))
with open(self.downloader.output_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
# self.assertIs(self.server_from_client.tcp_last_down, None)
# self.assertIsNot(bad_peer.tcp_last_down, None)
async def test_client_chunked_response(self):
self.server.stop_server()
class ChunkedServerProtocol(BlobServerProtocol):
def send_response(self, responses):
to_send = []
while responses:
to_send.append(responses.pop())
for byte in BlobResponse(to_send).serialize():
self.transport.write(bytes([byte]))
self.server.server_protocol_class = ChunkedServerProtocol
self.server.start_server(33333, '127.0.0.1')
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
+170
View File
@@ -0,0 +1,170 @@
import os
import shutil
import unittest
from unittest import mock
import asyncio
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.blob_exchange.serialization import BlobResponse
from lbrynet.blob_exchange.server import BlobServerProtocol
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.extras.daemon.storage import StoredStreamClaim
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.stream.descriptor import StreamDescriptor
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
def get_mock_node(loop):
mock_node = mock.Mock(spec=Node)
mock_node.joined = asyncio.Event(loop=loop)
mock_node.joined.set()
return mock_node
class TestManagedStream(BlobExchangeTestBase):
async def create_stream(self, blob_count: int = 10):
self.stream_bytes = b''
for _ in range(blob_count):
self.stream_bytes += os.urandom((MAX_BLOB_SIZE - 1))
# create the stream
file_path = os.path.join(self.server_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.stream_bytes)
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
self.sd_hash = descriptor.calculate_sd_hash()
return descriptor
async def setup_stream(self, blob_count: int = 10):
await self.create_stream(blob_count)
self.stream = ManagedStream(
self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir
)
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
await self.setup_stream(blob_count)
mock_node = mock.Mock(spec=Node)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
q2.put_nowait([self.server_from_client])
return q2, self.loop.create_task(_task())
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
await self.stream.finished_writing.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
if stop_when_done:
await self.stream.stop()
self.assertTrue(os.path.isfile(self.stream.full_path))
with open(self.stream.full_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
await asyncio.sleep(0.01)
async def test_transfer_stream(self):
await self._test_transfer_stream(10)
self.assertEqual(self.stream.status, "finished")
self.assertFalse(self.stream._running.is_set())
async def test_delayed_stop(self):
await self._test_transfer_stream(10, stop_when_done=False)
self.assertEqual(self.stream.status, "finished")
self.assertTrue(self.stream._running.is_set())
await asyncio.sleep(0.5, loop=self.loop)
self.assertTrue(self.stream._running.is_set())
await asyncio.sleep(0.6, loop=self.loop)
self.assertEqual(self.stream.status, "finished")
self.assertFalse(self.stream._running.is_set())
@unittest.SkipTest
async def test_transfer_hundred_blob_stream(self):
await self._test_transfer_stream(100)
async def test_transfer_stream_bad_first_peer_good_second(self):
await self.setup_stream(2)
mock_node = mock.Mock(spec=Node)
q = asyncio.Queue()
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
q2.put_nowait([bad_peer])
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
return q2, self.loop.create_task(_task())
mock_node.accumulate_peers = _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
await self.stream.finished_writing.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
with open(self.stream.full_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
await self.stream.stop()
# self.assertIs(self.server_from_client.tcp_last_down, None)
# self.assertIsNot(bad_peer.tcp_last_down, None)
async def test_client_chunked_response(self):
self.server.stop_server()
class ChunkedServerProtocol(BlobServerProtocol):
def send_response(self, responses):
to_send = []
while responses:
to_send.append(responses.pop())
for byte in BlobResponse(to_send).serialize():
self.transport.write(bytes([byte]))
self.server.server_protocol_class = ChunkedServerProtocol
self.server.start_server(33333, '127.0.0.1')
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
async def test_create_and_decrypt_one_blob_stream(self, blobs=1, corrupt=False):
descriptor = await self.create_stream(blobs)
# copy blob files
shutil.copy(os.path.join(self.server_blob_manager.blob_dir, self.sd_hash),
os.path.join(self.client_blob_manager.blob_dir, self.sd_hash))
self.stream = ManagedStream(self.loop, self.client_config, self.client_blob_manager, self.sd_hash,
self.client_dir)
for blob_info in descriptor.blobs[:-1]:
shutil.copy(os.path.join(self.server_blob_manager.blob_dir, blob_info.blob_hash),
os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash))
if corrupt and blob_info.length == MAX_BLOB_SIZE:
with open(os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash), "rb+") as handle:
handle.truncate()
handle.flush()
await self.stream.save_file()
await self.stream.finished_writing.wait()
if corrupt:
return self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
with open(os.path.join(self.client_dir, "test_file"), "rb") as f:
decrypted = f.read()
self.assertEqual(decrypted, self.stream_bytes)
self.assertEqual(True, self.client_blob_manager.get_blob(self.sd_hash).get_is_verified())
self.assertEqual(
True, self.client_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
)
#
# # its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
# self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
# self.assertEqual(
# [descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
# )
#
# await downloader_storage.close()
# await self.storage.close()
async def test_create_and_decrypt_multi_blob_stream(self):
await self.test_create_and_decrypt_one_blob_stream(10)
# async def test_create_truncate_and_handle_stream(self):
# # The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
# await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
+12 -5
View File
@@ -5,7 +5,7 @@ import shutil
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.stream.reflector.server import ReflectorServer
@@ -18,16 +18,18 @@ class TestStreamAssembler(AsyncioTestCase):
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
self.conf = Config()
self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf)
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
self.server_storage = SQLiteStorage(Config(), os.path.join(server_tmp_dir, "lbrynet.sqlite"))
self.server_conf = Config()
self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
@@ -60,3 +62,8 @@ class TestStreamAssembler(AsyncioTestCase):
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
async def test_announces(self):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
+9 -7
View File
@@ -9,7 +9,7 @@ from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.error import InvalidStreamDescriptorError
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.stream.descriptor import StreamDescriptor
@@ -20,9 +20,10 @@ class TestStreamDescriptor(AsyncioTestCase):
self.cleartext = os.urandom(20000000)
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
self.storage = SQLiteStorage(Config(), ":memory:")
self.conf = Config()
self.storage = SQLiteStorage(self.conf, ":memory:")
await self.storage.open()
self.blob_manager = BlobFileManager(self.loop, self.tmp_dir, self.storage)
self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.conf)
self.file_path = os.path.join(self.tmp_dir, "test_file")
with open(self.file_path, 'wb') as f:
@@ -83,9 +84,10 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
loop = asyncio.get_event_loop()
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
storage = SQLiteStorage(Config(), ":memory:")
self.conf = Config()
storage = SQLiteStorage(self.conf, ":memory:")
await storage.open()
blob_manager = BlobFileManager(loop, tmp_dir, storage)
blob_manager = BlobManager(loop, tmp_dir, storage, self.conf)
sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \
b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \
@@ -99,7 +101,7 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
blob = blob_manager.get_blob(sd_hash)
blob.set_length(len(sd_bytes))
writer = blob.open_for_writing()
writer = blob.get_blob_writer()
writer.write(sd_bytes)
await blob.verified.wait()
descriptor = await StreamDescriptor.from_stream_descriptor_blob(
@@ -116,7 +118,7 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
with open(os.path.join(tmp_dir, sd_hash), 'wb') as handle:
handle.write(b'doesnt work')
blob = BlobFile(loop, tmp_dir, sd_hash)
blob = BlobFile(loop, sd_hash, blob_directory=tmp_dir)
self.assertTrue(blob.file_exists)
self.assertIsNotNone(blob.length)
with self.assertRaises(InvalidStreamDescriptorError):
+20 -17
View File
@@ -45,7 +45,7 @@ def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
"claim_id": "c49566d631226492317d06ad7fdbe1ed32925124",
"claim_sequence": 1,
"decoded_claim": True,
"depth": 1057,
"confirmations": 1057,
"effective_amount": "0.1",
"has_signature": False,
"height": 514081,
@@ -64,10 +64,10 @@ def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
claim_obj.stream.fee.usd = Decimal(fee['amount'])
claim_obj.stream.title = "33rpm"
claim_obj.stream.languages.append("en")
claim_obj.stream.sd_hash = sd_hash
claim_obj.stream.media_type = "image/png"
claim_obj.stream.source.sd_hash = sd_hash
claim_obj.stream.source.media_type = "image/png"
claim['value'] = claim_obj
claim['hex'] = binascii.hexlify(claim_obj.to_bytes()).decode()
claim['protobuf'] = binascii.hexlify(claim_obj.to_bytes()).decode()
async def mock_resolve(*args):
await storage.save_claims([claim])
@@ -192,8 +192,8 @@ class TestStreamManager(BlobExchangeTestBase):
def check_post(event):
self.assertEqual(event['event'], 'Time To First Bytes')
self.assertEqual(event['properties']['error'], 'DownloadSDTimeout')
self.assertEqual(event['properties']['tried_peers_count'], None)
self.assertEqual(event['properties']['active_peer_count'], None)
self.assertEqual(event['properties']['tried_peers_count'], 0)
self.assertEqual(event['properties']['active_peer_count'], 0)
self.assertEqual(event['properties']['use_fixed_peers'], False)
self.assertEqual(event['properties']['added_fixed_peers'], False)
self.assertEqual(event['properties']['fixed_peer_delay'], None)
@@ -213,10 +213,10 @@ class TestStreamManager(BlobExchangeTestBase):
self.stream_manager.analytics_manager._post = check_post
self.assertSetEqual(self.stream_manager.streams, set())
self.assertDictEqual(self.stream_manager.streams, {})
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
stream_hash = stream.stream_hash
self.assertSetEqual(self.stream_manager.streams, {stream})
self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream})
self.assertTrue(stream.running)
self.assertFalse(stream.finished)
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
@@ -225,7 +225,7 @@ class TestStreamManager(BlobExchangeTestBase):
)
self.assertEqual(stored_status, "running")
await self.stream_manager.stop_stream(stream)
await stream.stop()
self.assertFalse(stream.finished)
self.assertFalse(stream.running)
@@ -235,8 +235,8 @@ class TestStreamManager(BlobExchangeTestBase):
)
self.assertEqual(stored_status, "stopped")
await self.stream_manager.start_stream(stream)
await stream.downloader.stream_finished_event.wait()
await stream.save_file(node=self.stream_manager.node)
await stream.finished_writing.wait()
await asyncio.sleep(0, loop=self.loop)
self.assertTrue(stream.finished)
self.assertFalse(stream.running)
@@ -247,7 +247,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.assertEqual(stored_status, "finished")
await self.stream_manager.delete_stream(stream, True)
self.assertSetEqual(self.stream_manager.streams, set())
self.assertDictEqual(self.stream_manager.streams, {})
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
stored_status = await self.client_storage.run_and_return_one_or_none(
"select status from file where stream_hash=?", stream_hash
@@ -257,7 +257,7 @@ class TestStreamManager(BlobExchangeTestBase):
async def _test_download_error_on_start(self, expected_error, timeout=None):
with self.assertRaises(expected_error):
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout=timeout)
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout)
async def _test_download_error_analytics_on_start(self, expected_error, timeout=None):
received = []
@@ -321,9 +321,9 @@ class TestStreamManager(BlobExchangeTestBase):
await self.setup_stream_manager(old_sort=old_sort)
self.stream_manager.analytics_manager._post = check_post
self.assertSetEqual(self.stream_manager.streams, set())
self.assertDictEqual(self.stream_manager.streams, {})
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
await stream.downloader.stream_finished_event.wait()
await stream.finished_writing.wait()
await asyncio.sleep(0, loop=self.loop)
self.stream_manager.stop()
self.client_blob_manager.stop()
@@ -333,8 +333,11 @@ class TestStreamManager(BlobExchangeTestBase):
await self.client_blob_manager.setup()
await self.stream_manager.start()
self.assertEqual(1, len(self.stream_manager.streams))
self.assertEqual(stream.sd_hash, list(self.stream_manager.streams)[0].sd_hash)
self.assertEqual('stopped', list(self.stream_manager.streams)[0].status)
self.assertListEqual([self.sd_hash], list(self.stream_manager.streams.keys()))
for blob_hash in [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]:
blob_status = await self.client_storage.get_blob_status(blob_hash)
self.assertEqual('pending', blob_status)
self.assertEqual('finished', self.stream_manager.streams[self.sd_hash].status)
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
self.assertTrue(sd_blob.file_exists)
+1 -1
View File
@@ -61,7 +61,7 @@ class CLITest(AsyncioTestCase):
# publish is ungrouped command, returns usage only implicitly
self.assertIn('publish (<name> | --name=<name>)', self.shell(['publish']))
# publish is ungrouped command, with explicit --help
self.assertIn('Create or update a stream claim at a given name', self.shell(['publish', '--help']))
self.assertIn('Create or replace a stream claim at a given name', self.shell(['publish', '--help']))
# account is a group, returns help implicitly
self.assertIn('Return the balance of an account', self.shell(['account']))
# account is a group, with explicit --help
+17
View File
@@ -95,3 +95,20 @@ class TestAccount(AsyncioTestCase):
account = Account.from_dict(self.ledger, Wallet(), account_data)
account_data['ledger'] = 'lbc_mainnet'
self.assertDictEqual(account_data, account.to_dict())
async def test_save_max_gap(self):
account = Account.generate(
self.ledger, Wallet(), 'lbryum', {
'name': 'deterministic-chain',
'receiving': {'gap': 3, 'maximum_uses_per_address': 2},
'change': {'gap': 4, 'maximum_uses_per_address': 2}
}
)
self.assertEqual(account.receiving.gap, 3)
self.assertEqual(account.change.gap, 4)
await account.save_max_gap()
self.assertEqual(account.receiving.gap, 20)
self.assertEqual(account.change.gap, 6)
# doesn't fail for single-address account
account2 = Account.generate(self.ledger, Wallet(), 'lbryum', {'name': 'single-address'})
await account2.save_max_gap()