Compare commits

..
355 Commits
Author SHA1 Message Date
Jack Robison 3b0ba3e534 v0.34.0rc2 2019-03-14 16:03:39 -04:00
Jack Robison 8106eb1fa5 Merge pull request #1986 from lbryio/fix-timeout-no-peers
fix default download timeout not being set
2019-03-14 15:47:12 -04:00
Jack Robison c9b2829227 fix/test fixed_peer_delay and added_fixed_peers 2019-03-14 15:08:26 -04:00
Jack Robison 4f2f8b16f3 fix pending downloads not being removed 2019-03-14 13:52:00 -04:00
Jack Robison 2ab2b846b1 fix default download timeout not being set 2019-03-14 13:07:11 -04:00
Jack Robison 2df7284c29 Merge pull request #1985 from lbryio/fix-ttfb-analytics
fix the time to the head blob including the time to the sd blob in analytics
2019-03-14 11:17:37 -04:00
Jack Robison ab5d71ff61 fix the time to the head blob including the time to the sd blob in analytics 2019-03-13 16:24:23 -04:00
Lex Berezhny 99b289f135 v0.34.0rc1 2019-03-12 20:29:10 -04:00
Lex Berezhny 6db02e4362 show build type setting when building 2019-03-12 20:23:10 -04:00
Lex Berezhny 98d8b7d463 sync uses new apply APIs in torba 2019-03-12 20:22:41 -04:00
Lex Berezhny 96f1883fc4 check modified_on 2019-03-12 20:22:41 -04:00
Lex Berezhny a914de155a working sync 2019-03-12 20:22:41 -04:00
Lex Berezhny 2dee0ff0bf wip account manifest command 2019-03-12 20:22:41 -04:00
Jack Robison 993cb43e5b fix tests
-fix cleanup of a failed download
2019-03-12 20:22:01 -04:00
Jack Robison 5d212a0f82 time to first bytes analytics 2019-03-12 20:22:01 -04:00
Jack Robison 0cc602ca53 Merge pull request #1978 from lbryio/sigint
handle sigint
2019-03-12 15:06:08 -04:00
Victor Shyba f390ff144d handle sigint 2019-03-12 13:43:12 -04:00
Jack Robison b266cde511 Merge pull request #1980 from lbryio/batched-channel-name-initialization
use _batched_select to populate channel names when initializing files
2019-03-12 13:41:58 -04:00
Jack Robison d0a1541d0a fix test 2019-03-12 13:00:43 -04:00
Jack Robison 4cbde1b08f use _batched_select to populate channel names when initializing files 2019-03-12 09:58:19 -04:00
Lex Berezhny c6740d8d70 v0.33.0 2019-03-08 20:00:45 -05:00
Lex Berezhny 46358142ed v0.33.0rc1 2019-03-06 22:18:04 -05:00
Lex Berezhny f0f651fc77 rely on torba cffi explicit version 2019-03-05 09:32:43 -05:00
Lex Berezhny 5d428138ed try older cffi on windows build 2019-03-05 09:32:43 -05:00
Brannon King 8d9da61c7d reverted temporary code for full local stack from 2019-03-05 09:32:43 -05:00
Brannon King c326a9382c changed to pull down the newer lbrycrd 2019-03-05 09:32:43 -05:00
Brannon King 5bdd87c904 removed name-to-claim lookup table
added unit test for Greek and made it work


pylint: revert bad move


revert claim_sequence changes


fixed broken test
2019-03-05 09:32:43 -05:00
Jack Robison ad134fe8bb v0.32.4 2019-03-01 17:24:55 -05:00
Jack Robison 2f3c616acd v0.32.4rc1 2019-03-01 16:33:59 -05:00
Jack Robison 3caa2a5039 Merge pull request #1968 from lbryio/analytics
restore old download analytics
2019-03-01 16:08:18 -05:00
Jack Robison e38a0f7a95 logging 2019-03-01 15:26:45 -05:00
Jack Robison 9de3f3ff09 don't block on sending download analytics events 2019-03-01 15:22:19 -05:00
Jack Robison df8934d472 restore old download analytics and update tests 2019-03-01 14:50:06 -05:00
Jack Robison be4a96c1b3 Merge pull request #1965 from lbryio/certifi
fix ssl errors, use certifi to make ssl contexts
2019-02-28 13:14:17 -05:00
Jack Robison d97d978961 use certifi for all operating systems 2019-02-28 12:45:56 -05:00
Jack Robison f52e44b56a typing 2019-02-28 12:42:23 -05:00
Jack Robison 10d73fe480 use certifi 2019-02-28 12:40:11 -05:00
Jack Robison 4fd2d994f2 Merge pull request #1964 from lbryio/update-aiohttp
update aiohttp and pylint
2019-02-28 12:38:17 -05:00
Jack Robison d95903d219 update aiohttp and pylint 2019-02-28 12:04:07 -05:00
Jack Robison 591e66bfbb Merge pull request #1952 from lbryio/no_trap
stop trapping CancelledError
2019-02-27 11:06:02 -05:00
Victor Shyba e047748961 log dropped api request instead of stack trace 2019-02-26 17:42:45 -05:00
Victor Shyba 9f071bbe36 pylint 2019-02-26 17:42:45 -05:00
Victor Shyba 2d8ebe25ed refactor so we can stop trapping CancelledError everywhere 2019-02-26 17:42:45 -05:00
Victor Shyba 7a96e742f2 stop trapping CancelledError 2019-02-26 17:42:45 -05:00
Jack Robison 13c7026b20 Merge pull request #1961 from lbryio/pin-requirements
Pin requirements
2019-02-26 17:42:13 -05:00
Jack Robison 876a3c68dd pin requirements 2019-02-26 17:20:16 -05:00
Jack Robison 573617d6b6 Merge pull request #1957 from lbryio/dht_memory
add LRU to KademliaPeer creation
2019-02-26 10:10:33 -05:00
Victor Shyba e8c9abb944 add LRU to KademliaPeer creation 2019-02-26 09:42:32 -05:00
Jack Robison 8cee627dfa Merge pull request #1959 from lbryio/pin-jsonschema
pin jsonschema version to 2.6.0
2019-02-26 09:41:20 -05:00
Jack Robison ab4a5cee01 pin jsonschema version 2019-02-26 09:23:41 -05:00
Lex Berezhny 5bb0a07fae pip with git does not appear to work in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 5ca594b9fd upgrade pip in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 3910964df1 plyvel no longer installed unless platform is explicitely linux in torba/setup.py 2019-02-22 20:52:42 -05:00
Jack Robison faa1935203 Merge pull request #1946 from lbryio/file-type-fix
lowercase/stripped comparison for known types
2019-02-22 13:07:09 -05:00
Jack Robison ba518e2a0e update test 2019-02-22 11:12:31 -05:00
Thomas Zarebczan 2a82ff7cc5 Update mime_types.py 2019-02-22 11:11:10 -05:00
Jack Robison 673b958020 v0.32.3 2019-02-21 19:38:14 -05:00
Jack Robison 3f4ed0a896 Merge pull request #1951 from lbryio/fix_invalid_blob_decode_startup
Fix invalid blob decode startup
2019-02-21 19:36:02 -05:00
Victor Shyba f49578cdbc fix integration test 2019-02-21 21:14:51 -03:00
Victor Shyba b834f29634 nullify length on blob delete 2019-02-21 21:00:28 -03:00
Victor Shyba 61132312dd test nullify length on blob delete 2019-02-21 21:00:20 -03:00
Victor Shyba fcdd9295b4 delete upon failing to decode JSON from sd blob 2019-02-21 20:53:21 -03:00
Victor Shyba edb5083a5e test that corrupted sd blobs gets delete if fails to decode 2019-02-21 20:53:04 -03:00
Victor Shyba 4e7d88311f raise proper exception on undecode-able blobs 2019-02-21 20:42:51 -03:00
Victor Shyba aa40337ded test that invalid blobs raise proper exception 2019-02-21 20:42:36 -03:00
Jack Robison 690e2e7e0c v0.32.3rc1 2019-02-21 18:37:19 -05:00
Jack Robison 0c67469321 Merge pull request #1950 from lbryio/fix-slow-ping-queue-shutdown
Fix slow ping queue shutdown
2019-02-21 18:12:22 -05:00
Jack Robison fc64d8d1a8 fix slow ping queue shutdown 2019-02-21 17:44:02 -05:00
Jack Robison 7fe9513857 v0.32.2 2019-02-21 14:46:36 -05:00
Jack Robison e76107c318 v0.32.2rc3 2019-02-21 14:28:13 -05:00
Jack Robison 3868dea819 don't join buckets if there is only one 2019-02-21 14:18:48 -05:00
Jack Robison c52541197f Merge pull request #1949 from lbryio/fix-join-buckets
fix cases where buckets could be unjoined after removing a peer
2019-02-21 14:08:34 -05:00
Jack Robison 03cd6b7200 fix cases where buckets could be unjoined after removing a peer 2019-02-21 13:53:35 -05:00
Jack Robison 6915a7a553 v0.32.2rc2 2019-02-21 13:16:32 -05:00
Jack Robison 7ad29b4bfb Merge pull request #1948 from lbryio/duplicates-in-routing
Test and fix edge cases for adding peers to routing
2019-02-21 13:11:34 -05:00
Jack Robison 4294bf742d simplify/fix ping queue 2019-02-21 13:00:23 -05:00
Jack Robison 57ad9f1c52 test_add_peer_after_handle_request 2019-02-21 11:34:25 -05:00
Jack Robison e260de59f1 fix updating/setting contact triple 2019-02-21 10:50:11 -05:00
Jack Robison 154702e6dd fix KBucket.add_peer 2019-02-21 10:50:11 -05:00
Jack Robison d06d998c81 improve KBucket test_add_peer 2019-02-21 10:50:11 -05:00
Jack Robison 8aff860bfe Merge pull request #1945 from lbryio/fix_delete_log
Fix delete log
2019-02-21 10:14:29 -05:00
Victor Shyba d7ba90bbea update ttfb script 2019-02-20 17:10:36 -03:00
Victor Shyba 59325ad8c1 fix delete log 2019-02-20 17:04:08 -03:00
Jack Robison 1f72e27272 improve time_to_first_byte.py 2019-02-20 17:04:08 -03:00
Jack Robison aa41b8b558 Merge pull request #1944 from lbryio/fix-seed-node-startup
fix seed node startup
2019-02-20 15:03:47 -05:00
Jack Robison 2496e7ed44 improve time_to_first_byte.py 2019-02-20 14:43:40 -05:00
Jack Robison ee12985135 dont ping outselves 2019-02-20 14:15:31 -05:00
Jack Robison 4c7f357441 v0.32.2rc1 2019-02-20 12:12:04 -05:00
Jack Robison 8944a36291 Merge pull request #1942 from lbryio/split_buckets_under_index
add split_buckets_under_index config setting for seed nodes
2019-02-20 12:00:46 -05:00
Jack Robison cb5d428d12 add split_buckets_under_index config setting for seed nodes 2019-02-20 11:25:50 -05:00
Jack Robison f175e8826c Merge pull request #1939 from lbryio/missing_await
refactor file_name and full_path as it can be None, fix missing await, stop using blob_manager.storage on stream_manager and use your own
2019-02-20 10:44:35 -05:00
Victor Shyba 38ed212c01 refactor, dont call storage from blob_manager + missing await 2019-02-19 21:45:14 -03:00
Victor Shyba d035c13883 file_path can be None, add helper to check if stream output file exists instead 2019-02-19 21:45:14 -03:00
Jack Robison 82ce2ccee6 update docs 2019-02-19 17:27:31 -05:00
Jack Robison 2cf9b2dd9e Merge pull request #1938 from lbryio/fix-kbucket-indexing
fix KBucket.key_in_range
2019-02-19 15:03:16 -05:00
Jack Robison fcbdaa336f fix KBucket.key_in_range
The range_min and range_max of a KBucket where mistakenly bounding the node id of a peer to be added rather than the distance from the local node id to the peer node id
2019-02-19 13:57:11 -05:00
Lex Berezhny 3ae6137edb v0.32.1 2019-02-18 22:56:35 -05:00
Lex Berezhny 0fdd2ae178 added tx to JSON response from get 2019-02-18 22:53:11 -05:00
Victor Shyba 8c5007fe07 assert amount transferred to correct account 2019-02-18 22:53:11 -05:00
Victor Shyba d2150cf142 fix balance check 2019-02-18 22:53:11 -05:00
Victor Shyba ce2bb22929 add tests for paid downloads 2019-02-18 22:53:11 -05:00
Lex Berezhny 1ce3adb6a0 reduce integration test verbosity 2019-02-18 21:45:37 -05:00
Lex Berezhny c2dc1f3c1d v0.32.1rc1 2019-02-18 17:17:20 -05:00
Victor Shyba b36c22e2f4 unban after 10 seconds, give up after 60 2019-02-18 17:16:10 -05:00
Victor Shyba 71b66ab337 test stall unban 2019-02-18 17:16:10 -05:00
Lex Berezhny 461c3c8408 downloading header from S3 correctly tracks progress 2019-02-18 17:15:40 -05:00
Thomas Zarebczan 03396c1993 remove invalid reflector_component test config 2019-02-18 17:10:42 -05:00
Jack Robison fa7e9923f4 Merge pull request #1932 from lbryio/fix-1930
Fix blob_list for sd_hash including stream terminator
2019-02-18 14:41:00 -05:00
Jack Robison 3f7e466c97 return failure message for invalid blob hash in blob_delete 2019-02-18 12:18:56 -05:00
Jack Robison 4483380352 fix terminator blob hash (None) being included when running blob_list for an sd hash 2019-02-18 12:12:14 -05:00
Lex Berezhny 4a0a6b0243 v0.32.0 2019-02-16 20:17:41 -05:00
Thomas Zarebczan a003e794ba listen on internal port instead of upnp redirect 2019-02-16 18:55:58 -05:00
Lex Berezhny 051fabc180 v0.32.0rc3 2019-02-16 15:07:07 -05:00
Jack Robison ca686041e7 Merge pull request #1925 from lbryio/fix-extension
fix application/x-ext mime types
2019-02-15 21:11:05 -05:00
Jack Robison fc450eb164 add TestMimeTypes, pylint 2019-02-15 20:42:55 -05:00
Thomas Zarebczan 652d7cbd24 consider . extension 2019-02-15 20:40:55 -05:00
Thomas Zarebczan 43b372c0fe fix extension - remove dot
The rest of our extensions don't have a dot. Also, strip off any characters and lowercase it to be safe.
2019-02-15 20:40:55 -05:00
Jack Robison eaacf7e034 Merge pull request #1926 from lbryio/fix_unfair_ban
Fix p2p ignoring good peers
2019-02-15 20:39:49 -05:00
Victor Shyba 576da1bc30 do not keep trying when blob is being written 2019-02-15 21:20:06 -03:00
Victor Shyba 5c217f1544 resovle reflector IP before adding it 2019-02-15 21:20:06 -03:00
Jack Robison cfbf0dbe0f Merge pull request #1919 from lbryio/sync-blobs-on-startup
Sync blobs on startup
2019-02-15 18:40:33 -05:00
Jack Robison fd081496c0 fix blocking on data being written for resumed streams 2019-02-15 17:35:46 -05:00
Jack Robison 32b4405a4c logging
fix https://github.com/lbryio/lbry/issues/1881
2019-02-15 17:26:28 -05:00
Jack Robison 0d558bd552 fix default file list sorting 2019-02-15 17:26:24 -05:00
Jack Robison ca835f3f80 fix 1883 2019-02-15 16:26:13 -05:00
Jack Robison dbb6ba6241 recover streams with missing sd blobs, handle previous sd blob bugs
-test download and recover stream with old key sorting
2019-02-15 15:21:29 -05:00
Jack Robison a228d20137 move store_stream and store_file to standalone functions 2019-02-15 15:21:07 -05:00
Jack Robison f1b60e3ef2 fix get_blobs_for_stream 2019-02-15 15:21:07 -05:00
Jack Robison 39737c790f sync blob files in database with those in directory on startup 2019-02-15 15:21:07 -05:00
Jack Robison dba4a09d38 Merge pull request #1922 from lbryio/supress_invalid_token_error_to_debug
supress invalid token error to debug
2019-02-15 15:19:58 -05:00
Victor Shyba 918122e7cf supress invalid token error to debug 2019-02-15 16:31:47 -03:00
Victor Shyba a49f479a5f move wait to where its needed on assembler 2019-02-15 10:26:05 -05:00
Victor Shyba 320aabbd70 delete duplicated method accident from rebase 2019-02-15 10:26:05 -05:00
Jack Robison 19fdcad5af Merge pull request #1905 from lbryio/faster-file-delete
Faster file delete
2019-02-14 17:23:55 -05:00
Jack Robison 1734e08297 fix test 2019-02-14 16:34:37 -05:00
Jack Robison 9d89f9b834 use executemany instead of batched_operation 2019-02-14 15:57:24 -05:00
Jack Robison 56c41b2fea fix integration test
-cancel running reflector uploads from publishes upon shutdown
-refactor blob.delete to be non-async
-delete blobs synchronously
2019-02-14 15:43:26 -05:00
Jack Robison 9c8593d88e fix file_delete being slow 2019-02-14 12:33:48 -05:00
Jack Robison 85f41887fd don't create BlobFile object when deleting if it doesn't already exist 2019-02-14 12:28:42 -05:00
Mark Michon 8eede6dfc7 Update LICENSE
Updating license to 2019
2019-02-14 08:47:26 -05:00
Victor Shyba 0d8c37a63b resolve uri->url on ttfb 2019-02-14 00:30:24 -05:00
Lex Berezhny d2ba3fdcfb v0.32.0rc2 2019-02-14 00:18:21 -05:00
Victor Shyba be5ab0f307 test get retry 2019-02-14 00:10:44 -05:00
Victor Shyba 04f70d1f31 stop when data timeout 2019-02-14 00:10:44 -05:00
Victor Shyba 60ba07cc17 add tests for incomplete files, update existing one 2019-02-14 00:10:44 -05:00
Victor Shyba dbc8cfe5ae delete incomplete files on assembler stop 2019-02-14 00:10:44 -05:00
Victor Shyba d91e8f5996 revert import to original one + remove print 2019-02-13 23:33:49 -05:00
Victor Shyba b6df2b9d28 missing None for optional field + pylint 2019-02-13 23:33:49 -05:00
Victor Shyba 5c1ab42277 resume should put the stream into wait as done usually on get calls 2019-02-13 23:33:49 -05:00
Victor Shyba bbdc96e762 file_name as a property of managed stream, to reflect internal changes 2019-02-13 23:33:49 -05:00
Victor Shyba fc0c949187 properly update file name on downloader/assembler 2019-02-13 23:33:49 -05:00
Victor Shyba 3fce8e4023 tests resuming downloads with conflicting file names 2019-02-13 23:33:49 -05:00
Victor Shyba 37bb765a2e resume downloads if DHT disabled 2019-02-13 23:33:49 -05:00
hackrush 21ca4f60f7 Stop the daemon on calling lbrynet stop 2019-02-13 14:22:23 -05:00
Victor Shyba 119d498bce apply a timeout on test wait so if it hangs it fails fast 2019-02-13 13:10:27 -05:00
Victor Shyba 8108aa39c1 verify all blobs got transferred 2019-02-13 13:10:27 -05:00
Victor Shyba 0a068011d8 fix chunked response parsing 2019-02-13 13:10:27 -05:00
Victor Shyba 1193d2cea4 adds tests for chunked responses 2019-02-13 13:10:27 -05:00
Lex Berezhny 76ecb5d50d renamed resolve argument url to urls 2019-02-13 11:55:12 -05:00
Lex Berezhny dbc1164562 v0.32.0rc1 2019-02-12 21:35:41 -05:00
Victor Shyba e2170ec83d fix download timeout types 2019-02-12 20:58:31 -05:00
Victor Shyba 266e777655 add integration test for different download errors 2019-02-12 20:58:31 -05:00
Lex Berezhny 68905d82b9 pass conf instead of data_dir to migrate_db, fixes #1898 2019-02-12 18:10:34 -05:00
Lex Berezhny 396c49f7c6 passing get, file list and file delete tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 5e509c9fd6 testing get 2019-02-12 11:54:00 -05:00
Lex Berezhny e6b6a3f55e dedicated transaction commands tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 019af119d9 moved test_cli.py up one level 2019-02-12 11:54:00 -05:00
Lex Berezhny 58da2f16c2 use absolute imports instead of relative 2019-02-12 11:54:00 -05:00
Lex Berezhny 8aa50aa14f cleaned up resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny 65c6c83b51 reusable command testcase 2019-02-12 11:54:00 -05:00
Lex Berezhny f497989871 split out claim commands 2019-02-12 11:54:00 -05:00
Lex Berezhny 5366779b1b moved all other tests out of test_chris45.py and into dedicated files 2019-02-12 11:54:00 -05:00
Lex Berezhny a3f59f041a dedicated integration tests for account commands 2019-02-12 11:54:00 -05:00
Lex Berezhny d560b0c1cd dedicated integration tests for resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny e94229121a explicitly set LEDGER 2019-02-12 11:54:00 -05:00
Lex Berezhny 097291a498 moved wallet integration tests up one level to simplify 2019-02-12 11:54:00 -05:00
Jack Robison afdc44ca13 Merge pull request #1894 from lbryio/fix-max-key-fee
fix max_key_fee not being followed
2019-02-11 16:27:29 -05:00
Jack Robison 6732b302bb logging 2019-02-11 15:31:31 -05:00
Jack Robison c9d54bb049 fix max_key_fee 2019-02-11 15:26:13 -05:00
Alex Grintsvayg f65e70df2f better resolve description 2019-02-11 12:34:57 -05:00
Jack Robison 4541e754dd Merge pull request #1892 from lbryio/batch-query-files
Improve startup time
2019-02-08 20:43:52 -05:00
Jack Robison 9f89c62e8a download full streams in time_to_first_byte.py 2019-02-08 20:14:10 -05:00
Jack Robison 6386815a29 fix reflect_streams not being followed 2019-02-08 20:13:26 -05:00
Jack Robison 0c93090d2c faster is_valid_hashcharacter 2019-02-08 19:57:26 -05:00
Jack Robison 45404b6ad4 faster is_valid_blobhash 2019-02-08 19:54:59 -05:00
Jack Robison aeb2891baa faster population of channel names 2019-02-08 19:51:13 -05:00
Jack Robison 8abfdc82b2 faster blob manager startup 2019-02-08 17:37:35 -05:00
Jack Robison da9b4e317a limit blobs to announce 2019-02-08 16:29:55 -05:00
Jack Robison e414cc5c48 whitespace 2019-02-08 16:11:56 -05:00
Jack Robison 939edb2f18 logging
-call loop.set_debug when using --verbose
2019-02-08 16:06:38 -05:00
Jack Robison 868110a6f2 populate stream manager with a single batch query 2019-02-08 16:06:35 -05:00
Jack Robison 869a8b712b Merge pull request #1889 from lbryio/download_stuc
Refactor peer scoring and maintain connections to fix stuck downloads, remove locking from ping queue
2019-02-08 15:10:13 -05:00
Victor Shyba 9ecec52412 check lock acquired to prevent races 2019-02-08 04:39:28 -03:00
Victor Shyba 8b25acff6e do not wait blob to be written, log errors from races 2019-02-08 04:32:53 -03:00
Victor Shyba e2b06677b5 defer db insert during downloads 2019-02-08 03:45:23 -03:00
Victor Shyba af1619ebfb disable race limiting for now 2019-02-08 03:32:38 -03:00
Victor Shyba 3352e0e4f4 score peers by speed 2019-02-08 03:08:08 -03:00
Victor Shyba 141d68a2cd fix write race condition 2019-02-08 02:39:29 -03:00
Victor Shyba c06ec6cd69 manage connections, reusing them when possible 2019-02-08 02:27:58 -03:00
Victor Shyba 1be5dce30e stop racing during long streams 2019-02-08 01:28:03 -03:00
Victor Shyba 73da223ae1 conf: give up first contact earlier, but let the transfer go longer 2019-02-08 01:05:58 -03:00
Victor Shyba 2d7eb83518 change score calculation, wait for active peers too, simplify peer sorting/keeping 2019-02-08 01:04:38 -03:00
Victor Shyba ad03f91d24 let cancelled errors flow 2019-02-08 01:03:13 -03:00
Victor Shyba 5586a226c2 bypass parser during download 2019-02-07 21:19:31 -03:00
Victor Shyba a616582733 wait on connection tasks 2019-02-07 20:11:57 -03:00
Victor Shyba 43ac928f0b remove dht locks 2019-02-07 20:11:57 -03:00
Lex Berezhny a121110743 v0.31.0 2019-02-06 14:28:40 -05:00
Jack Robison 8bc5bb4e14 Merge pull request #1873 from lbryio/fix_incomplete_downloads_assembly
test incomplete streams, respect real length, delete on incomplete as…
2019-02-06 14:26:40 -05:00
Victor Shyba 3a15ff4bcd test incomplete streams, respect real length, delete on incomplete assembly 2019-02-06 15:56:06 -03:00
Jack Robison 3508da4993 Merge pull request #1866 from lbryio/non-async-close-blob
refactor blob.close to be non-async, speed up deleting blobs and streams
2019-02-06 13:52:37 -05:00
Jack Robison dfdc8eda00 improve test, fix stream_finished_event 2019-02-06 13:32:50 -05:00
Jack Robison 58f6cb71c6 test stream manager 2019-02-06 13:08:57 -05:00
Jack Robison ae11c5bb4b fix setting finished event 2019-02-06 12:18:24 -05:00
Jack Robison f22b5da170 fix raised error for a peer address mismatch 2019-02-06 12:11:22 -05:00
Jack Robison c1c6d5bc99 fix deleting partial downloads when stopped and previous streams when updating a publish 2019-02-06 11:43:05 -05:00
Jack Robison 0e972ec2ae refactor BlobFile.close to be non async 2019-02-06 11:00:57 -05:00
Jack Robison f8dc0f298e Merge pull request #1868 from lbryio/fix_error_response
format error dict on get
2019-02-06 10:37:31 -05:00
Victor Shyba 8b5803c945 format error dict on get 2019-02-06 11:55:12 -03:00
Lex Berezhny 1314f75219 release script shows PRs that did not get included in changelog 2019-02-06 09:51:09 -05:00
Jack Robison be90ec51e4 release-text 2019-02-06 09:00:59 -05:00
Jack Robison e06c903731 Merge pull request #1867 from lbryio/stream-descriptor-validation
Add stream descriptor tests
2019-02-05 21:43:45 -05:00
Jack Robison 76c8447a56 add stream descriptor tests 2019-02-05 21:25:31 -05:00
Jack Robison 82b53e108c Merge pull request #1865 from lbryio/fix-delete-previous-stream
Fix deleting incomplete downloads when they are stopped
2019-02-05 15:43:55 -05:00
Jack Robison 7c8ad0fe30 fix deleting incomplete downloads upon stop
fix deleting previous streams (not assembled/published files) for a claim being updated
2019-02-05 15:33:08 -05:00
Jack Robison 073ffdd049 proto and type args 2019-02-05 13:36:25 -05:00
Jack Robison 7fe211d134 fix resolve_host 2019-02-05 13:31:57 -05:00
Jack Robison 43d424978d Merge pull request #1862 from lbryio/fix_announce
adds announce tests on stream creation/assemble, refactor calls to set announce
2019-02-05 12:58:56 -05:00
Victor Shyba a4ced4dbd2 set verified after callback 2019-02-05 13:17:00 -03:00
Victor Shyba c6b910a5f3 use time_getter on storage 2019-02-05 13:14:59 -03:00
Victor Shyba 63d1ec3c28 fixes from jack comments 2019-02-05 13:01:12 -03:00
Victor Shyba c31e61586d tests: fix value, compare to blobs len 2019-02-05 13:01:12 -03:00
Victor Shyba 7b8f42e6d0 remove dead set_should_announce code 2019-02-05 13:01:12 -03:00
Victor Shyba 2c275efa48 remove redundant calls to should announce 2019-02-05 13:01:12 -03:00
Victor Shyba 30846f932b test that creating a stream marks sd and head to announce 2019-02-05 13:01:12 -03:00
Victor Shyba 6ead932ccb announces naturally, removes redundant calls 2019-02-05 13:01:12 -03:00
Victor Shyba e9b58577dd tests: verify completed and to announce after assembling streams 2019-02-05 13:01:12 -03:00
Victor Shyba f766dbefe0 announce and set completed after inserts 2019-02-05 13:01:12 -03:00
Jack Robison f8a7d2ebe2 add release_texts to release.py 2019-02-05 09:58:42 -05:00
Jack Robison af43b21ef6 Merge pull request #1860 from lbryio/fix-announce
Fix setting blob completed and next announce time
2019-02-04 18:39:10 -05:00
Jack Robison 0685bc5689 include torba in log file 2019-02-04 18:25:47 -05:00
Jack Robison 8cda3143f2 fix setting should_announce and blob status 2019-02-04 18:22:44 -05:00
Jack Robison 5af351a234 reflector logging 2019-02-04 17:33:25 -05:00
Jack Robison b9934a7eb7 Merge pull request #1859 from lbryio/fix_invalid_data_client
handle and log if downloading from bad sendfile implementations
2019-02-04 17:19:57 -05:00
Victor Shyba b82d2c3120 handle and log if downloading from bad sendfile implementations 2019-02-04 19:13:31 -03:00
Jack Robison 8e607fcc2a Merge pull request #1858 from lbryio/fix-blob-sendfile
Fix blob sendfile
2019-02-04 17:12:50 -05:00
Jack Robison 2bce7cecfa fix sendfile on windows 2019-02-04 16:48:48 -05:00
Jack Robison e141cc9e22 Merge pull request #1857 from lbryio/handle-invalid-data-and-fix-cancel-blob-request
Handle invalid data and fix cancel blob request
2019-02-04 16:47:35 -05:00
Jack Robison 8883587cc7 logging 2019-02-04 16:19:27 -05:00
Jack Robison c85991704d catch invalid data error 2019-02-04 16:19:27 -05:00
Jack Robison 377442d9f0 fix request_blob cancellation 2019-02-04 16:19:24 -05:00
Jack Robison ad30b7df6d Merge pull request #1855 from lbryio/findvalue-always-include-close-contacts
Always return close peers from findValue
2019-02-04 15:30:17 -05:00
Jack Robison f0b49fc153 always return close peers from findValue 2019-02-04 15:07:42 -05:00
Jack Robison 6033399f05 Merge pull request #1854 from lbryio/no-self-queries
fix trying downloading from oneself, prune querying peer from findValue results
2019-02-04 15:00:13 -05:00
Jack Robison ae1162f5b0 don't try downloading from oneself
-don't return the querying peer in results for findValue
2019-02-04 14:50:53 -05:00
Jack Robison 7d7d4c35c3 Merge pull request #1852 from lbryio/close_protocol_reflector
refactor reflect to close protocol
2019-02-04 14:42:18 -05:00
Victor Shyba ad71a9a89d refactor reflect to close protocol 2019-02-04 13:34:18 -03:00
Jack Robison 13b3e4ba8e v0.31.0rc5 2019-02-04 10:22:43 -05:00
Jack Robison 43799168ba Merge pull request #1850 from lbryio/resume
Properly resume streams and fixup with statement
2019-02-04 10:18:31 -05:00
Victor Shyba 736a949cb5 resume streams on startup 2019-02-03 20:49:09 -03:00
Victor Shyba 11bb1cc7ee use with clause to avoid leaking fds 2019-02-03 20:41:47 -03:00
Jack Robison dbce49e567 Merge pull request #1849 from lbryio/fix-connection-status
Fix connection status
2019-02-03 17:00:14 -05:00
Jack Robison dcd78bd86e always delete file from downloads if download was not complete 2019-02-03 16:43:40 -05:00
Jack Robison e5af041627 fix connection status message 2019-02-03 16:20:52 -05:00
Jack Robison 73f4e954e5 Merge pull request #1848 from lbryio/fix-announce-time
Fix setting blob announce time
2019-02-03 16:18:33 -05:00
Jack Robison 0a027cf2de logging 2019-02-03 15:47:19 -05:00
Jack Robison 3f98b16800 fix blob announce time, use time.time instead of loop.time 2019-02-03 15:47:15 -05:00
Jack Robison bbadece75d remove unused packages from extension-pkg-whitelist in pylintrc 2019-02-03 15:02:43 -05:00
Jack Robison 8f5a8ab8ba fix peer_ping 2019-02-02 02:02:19 -05:00
Jack Robison 91e581422c v0.31.0rc4 2019-02-02 00:30:57 -05:00
Jack Robison b54d7185a1 Merge pull request #1847 from lbryio/re-reflect
add re-reflect task to stream manager, add concurrent_reflector_uploads to config
2019-02-01 23:11:06 -05:00
Jack Robison f2968aab22 add re-reflect task to stream manager, add concurrent_reflector_uploads to config 2019-02-01 22:59:41 -05:00
Jack Robison 4e27eaf8e6 Merge pull request #1846 from lbryio/create-directories-if-dont-exist
create data. download, and wallet directories if they don't exist on startup
2019-02-01 22:57:27 -05:00
Jack Robison ccbe8aa0eb create data. download, and wallet directories if they don't exist on startup 2019-02-01 22:45:28 -05:00
Jack Robison 478f79353a fix bytes error 2019-02-01 22:20:11 -05:00
Jack Robison c3b0151943 fix dht status error during startup 2019-02-01 22:12:07 -05:00
Jack Robison ba818354af Merge pull request #1844 from lbryio/reassemble-when-deleted-from-downloads
Reassemble when deleted from downloads
2019-02-01 21:59:36 -05:00
Jack Robison e96b75a0d0 fix file/download paths
move download-from-uri logic into stream manager
2019-02-01 21:40:35 -05:00
Jack Robison efe4afd09e fix blob announce time 2019-02-01 21:28:59 -05:00
Jack Robison c7cb6822cc logging 2019-02-01 21:28:59 -05:00
Jack Robison c75665d3f0 update file_list docs, add blobs_remaining to file list args and to file dict result 2019-02-01 21:28:59 -05:00
Jack Robison 744375b2c0 re-assemble file / resume downloads 2019-02-01 21:28:59 -05:00
Jack Robison c1b4a012ec Merge pull request #1845 from lbryio/stuck_peers_fixup
fix active peers not being removed after closing + parsing of data stream
2019-02-01 21:28:20 -05:00
Victor Shyba 40da5b6cb6 rework data parser 2019-02-01 20:09:14 -03:00
Victor Shyba b36cf3e36c remove closing peers from active connections 2019-02-01 20:09:14 -03:00
Jack Robison 4e3b529aec Merge pull request #1840 from lbryio/close-transport-on-sendfile-fail
close blob server transport on sendfile failing
2019-02-01 15:13:07 -05:00
Jack Robison 0a08f445a4 close blob server transport on sendfile failing 2019-02-01 15:11:37 -05:00
Jack Robison 3eac99c509 Merge pull request #1841 from lbryio/general_fixes_downloader
General fixes downloader
2019-02-01 15:10:36 -05:00
Victor Shyba f5bf1463af fixes from review 2019-02-01 17:09:37 -03:00
Victor Shyba ca4a55ef28 set verified event earlier and remove stop awaits 2019-02-01 16:05:14 -03:00
Victor Shyba 832be0758b let cancel propagate 2019-02-01 16:05:14 -03:00
Victor Shyba 25050fdeeb fixes for writers (check inside lock and dont hold verified bytes) 2019-02-01 16:05:14 -03:00
Jack Robison df0635103e Merge pull request #1839 from lbryio/faster-stream-manager-startup
faster stream manager startup
2019-02-01 14:00:19 -05:00
Jack Robison dabd9cbe1e faster stream manager startup 2019-02-01 13:53:03 -05:00
Jack Robison 42d7fbdbb3 Merge pull request #1838 from lbryio/fix-update-claim-missing-stream
Fix publishing an update where we don't have the previous stream
2019-02-01 13:52:34 -05:00
Jack Robison 2336015f9a fix publish
fixes https://github.com/lbryio/lbry/issues/1826
- remove `sources` argument from `publish`
- only add/update files we have
- delete existing stream if updating with a new one
2019-02-01 13:35:18 -05:00
Jack Robison 5a28128b69 Merge pull request #1835 from lbryio/remove-peer-manager-lock
Remove peer manager lock
2019-01-31 21:04:21 -05:00
Jack Robison 31445c7797 remove lock from PeerManager 2019-01-31 20:57:48 -05:00
Jack Robison 16d0ff8376 Merge pull request #1833 from lbryio/fix-publish-update
Fix publish when making an update with an existing stream
2019-01-31 20:56:34 -05:00
Jack Robison 87f5d1bd4a fix updating claim with existing stream 2019-01-31 20:49:19 -05:00
Jack Robison be6e8744b0 Merge pull request #1834 from lbryio/remove-blob-request-lock
remove blob request asyncio.Lock
2019-01-31 20:46:01 -05:00
Jack Robison 2e9acd1baa remove blob request asyncio.Lock 2019-01-31 20:19:50 -05:00
Jack Robison 607ccefe90 Merge pull request #1831 from lbryio/cleanup-downloader
Download and publish fixes
2019-01-31 18:17:26 -05:00
Jack Robison df86937f4e fix deleting a file mid download 2019-01-31 17:59:42 -05:00
Jack Robison 957d0290df add torba logging 2019-01-31 16:53:54 -05:00
Jack Robison 40b90ce79b fix hanging download 2019-01-31 16:53:27 -05:00
Jack Robison aa7f0845cc catch sendfile errors 2019-01-31 16:10:27 -05:00
Jack Robison 6e69d306de tests 2019-01-31 15:39:23 -05:00
Jack Robison d85d3dfaa4 increase default max_connections_per_download 2019-01-31 15:39:20 -05:00
Jack Robison 2b035009ef handle both urls and ip addresses for fixed nodes and reflector servers 2019-01-31 15:39:20 -05:00
Jack Robison f9fd62c214 add fixed peers immediately if the dht has no peers 2019-01-31 15:39:20 -05:00
Jack Robison e8a1953815 add fixed peers immediately if dht is skipped 2019-01-31 15:39:20 -05:00
Jack Robison 34776e7eb9 fix test 2019-01-31 15:39:20 -05:00
Jack Robison a4419f562f pylint 2019-01-31 15:39:20 -05:00
Jack Robison c0160ab4e9 update time_to_first_byte.py 2019-01-31 15:39:20 -05:00
Jack Robison e60e5b3919 fix updating content claim for a file
-fix file name and download dir not being hex encoded during publish, add repair script
2019-01-31 15:39:20 -05:00
Jack Robison 3589cc9977 call update_reflected_stream upon reflector finishing 2019-01-31 15:39:20 -05:00
Jack Robison 2e978c00b2 prioritize peers to download from 2019-01-31 15:39:20 -05:00
Jack Robison 2335243de9 pylint 2019-01-31 15:39:20 -05:00
Jack Robison 6ad68eabd9 tests 2019-01-31 15:39:20 -05:00
Jack Robison ca5c638124 tests 2019-01-31 15:39:20 -05:00
Jack Robison 262b9a624b update file_reflect 2019-01-31 15:39:20 -05:00
Jack Robison 05915c540a set fully reflected if the server indicates it had all of the stream 2019-01-31 15:39:20 -05:00
Jack Robison 6aef6a80b7 refactor downloader
split peer accumulation out, use Queues instead of locks
2019-01-31 15:39:20 -05:00
Jack Robison 16efe9ba95 add fixed_peer_delay setting 2019-01-31 15:39:20 -05:00
Jack Robison fcda9500fe add accumulate_peers to Node
simplifies collecting peers during a download by giving a queue of hashes to search for and a queue or peers as they are found
2019-01-31 15:39:20 -05:00
Alex Grin 64dec821b7 Update README.md 2019-01-31 14:24:44 -05:00
Jack Robison d5f6e58aad Update README.md 2019-01-31 14:15:38 -05:00
Jack Robison 087b65522e Merge pull request #1830 from lbryio/dht_fixes
Dht fixes
2019-01-31 13:52:00 -05:00
Victor Shyba 5ee4b2173a fix probes bugs, partial DHT refactor 2019-01-31 01:49:52 -03:00
Victor Shyba 3e1c3d1e4b use node_id parameter instead of your own 2019-01-31 01:11:09 -03:00
Victor Shyba 21f2b2dcd7 cast timestamp to integer before querying 2019-01-31 01:09:57 -03:00
Jack Robison ffc7fc0793 fix uncaught exception in BlobServerProtocol when client disconnects during a blob transfer 2019-01-30 12:03:05 -05:00
Jack Robison 15dbe5e54e Merge pull request #1828 from lbryio/fix_chunked
fix chunked requests for server
2019-01-30 10:25:30 -05:00
Victor Shyba 2e633c0ee6 fix chunked requests for server 2019-01-30 12:11:05 -03:00
Jack Robison 933d58d49b logging 2019-01-29 20:47:02 -05:00
Jack Robison 8f7cf0b38f verbose logging 2019-01-29 13:08:11 -05:00
Jack Robison adf378b6e9 Merge pull request #1825 from lbryio/fix-hanging-download
Fix hanging download
2019-01-28 23:44:42 -05:00
Jack Robison 1a4f083e76 skip test 2019-01-28 22:50:06 -05:00
Jack Robison 7bde09dcf8 add download_blob_from_peer.py 2019-01-28 22:41:16 -05:00
Jack Robison 4630fd38aa fix async generator junction tests 2019-01-28 22:39:36 -05:00
Jack Robison 418def56ac fix log 2019-01-28 22:38:17 -05:00
Jack Robison cefe3eb520 fix file_delete 2019-01-28 22:01:43 -05:00
Jack Robison 64dffa306f logging, cleanup 2019-01-28 22:01:43 -05:00
Jack Robison 7d33b4f1f3 fix unawaited task 2019-01-28 22:01:43 -05:00
Jack Robison f507d95198 always add fixed peers if configured after 2s 2019-01-28 22:01:43 -05:00
Jack Robison 2a7d80f8b4 logging 2019-01-28 22:01:43 -05:00
Jack Robison 305034ffb2 Merge pull request #1824 from lbryio/fix-ping-timeout
fix timeout error
2019-01-28 21:53:32 -05:00
Jack Robison bfaebf0085 fix timeout error 2019-01-28 17:20:14 -05:00
Jack Robison a94cf04e87 Merge pull request #1822 from lbryio/update-settings
Update settings
2019-01-28 17:17:56 -05:00
Jack Robison 71f9f8ae9c fix race condition in reflector server 2019-01-28 16:51:11 -05:00
Jack Robison 330862e487 fix https://github.com/lbryio/lbry/issues/1297 2019-01-28 16:00:37 -05:00
Jack Robison 778d3826ab clean up settings and use them 2019-01-28 15:51:27 -05:00
Jack Robison 4a11cf007f remove unused settings and constants 2019-01-27 19:51:44 -05:00
Jack Robison f6e7700c4c Merge pull request #1818 from lbryio/fix-find-value
fix error encoding find value response
2019-01-26 17:26:23 -05:00
Jack Robison 573285a5bb fix error encoding find value response 2019-01-26 17:04:11 -05:00
118 changed files with 4627 additions and 5510 deletions

No files matched your search

+1
View File
@@ -3,6 +3,7 @@
/.tox
/.idea
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
+1 -3
View File
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
miniupnpc,
unqlite
# extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
+1 -13
View File
@@ -9,9 +9,7 @@ jobs:
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
@@ -106,16 +104,6 @@ jobs:
before_install:
- brew upgrade python || true
- brew upgrade python || true
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2018 LBRY Inc
Copyright (c) 2015-2019 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
+3 -3
View File
@@ -1,12 +1,12 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
-5
View File
@@ -1,5 +0,0 @@
build:
python setup.py apk
adb install -r lbry-0.1-service-debug.apk
.PHONY: build
-49
View File
@@ -1,49 +0,0 @@
import inspect
from os.path import dirname, join
import sh
from pythonforandroid.util import ensure_dir
from pythonforandroid.toolchain import (
Bootstrap, current_directory, info, info_main, shprint
)
class LBRYServiceBootstrap(Bootstrap):
name = 'lbry-service'
recipe_depends = ['genericndkbuild', 'python3']
bootstrap_dir = dirname(__file__)
def get_common_dir(self):
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
info('This currently just copies the build stuff straight from the build dir.')
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
python_bundle_dir = join('_python_bundle', '_python_bundle')
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super().run_distribute()
-89
View File
@@ -1,89 +0,0 @@
# prevent user to include invalid extensions
*.apk
*.pxd
# eggs
*.egg-info
# unit test
unittest/*
# python config
config/makesetup
# unused kivy files (platform specific)
kivy/input/providers/wm_*
kivy/input/providers/mactouch*
kivy/input/providers/probesysfs*
kivy/input/providers/mtdev*
kivy/input/providers/hidinput*
kivy/core/camera/camera_videocapture*
kivy/core/spelling/*osx*
kivy/core/video/video_pyglet*
kivy/tools
kivy/tests/*
kivy/*/*.h
kivy/*/*.pxi
# unused encodings
lib-dynload/*codec*
encodings/cp*.pyo
encodings/tis*
encodings/shift*
encodings/bz2*
encodings/iso*
encodings/undefined*
encodings/johab*
encodings/p*
encodings/m*
encodings/euc*
encodings/k*
encodings/unicode_internal*
encodings/quo*
encodings/gb*
encodings/big5*
encodings/hp*
encodings/hz*
# unused python modules
bsddb/*
wsgiref/*
hotshot/*
pydoc_data/*
tty.pyo
anydbm.pyo
nturl2path.pyo
LICENCE.txt
macurl2path.pyo
dummy_threading.pyo
audiodev.pyo
antigravity.pyo
dumbdbm.pyo
sndhdr.pyo
__phello__.foo.pyo
sunaudio.pyo
os2emxpath.pyo
multiprocessing/dummy*
# unused binaries python modules
lib-dynload/termios.so
lib-dynload/_lsprof.so
lib-dynload/*audioop.so
lib-dynload/_hotshot.so
lib-dynload/_heapq.so
lib-dynload/_json.so
lib-dynload/grp.so
lib-dynload/resource.so
lib-dynload/pyexpat.so
lib-dynload/_ctypes_test.so
lib-dynload/_testcapi.so
# odd files
plat-linux3/regen
#>sqlite3
# conditionnal include depending if some recipes are included or not.
sqlite3/*
lib-dynload/_sqlite3.so
#<sqlite3
-1
View File
@@ -1 +0,0 @@
include $(call all-subdir-makefiles)
@@ -1,7 +0,0 @@
# Uncomment this if you're using STL in your project
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
# APP_STL := stlport_static
# APP_ABI := armeabi armeabi-v7a x86
APP_ABI := $(ARCH)
@@ -1,22 +0,0 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
# Add your application source files here...
LOCAL_SRC_FILES := start.c pyjniusjni.c
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
LOCAL_SHARED_LIBRARIES := python_shared
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
include $(BUILD_SHARED_LIBRARY)
ifdef CRYSTAX_PYTHON_VERSION
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
endif
@@ -1,10 +0,0 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
LOCAL_SRC_FILES := YourSourceHere.c
include $(BUILD_SHARED_LIBRARY)
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
@@ -1,6 +0,0 @@
#define BOOTSTRAP_NAME_SERVICEONLY
#define BOOTSTRAP_USES_NO_SDL_HEADERS
const char bootstrap_name[] = "service_only";
@@ -1,103 +0,0 @@
#include <pthread.h>
#include <jni.h>
#define LOGI(...) do {} while (0)
#define LOGE(...) do {} while (0)
#include "android/log.h"
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
/* #define LOGP(x) LOG("python", (x)) */
#define LOG_TAG "Python_android"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
/* Function headers */
JNIEnv* Android_JNI_GetEnv(void);
static void Android_JNI_ThreadDestroyed(void*);
static pthread_key_t mThreadKey;
static JavaVM* mJavaVM;
int Android_JNI_SetupThread(void)
{
Android_JNI_GetEnv();
return 1;
}
/* Library init */
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
{
JNIEnv *env;
mJavaVM = vm;
LOGI("JNI_OnLoad called");
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
LOGE("Failed to get the environment using GetEnv()");
return -1;
}
/*
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
*/
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
}
Android_JNI_SetupThread();
return JNI_VERSION_1_4;
}
JNIEnv* Android_JNI_GetEnv(void)
{
/* From http://developer.android.com/guide/practices/jni.html
* All threads are Linux threads, scheduled by the kernel.
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
* and cannot make JNI calls.
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
* is a no-op.
* Note: You can call this function any number of times for the same thread, there's no harm in it
*/
JNIEnv *env;
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
if(status < 0) {
LOGE("failed to attach current thread");
return 0;
}
/* From http://developer.android.com/guide/practices/jni.html
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
* Note: The destructor is not called unless the stored value is != NULL
* Note: You can call this function any number of times for the same thread, there's no harm in it
* (except for some lost CPU cycles)
*/
pthread_setspecific(mThreadKey, (void*) env);
return env;
}
static void Android_JNI_ThreadDestroyed(void* value)
{
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
JNIEnv *env = (JNIEnv*) value;
if (env != NULL) {
(*mJavaVM)->DetachCurrentThread(mJavaVM);
pthread_setspecific(mThreadKey, NULL);
}
}
void *WebView_AndroidGetJNIEnv()
{
return Android_JNI_GetEnv();
}
Whitespace-only changes.
@@ -1,141 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class Octal {
/**
* Parse an octal string from a header buffer. This is used for the file
* permission mode value.
*
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal string.
*/
public static long parseOctal(byte[] header, int offset, int length) {
long result = 0;
boolean stillPadding = true;
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
if (header[i] == (byte) ' ' || header[i] == '0') {
if (stillPadding)
continue;
if (header[i] == (byte) ' ')
break;
}
stillPadding = false;
result = ( result << 3 ) + ( header[i] - '0' );
}
return result;
}
/**
* Parse an octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The integer value of the octal bytes.
*/
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
int idx = length - 1;
buf[offset + idx] = 0;
--idx;
buf[offset + idx] = (byte) ' ';
--idx;
if (value == 0) {
buf[offset + idx] = (byte) '0';
--idx;
} else {
for (long val = value; idx >= 0 && val > 0; --idx) {
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
val = val >> 3;
}
}
for (; idx >= 0; --idx) {
buf[offset + idx] = (byte) ' ';
}
return offset + length;
}
/**
* Parse the checksum octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The integer value of the entry's checksum.
*/
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
getOctalBytes( value, buf, offset, length );
buf[offset + length - 1] = (byte) ' ';
buf[offset + length - 2] = 0;
return offset + length;
}
/**
* Parse an octal long integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal bytes.
*/
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
byte[] temp = new byte[length + 1];
getOctalBytes( value, temp, 0, length + 1 );
System.arraycopy( temp, 0, buf, offset, length );
return offset + length;
}
}
@@ -1,28 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class TarConstants {
public static final int EOF_BLOCK = 1024;
public static final int DATA_BLOCK = 512;
public static final int HEADER_BLOCK = 512;
}
@@ -1,284 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
import java.util.Date;
/**
* @author Kamran Zafar
*
*/
public class TarEntry {
protected File file;
protected TarHeader header;
private TarEntry() {
this.file = null;
header = new TarHeader();
}
public TarEntry(File file, String entryName) {
this();
this.file = file;
this.extractTarHeader(entryName);
}
public TarEntry(byte[] headerBuf) {
this();
this.parseTarHeader(headerBuf);
}
/**
* Constructor to create an entry from an existing TarHeader object.
*
* This method is useful to add new entries programmatically (e.g. for
* adding files or directories that do not exist in the file system).
*
* @param header
*
*/
public TarEntry(TarHeader header) {
this.file = null;
this.header = header;
}
public boolean equals(TarEntry it) {
return header.name.toString().equals(it.header.name.toString());
}
public boolean isDescendent(TarEntry desc) {
return desc.header.name.toString().startsWith(header.name.toString());
}
public TarHeader getHeader() {
return header;
}
public String getName() {
String name = header.name.toString();
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
name = header.namePrefix.toString() + "/" + name;
}
return name;
}
public void setName(String name) {
header.name = new StringBuffer(name);
}
public int getUserId() {
return header.userId;
}
public void setUserId(int userId) {
header.userId = userId;
}
public int getGroupId() {
return header.groupId;
}
public void setGroupId(int groupId) {
header.groupId = groupId;
}
public String getUserName() {
return header.userName.toString();
}
public void setUserName(String userName) {
header.userName = new StringBuffer(userName);
}
public String getGroupName() {
return header.groupName.toString();
}
public void setGroupName(String groupName) {
header.groupName = new StringBuffer(groupName);
}
public void setIds(int userId, int groupId) {
this.setUserId(userId);
this.setGroupId(groupId);
}
public void setModTime(long time) {
header.modTime = time / 1000;
}
public void setModTime(Date time) {
header.modTime = time.getTime() / 1000;
}
public Date getModTime() {
return new Date(header.modTime * 1000);
}
public File getFile() {
return this.file;
}
public long getSize() {
return header.size;
}
public void setSize(long size) {
header.size = size;
}
/**
* Checks if the org.kamrazafar.jtar entry is a directory
*
* @return
*/
public boolean isDirectory() {
if (this.file != null)
return this.file.isDirectory();
if (header != null) {
if (header.linkFlag == TarHeader.LF_DIR)
return true;
if (header.name.toString().endsWith("/"))
return true;
}
return false;
}
/**
* Extract header from File
*
* @param entryName
*/
public void extractTarHeader(String entryName) {
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
}
/**
* Calculate checksum
*
* @param buf
* @return
*/
public long computeCheckSum(byte[] buf) {
long sum = 0;
for (int i = 0; i < buf.length; ++i) {
sum += 255 & buf[i];
}
return sum;
}
/**
* Writes the header to the byte buffer
*
* @param outbuf
*/
public void writeEntryHeader(byte[] outbuf) {
int offset = 0;
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
long size = header.size;
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
int csOffset = offset;
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
outbuf[offset++] = (byte) ' ';
outbuf[offset++] = header.linkFlag;
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
for (; offset < outbuf.length;)
outbuf[offset++] = 0;
long checkSum = this.computeCheckSum(outbuf);
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
}
/**
* Parses the tar header to the byte buffer
*
* @param header
* @param bh
*/
public void parseTarHeader(byte[] bh) {
int offset = 0;
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
offset += TarHeader.MODELEN;
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
offset += TarHeader.UIDLEN;
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
offset += TarHeader.GIDLEN;
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
offset += TarHeader.SIZELEN;
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
offset += TarHeader.MODTIMELEN;
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
offset += TarHeader.CHKSUMLEN;
header.linkFlag = bh[offset++];
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
offset += TarHeader.USTAR_MAGICLEN;
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
offset += TarHeader.USTAR_USER_NAMELEN;
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset += TarHeader.USTAR_GROUP_NAMELEN;
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
}
}
@@ -1,243 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* Header
*
* <pre>
* Offset Size Field
* 0 100 File name
* 100 8 File mode
* 108 8 Owner's numeric user ID
* 116 8 Group's numeric user ID
* 124 12 File size in bytes
* 136 12 Last modification time in numeric Unix time format
* 148 8 Checksum for header block
* 156 1 Link indicator (file type)
* 157 100 Name of linked file
* </pre>
*
*
* File Types
*
* <pre>
* Value Meaning
* '0' Normal file
* (ASCII NUL) Normal file (now obsolete)
* '1' Hard link
* '2' Symbolic link
* '3' Character special
* '4' Block special
* '5' Directory
* '6' FIFO
* '7' Contigous
* </pre>
*
*
*
* Ustar header
*
* <pre>
* Offset Size Field
* 257 6 UStar indicator "ustar"
* 263 2 UStar version "00"
* 265 32 Owner user name
* 297 32 Owner group name
* 329 8 Device major number
* 337 8 Device minor number
* 345 155 Filename prefix
* </pre>
*/
public class TarHeader {
/*
* Header
*/
public static final int NAMELEN = 100;
public static final int MODELEN = 8;
public static final int UIDLEN = 8;
public static final int GIDLEN = 8;
public static final int SIZELEN = 12;
public static final int MODTIMELEN = 12;
public static final int CHKSUMLEN = 8;
public static final byte LF_OLDNORM = 0;
/*
* File Types
*/
public static final byte LF_NORMAL = (byte) '0';
public static final byte LF_LINK = (byte) '1';
public static final byte LF_SYMLINK = (byte) '2';
public static final byte LF_CHR = (byte) '3';
public static final byte LF_BLK = (byte) '4';
public static final byte LF_DIR = (byte) '5';
public static final byte LF_FIFO = (byte) '6';
public static final byte LF_CONTIG = (byte) '7';
/*
* Ustar header
*/
public static final String USTAR_MAGIC = "ustar"; // POSIX
public static final int USTAR_MAGICLEN = 8;
public static final int USTAR_USER_NAMELEN = 32;
public static final int USTAR_GROUP_NAMELEN = 32;
public static final int USTAR_DEVLEN = 8;
public static final int USTAR_FILENAME_PREFIX = 155;
// Header values
public StringBuffer name;
public int mode;
public int userId;
public int groupId;
public long size;
public long modTime;
public int checkSum;
public byte linkFlag;
public StringBuffer linkName;
public StringBuffer magic; // ustar indicator and version
public StringBuffer userName;
public StringBuffer groupName;
public int devMajor;
public int devMinor;
public StringBuffer namePrefix;
public TarHeader() {
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
this.name = new StringBuffer();
this.linkName = new StringBuffer();
String user = System.getProperty("user.name", "");
if (user.length() > 31)
user = user.substring(0, 31);
this.userId = 0;
this.groupId = 0;
this.userName = new StringBuffer(user);
this.groupName = new StringBuffer("");
this.namePrefix = new StringBuffer();
}
/**
* Parse an entry name from a header buffer.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The header's entry name.
*/
public static StringBuffer parseName(byte[] header, int offset, int length) {
StringBuffer result = new StringBuffer(length);
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
result.append((char) header[i]);
}
return result;
}
/**
* Determine the number of bytes in an entry name.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The number of bytes in a header's entry name.
*/
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
int i;
for (i = 0; i < length && i < name.length(); ++i) {
buf[offset + i] = (byte) name.charAt(i);
}
for (; i < length; ++i) {
buf[offset + i] = 0;
}
return offset + length;
}
/**
* Creates a new header for a file/directory entry.
*
*
* @param name
* File name
* @param size
* File size in bytes
* @param modTime
* Last modification time in numeric Unix time format
* @param dir
* Is directory
*
* @return
*/
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
String name = entryName;
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
TarHeader header = new TarHeader();
header.linkName = new StringBuffer("");
if (name.length() > 100) {
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
} else {
header.name = new StringBuffer(name);
}
if (dir) {
header.mode = 040755;
header.linkFlag = TarHeader.LF_DIR;
if (header.name.charAt(header.name.length() - 1) != '/') {
header.name.append("/");
}
header.size = 0;
} else {
header.mode = 0100644;
header.linkFlag = TarHeader.LF_NORMAL;
header.size = size;
}
header.modTime = modTime;
header.checkSum = 0;
header.devMajor = 0;
header.devMinor = 0;
return header;
}
}
@@ -1,249 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* @author Kamran Zafar
*
*/
public class TarInputStream extends FilterInputStream {
private static final int SKIP_BUFFER_SIZE = 2048;
private TarEntry currentEntry;
private long currentFileSize;
private long bytesRead;
private boolean defaultSkip = false;
public TarInputStream(InputStream in) {
super(in);
currentFileSize = 0;
bytesRead = 0;
}
@Override
public boolean markSupported() {
return false;
}
/**
* Not supported
*
*/
@Override
public synchronized void mark(int readlimit) {
}
/**
* Not supported
*
*/
@Override
public synchronized void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
/**
* Read a byte
*
* @see java.io.FilterInputStream#read()
*/
@Override
public int read() throws IOException {
byte[] buf = new byte[1];
int res = this.read(buf, 0, 1);
if (res != -1) {
return 0xFF & buf[0];
}
return res;
}
/**
* Checks if the bytes being read exceed the entry size and adjusts the byte
* array length. Updates the byte counters
*
*
* @see java.io.FilterInputStream#read(byte[], int, int)
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (currentEntry != null) {
if (currentFileSize == currentEntry.getSize()) {
return -1;
} else if ((currentEntry.getSize() - currentFileSize) < len) {
len = (int) (currentEntry.getSize() - currentFileSize);
}
}
int br = super.read(b, off, len);
if (br != -1) {
if (currentEntry != null) {
currentFileSize += br;
}
bytesRead += br;
}
return br;
}
/**
* Returns the next entry in the tar file
*
* @return TarEntry
* @throws IOException
*/
public TarEntry getNextEntry() throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
int tr = 0;
// Read full header
while (tr < TarConstants.HEADER_BLOCK) {
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
if (res < 0) {
break;
}
System.arraycopy(theader, 0, header, tr, res);
tr += res;
}
// Check if record is null
boolean eof = true;
for (byte b : header) {
if (b != 0) {
eof = false;
break;
}
}
if (!eof) {
currentEntry = new TarEntry(header);
}
return currentEntry;
}
/**
* Returns the current offset (in bytes) from the beginning of the stream.
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
*/
public long getCurrentOffset() {
return bytesRead;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
// Not fully read, skip rest of the bytes
long bs = 0;
while (bs < currentEntry.getSize() - currentFileSize) {
long res = skip(currentEntry.getSize() - currentFileSize - bs);
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
// I suspect file corruption
throw new IOException("Possible tar file corruption");
}
bs += res;
}
}
currentEntry = null;
currentFileSize = 0L;
skipPad();
}
}
/**
* Skips the pad at the end of each tar entry file content
*
* @throws IOException
*/
protected void skipPad() throws IOException {
if (bytesRead > 0) {
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
if (extra > 0) {
long bs = 0;
while (bs < TarConstants.DATA_BLOCK - extra) {
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
bs += res;
}
}
}
}
/**
* Skips 'n' bytes on the InputStream<br>
* Overrides default implementation of skip
*
*/
@Override
public long skip(long n) throws IOException {
if (defaultSkip) {
// use skip method of parent stream
// may not work if skip not implemented by parent
long bs = super.skip(n);
bytesRead += bs;
return bs;
}
if (n <= 0) {
return 0;
}
long left = n;
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
while (left > 0) {
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
if (res < 0) {
break;
}
left -= res;
}
return n - left;
}
public boolean isDefaultSkip() {
return defaultSkip;
}
public void setDefaultSkip(boolean defaultSkip) {
this.defaultSkip = defaultSkip;
}
}
@@ -1,163 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
/**
* @author Kamran Zafar
*
*/
public class TarOutputStream extends OutputStream {
private final OutputStream out;
private long bytesWritten;
private long currentFileSize;
private TarEntry currentEntry;
public TarOutputStream(OutputStream out) {
this.out = out;
bytesWritten = 0;
currentFileSize = 0;
}
public TarOutputStream(final File fout) throws FileNotFoundException {
this.out = new BufferedOutputStream(new FileOutputStream(fout));
bytesWritten = 0;
currentFileSize = 0;
}
/**
* Opens a file for writing.
*/
public TarOutputStream(final File fout, final boolean append) throws IOException {
@SuppressWarnings("resource")
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
final long fileSize = fout.length();
if (append && fileSize > TarConstants.EOF_BLOCK) {
raf.seek(fileSize - TarConstants.EOF_BLOCK);
}
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
}
/**
* Appends the EOF record and closes the stream
*
* @see java.io.FilterOutputStream#close()
*/
@Override
public void close() throws IOException {
closeCurrentEntry();
write( new byte[TarConstants.EOF_BLOCK] );
out.close();
}
/**
* Writes a byte to the stream and updates byte counters
*
* @see java.io.FilterOutputStream#write(int)
*/
@Override
public void write(int b) throws IOException {
out.write( b );
bytesWritten += 1;
if (currentEntry != null) {
currentFileSize += 1;
}
}
/**
* Checks if the bytes being written exceed the current entry size.
*
* @see java.io.FilterOutputStream#write(byte[], int, int)
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
if (currentEntry != null && !currentEntry.isDirectory()) {
if (currentEntry.getSize() < currentFileSize + len) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
+ "] being written." );
}
}
out.write( b, off, len );
bytesWritten += len;
if (currentEntry != null) {
currentFileSize += len;
}
}
/**
* Writes the next tar entry header on the stream
*
* @param entry
* @throws IOException
*/
public void putNextEntry(TarEntry entry) throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
entry.writeEntryHeader( header );
write( header );
currentEntry = entry;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
+ currentEntry.getSize() + "] has not been fully written." );
}
currentEntry = null;
currentFileSize = 0;
pad();
}
}
/**
* Pads the last content block
*
* @throws IOException
*/
protected void pad() throws IOException {
if (bytesWritten > 0) {
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
if (extra > 0) {
write( new byte[TarConstants.DATA_BLOCK - extra] );
}
}
}
}
@@ -1,96 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* @author Kamran
*
*/
public class TarUtils {
/**
* Determines the tar file size of the given folder/file path
*
* @param path
* @return
*/
public static long calculateTarSize(File path) {
return tarSize(path) + TarConstants.EOF_BLOCK;
}
private static long tarSize(File dir) {
long size = 0;
if (dir.isFile()) {
return entrySize(dir.length());
} else {
File[] subFiles = dir.listFiles();
if (subFiles != null && subFiles.length > 0) {
for (File file : subFiles) {
if (file.isFile()) {
size += entrySize(file.length());
} else {
size += tarSize(file);
}
}
} else {
// Empty folder header
return TarConstants.HEADER_BLOCK;
}
}
return size;
}
private static long entrySize(long fileSize) {
long size = 0;
size += TarConstants.HEADER_BLOCK; // Header
size += fileSize; // File size
long extra = size % TarConstants.DATA_BLOCK;
if (extra > 0) {
size += (TarConstants.DATA_BLOCK - extra); // pad
}
return size;
}
public static String trim(String s, char c) {
StringBuffer tmp = new StringBuffer(s);
for (int i = 0; i < tmp.length(); i++) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
for (int i = tmp.length() - 1; i >= 0; i--) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
return tmp.toString();
}
}
@@ -1,400 +0,0 @@
package org.kivy.android;
import java.net.Socket;
import java.net.InetSocketAddress;
import android.os.SystemClock;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import android.app.*;
import android.content.*;
import android.view.*;
import android.view.SurfaceView;
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
import android.widget.Toast;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.PowerManager;
import android.graphics.PixelFormat;
import android.view.SurfaceHolder;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.content.pm.PackageManager;
import android.content.pm.ApplicationInfo;
import android.content.Intent;
import android.widget.ImageView;
import java.io.InputStream;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.widget.AbsoluteLayout;
import android.webkit.WebViewClient;
import android.webkit.WebView;
import org.kivy.android.PythonUtil;
import org.renpy.android.ResourceManager;
import org.renpy.android.AssetExtract;
public class PythonActivity extends Activity {
// This activity is modified from a mixture of the SDLActivity and
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
// specifics.
private static final String TAG = "PythonActivity";
public static PythonActivity mActivity = null;
/** If shared libraries (e.g. the native application) could not be loaded. */
public static boolean mBrokenLibraries;
protected static Thread mPythonThread;
private ResourceManager resourceManager = null;
private Bundle mMetaData = null;
private PowerManager.WakeLock mWakeLock = null;
public String getAppRoot() {
String app_root = getFilesDir().getAbsolutePath() + "/app";
return app_root;
}
public static void initialize() {
// The static nature of the singleton and Android quirkiness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
mBrokenLibraries = false;
}
@Override
protected void onCreate(Bundle savedInstanceState) {
Log.v(TAG, "My oncreate running");
resourceManager = new ResourceManager(this);
Log.v(TAG, "Ready to unpack");
File app_root_file = new File(getAppRoot());
unpackData("private", app_root_file);
Log.v(TAG, "About to do super onCreate");
super.onCreate(savedInstanceState);
Log.v(TAG, "Did super onCreate");
this.mActivity = this;
//this.showLoadingScreen();
Log.v("Python", "Device: " + android.os.Build.DEVICE);
Log.v("Python", "Model: " + android.os.Build.MODEL);
//Log.v(TAG, "Ready to unpack");
//new UnpackFilesTask().execute(getAppRoot());
PythonActivity.initialize();
// Load shared libraries
String errorMsgBrokenLib = "";
try {
loadLibraries();
} catch(UnsatisfiedLinkError e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
} catch(Exception e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
}
if (mBrokenLibraries)
{
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
+ System.getProperty("line.separator")
+ System.getProperty("line.separator")
+ "Error: " + errorMsgBrokenLib);
dlgAlert.setTitle("Python Error");
dlgAlert.setPositiveButton("Exit",
new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog,int id) {
// if this button is clicked, close current activity
PythonActivity.mActivity.finish();
}
});
dlgAlert.setCancelable(false);
dlgAlert.create().show();
return;
}
// Set up the Python environment
String app_root_dir = getAppRoot();
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
try {
Log.v(TAG, "Access to our meta-data...");
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
mActivity.mWakeLock.acquire();
}
} catch (PackageManager.NameNotFoundException e) {
}
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
PythonActivity.mPythonThread = pythonThread;
pythonThread.start();
}
@Override
public void onDestroy() {
Log.i("Destroy", "end of app");
super.onDestroy();
// make sure all child threads (python_thread) are stopped
android.os.Process.killProcess(android.os.Process.myPid());
}
public void loadLibraries() {
String app_root = new String(getAppRoot());
File app_root_file = new File(app_root);
PythonUtil.loadLibraries(app_root_file);
}
public void recursiveDelete(File f) {
if (f.isDirectory()) {
for (File r : f.listFiles()) {
recursiveDelete(r);
}
}
f.delete();
}
/**
* Show an error using a toast. (Only makes sense from non-UI
* threads.)
*/
public void toastError(final String msg) {
final Activity thisActivity = this;
runOnUiThread(new Runnable () {
public void run() {
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
}
});
// Wait to show the error.
synchronized (this) {
try {
this.wait(1000);
} catch (InterruptedException e) {
}
}
}
public void unpackData(final String resource, File target) {
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
// The version of data in memory and on disk.
String data_version = resourceManager.getString(resource + "_version");
String disk_version = null;
Log.v(TAG, "Data version is " + data_version);
// If no version, no unpacking is necessary.
if (data_version == null) {
return;
}
// Check the current disk version, if any.
String filesDir = target.getAbsolutePath();
String disk_version_fn = filesDir + "/" + resource + ".version";
try {
byte buf[] = new byte[64];
InputStream is = new FileInputStream(disk_version_fn);
int len = is.read(buf);
disk_version = new String(buf, 0, len);
is.close();
} catch (Exception e) {
disk_version = "";
}
// If the disk data is out of date, extract it and write the
// version file.
// if (! data_version.equals(disk_version)) {
if (! data_version.equals(disk_version)) {
Log.v(TAG, "Extracting " + resource + " assets.");
recursiveDelete(target);
target.mkdirs();
AssetExtract ae = new AssetExtract(this);
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
toastError("Could not extract " + resource + " data.");
}
try {
// Write .nomedia.
new File(target, ".nomedia").createNewFile();
// Write version file.
FileOutputStream os = new FileOutputStream(disk_version_fn);
os.write(data_version.getBytes());
os.close();
} catch (Exception e) {
Log.w("python", e);
}
}
}
long lastBackClick = SystemClock.elapsedRealtime();
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
// If it wasn't the Back key or there's no web page history, bubble up to the default
// system behavior (probably exit the activity)
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
lastBackClick = SystemClock.elapsedRealtime();
Toast.makeText(this, "Click again to close the app",
Toast.LENGTH_LONG).show();
return true;
}
lastBackClick = SystemClock.elapsedRealtime();
return super.onKeyDown(keyCode, event);
}
//----------------------------------------------------------------------------
// Listener interface for onNewIntent
//
public interface NewIntentListener {
void onNewIntent(Intent intent);
}
private List<NewIntentListener> newIntentListeners = null;
public void registerNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
this.newIntentListeners.add(listener);
}
public void unregisterNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
return;
this.newIntentListeners.remove(listener);
}
@Override
protected void onNewIntent(Intent intent) {
if ( this.newIntentListeners == null )
return;
this.onResume();
synchronized ( this.newIntentListeners ) {
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
while ( iterator.hasNext() ) {
(iterator.next()).onNewIntent(intent);
}
}
}
//----------------------------------------------------------------------------
// Listener interface for onActivityResult
//
public interface ActivityResultListener {
void onActivityResult(int requestCode, int resultCode, Intent data);
}
private List<ActivityResultListener> activityResultListeners = null;
public void registerActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
this.activityResultListeners.add(listener);
}
public void unregisterActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
return;
this.activityResultListeners.remove(listener);
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
if ( this.activityResultListeners == null )
return;
this.onResume();
synchronized ( this.activityResultListeners ) {
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
while ( iterator.hasNext() )
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
}
}
public static void start_service(String serviceTitle, String serviceDescription,
String pythonServiceArgument) {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
serviceIntent.putExtra("serviceTitle", serviceTitle);
serviceIntent.putExtra("serviceDescription", serviceDescription);
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
PythonActivity.mActivity.startService(serviceIntent);
}
public static void stop_service() {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
PythonActivity.mActivity.stopService(serviceIntent);
}
public static native void nativeSetEnv(String j_name, String j_value);
public static native int nativeInit(Object arguments);
}
class PythonMain implements Runnable {
@Override
public void run() {
PythonActivity.nativeInit(new String[0]);
}
}
@@ -1,45 +0,0 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonEvent {
private final Lock lock = new ReentrantLock();
private final Condition cond = lock.newCondition();
private boolean flag = false;
public void set() {
lock.lock();
try {
flag = true;
cond.signalAll();
} finally {
lock.unlock();
}
}
public void wait_() throws InterruptedException {
lock.lock();
try {
while (!flag) {
cond.await();
}
} finally {
lock.unlock();
}
}
public void clear() {
lock.lock();
try {
flag = false;
cond.signalAll();
} finally {
lock.unlock();
}
}
}
@@ -1,19 +0,0 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonLock {
private final Lock lock = new ReentrantLock();
public void acquire() {
lock.lock();
}
public void release() {
lock.unlock();
}
}
@@ -1,257 +0,0 @@
package org.renpy.android;
import java.util.List;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.Sensor;
import android.hardware.SensorEvent;
import android.hardware.SensorEventListener;
import android.hardware.SensorManager;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import android.net.wifi.ScanResult;
import android.net.wifi.WifiManager;
import android.os.Vibrator;
import android.view.View;
/**
* Methods that are expected to be called via JNI, to access the device's
* non-screen hardware. (For example, the vibration and accelerometer.)
*/
public class Hardware {
// The context.
static Context context;
static View view;
/**
* Vibrate for s seconds.
*/
public static void vibrate(double s) {
Vibrator v = (Vibrator) context
.getSystemService(Context.VIBRATOR_SERVICE);
if (v != null) {
v.vibrate((int) (1000 * s));
}
}
/**
* Get an Overview of all Hardware Sensors of an Android Device
*/
public static String getHardwareSensors() {
SensorManager sm = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
if (allSensors != null) {
String resultString = "";
for (Sensor s : allSensors) {
resultString += String.format("Name=" + s.getName());
resultString += String.format(",Vendor=" + s.getVendor());
resultString += String.format(",Version=" + s.getVersion());
resultString += String.format(",MaximumRange="
+ s.getMaximumRange());
// XXX MinDelay is not in the 2.2
// resultString += String.format(",MinDelay=" +
// s.getMinDelay());
resultString += String.format(",Power=" + s.getPower());
resultString += String.format(",Type=" + s.getType() + "\n");
}
return resultString;
}
return "";
}
/**
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
* Magnetic Field Sensors
*/
public static class generic3AxisSensor implements SensorEventListener {
private final SensorManager sSensorManager;
private final Sensor sSensor;
private final int sSensorType;
SensorEvent sSensorEvent;
public generic3AxisSensor(int sensorType) {
sSensorType = sensorType;
sSensorManager = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
sSensor = sSensorManager.getDefaultSensor(sSensorType);
}
public void onAccuracyChanged(Sensor sensor, int accuracy) {
}
public void onSensorChanged(SensorEvent event) {
sSensorEvent = event;
}
/**
* Enable or disable the Sensor by registering/unregistering
*/
public void changeStatus(boolean enable) {
if (enable) {
sSensorManager.registerListener(this, sSensor,
SensorManager.SENSOR_DELAY_NORMAL);
} else {
sSensorManager.unregisterListener(this, sSensor);
}
}
/**
* Read the Sensor
*/
public float[] readSensor() {
if (sSensorEvent != null) {
return sSensorEvent.values;
} else {
float rv[] = { 0f, 0f, 0f };
return rv;
}
}
}
public static generic3AxisSensor accelerometerSensor = null;
public static generic3AxisSensor orientationSensor = null;
public static generic3AxisSensor magneticFieldSensor = null;
/**
* functions for backward compatibility reasons
*/
public static void accelerometerEnable(boolean enable) {
if (accelerometerSensor == null)
accelerometerSensor = new generic3AxisSensor(
Sensor.TYPE_ACCELEROMETER);
accelerometerSensor.changeStatus(enable);
}
public static float[] accelerometerReading() {
float rv[] = { 0f, 0f, 0f };
if (accelerometerSensor == null)
return rv;
return (float[]) accelerometerSensor.readSensor();
}
public static void orientationSensorEnable(boolean enable) {
if (orientationSensor == null)
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
orientationSensor.changeStatus(enable);
}
public static float[] orientationSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (orientationSensor == null)
return rv;
return (float[]) orientationSensor.readSensor();
}
public static void magneticFieldSensorEnable(boolean enable) {
if (magneticFieldSensor == null)
magneticFieldSensor = new generic3AxisSensor(
Sensor.TYPE_MAGNETIC_FIELD);
magneticFieldSensor.changeStatus(enable);
}
public static float[] magneticFieldSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (magneticFieldSensor == null)
return rv;
return (float[]) magneticFieldSensor.readSensor();
}
/**
* Scan WiFi networks
*/
static List<ScanResult> latestResult;
public static void enableWifiScanner() {
IntentFilter i = new IntentFilter();
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
// occurs
WifiManager w = (WifiManager) c
.getSystemService(Context.WIFI_SERVICE);
latestResult = w.getScanResults(); // Returns a <list> of
// scanResults
}
}, i);
}
public static String scanWifi() {
// Now you can call this and it should execute the broadcastReceiver's
// onReceive()
WifiManager wm = (WifiManager) context
.getSystemService(Context.WIFI_SERVICE);
boolean a = wm.startScan();
if (latestResult != null) {
String latestResultString = "";
for (ScanResult result : latestResult) {
latestResultString += String.format("%s\t%s\t%d\n",
result.SSID, result.BSSID, result.level);
}
return latestResultString;
}
return "";
}
/**
* network state
*/
public static boolean network_state = false;
/**
* Check network state directly
*
* (only one connection can be active at a given moment, detects all network
* type)
*
*/
public static boolean checkNetwork() {
boolean state = false;
final ConnectivityManager conMgr = (ConnectivityManager) context
.getSystemService(Context.CONNECTIVITY_SERVICE);
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
if (activeNetwork != null && activeNetwork.isConnected()) {
state = true;
} else {
state = false;
}
return state;
}
/**
* To recieve network state changes
*/
public static void registerNetworkCheck() {
IntentFilter i = new IntentFilter();
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
network_state = checkNetwork();
}
}, i);
}
}
Whitespace-only changes.
Whitespace-only changes.
@@ -1,93 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="{{ args.package }}"
android:versionCode="{{ args.numeric_version }}"
android:versionName="{{ args.version }}"
android:installLocation="auto">
<supports-screens
android:smallScreens="true"
android:normalScreens="true"
android:largeScreens="true"
android:anyDensity="true"
{% if args.min_sdk_version >= 9 %}
android:xlargeScreens="true"
{% endif %}
/>
<!-- Android 2.3.3 -->
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
<!-- Set permissions -->
{% for perm in args.permissions %}
{% if '.' in perm %}
<uses-permission android:name="{{ perm }}" />
{% else %}
<uses-permission android:name="android.permission.{{ perm }}" />
{% endif %}
{% endfor %}
{% if args.wakelock %}
<uses-permission android:name="android.permission.WAKE_LOCK" />
{% endif %}
{% if args.billing_pubkey %}
<uses-permission android:name="com.android.vending.BILLING" />
{% endif %}
<!-- Create a Java class extending SDLActivity and place it in a
directory under src matching the package, e.g.
src/com/gamemaker/game/MyGame.java
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
in the XML below.
An example Java class can be found in README-android.txt
-->
<application android:label="@string/app_name"
android:icon="@drawable/icon"
android:allowBackup="true"
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
android:hardwareAccelerated="true" >
{% for m in args.meta_data %}
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
<activity android:name="org.kivy.android.PythonActivity"
android:label="@string/app_name"
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
>
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
{%- if args.intent_filters -%}
{{- args.intent_filters -}}
{%- endif -%}
</activity>
{% if service %}
<service android:name="org.kivy.android.PythonService"
android:process=":pythonservice" />
{% endif %}
{% for name in service_names %}
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
android:process=":service_{{ name }}" />
{% endfor %}
{% if args.billing_pubkey %}
<service android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbilling" />
<receiver android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbillingreceiver">
<intent-filter>
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
</intent-filter>
</receiver>
{% endif %}
</application>
</manifest>
@@ -1,74 +0,0 @@
package {{ args.package }};
import android.os.Binder;
import android.os.IBinder;
import android.content.Intent;
import android.content.Context;
import org.kivy.android.PythonService;
public class Service{{ name|capitalize }} extends PythonService {
/**
* Binder given to clients
*/
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
{% if sticky %}
/**
* {@inheritDoc}
*/
@Override
public int getStartType() {
return START_STICKY;
}
{% endif %}
{% if foreground %}
/**
* {@inheritDoc}
*/
@Override
public boolean getStartForeground() {
return true;
}
{% endif %}
public static void start(Context ctx, String pythonServiceArgument) {
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
intent.putExtra("androidPrivate", argument);
intent.putExtra("androidArgument", argument);
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
intent.putExtra("serviceDescription", "");
intent.putExtra("pythonName", "{{ name }}");
intent.putExtra("pythonHome", argument);
intent.putExtra("androidUnpack", argument);
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
ctx.startService(intent);
}
public static void stop(Context ctx) {
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
ctx.stopService(intent);
}
/**
* {@inheritDoc}
*/
@Override
public IBinder onBind(Intent intent) {
return mBinder;
}
/**
* Class used for the client Binder. Because we know this service always
* runs in the same process as its clients, we don't need to deal with IPC.
*/
public class Service{{ name|capitalize }}Binder extends Binder {
Service{{ name|capitalize }} getService() {
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
return Service{{ name|capitalize }}.this;
}
}
}
@@ -1,5 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="app_name">{{ args.name }}</string>
<string name="private_version">{{ private_version }}</string>
</resources>
-68
View File
@@ -1,68 +0,0 @@
from lbrynet.extras.daemon.Daemon import Daemon
from jnius import PythonJavaClass, autoclass, java_method
JAVA_NAMESPACE = 'org.kivy.android'
JNI_NAMESPACE = 'org/kivy/android'
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
_callbacks = {
'on_new_intent': [],
'on_activity_result': [],
}
class NewIntentListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
__javacontext__ = 'app'
def __init__(self, callback, **kwargs):
super(NewIntentListener, self).__init__(**kwargs)
self.callback = callback
@java_method('(Landroid/content/Intent;)V')
def onNewIntent(self, intent):
self.callback(intent)
class ActivityResultListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
__javacontext__ = 'app'
def __init__(self, callback):
super(ActivityResultListener, self).__init__()
self.callback = callback
@java_method('(IILandroid/content/Intent;)V')
def onActivityResult(self, requestCode, resultCode, intent):
self.callback(requestCode, resultCode, intent)
def bind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
elif event == 'on_new_intent':
listener = NewIntentListener(callback)
_activity.registerNewIntentListener(listener)
_callbacks[event].append(listener)
elif event == 'on_activity_result':
listener = ActivityResultListener(callback)
_activity.registerActivityResultListener(listener)
_callbacks[event].append(listener)
def unbind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
else:
for listener in _callbacks[event][:]:
if listener.callback == callback:
_callbacks[event].remove(listener)
if event == 'on_new_intent':
_activity.unregisterNewIntentListener(listener)
elif event == 'on_activity_result':
_activity.unregisterActivityResultListener(listener)
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
-49
View File
@@ -1,49 +0,0 @@
from os.path import join, dirname, abspath
from pythonforandroid.toolchain import Bootstrap
from setuptools import setup, find_packages
from bootstrap import LBRYServiceBootstrap
Bootstrap.bootstraps = {
'lbry-service': LBRYServiceBootstrap()
}
setup(
name='lbryservice',
version='0.1',
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
description="Android Service for LBRY Network.",
license='MIT',
python_requires='>=3.7',
packages=find_packages(),
package_data={'service': ['*.py']},
options={
'apk': {
'dist_name': 'lbry-service',
'bootstrap': 'lbry-service',
'package': 'io.lbry.service',
'permissions': ['INTERNET'],
'requirements': ','.join([
# needed by aiohttp
'multidict', 'yarl', 'async_timeout', 'chardet',
# minimum needed by torba:
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
abspath(join(dirname(__file__), '..', '..', 'torba')),
# minimum needed by lbrynet
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
abspath(join(dirname(__file__), '..')),
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
]),
'android-api': '26',
'ndk-api': '21',
'ndk-version': 'r17c',
'arch': 'armeabi-v7a',
'sdk-dir': '/home/lex/projects/android',
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
}
}
)
+46 -262
View File
@@ -76,7 +76,7 @@
},
{
"name": "account_decrypt",
"description": "Decrypt an encrypted account, this will remove the wallet password",
"description": "Decrypt an encrypted account, this will remove the wallet password. The account must be unlocked to decrypt it",
"arguments": [
{
"name": "account_id",
@@ -376,31 +376,6 @@
],
"returns": "(bool) true if successful"
},
{
"name": "blob_availability",
"description": "Get blob availability",
"arguments": [
{
"name": "blob_hash",
"type": "str",
"description": "check availability for this blob hash",
"is_required": true
},
{
"name": "search_timeout",
"type": "int",
"description": "how long to search for peers for the blob in the dht",
"is_required": false
},
{
"name": "blob_timeout",
"type": "int",
"description": "how long to try downloading from a peer",
"is_required": false
}
],
"returns": "(dict) {\n \"is_available\": <bool, true if blob is available from a peer from peer list>\n \"reachable_peers\": [\"<ip>:<port>\"],\n \"unreachable_peers\": [\"<ip>:<port>\"]\n }"
},
{
"name": "blob_delete",
"description": "Delete a blob",
@@ -429,18 +404,6 @@
"type": "int",
"description": "timeout in number of seconds",
"is_required": false
},
{
"name": "encoding",
"type": "str",
"description": "by default no attempt at decoding is made, can be set to one of the following decoders: 'json'",
"is_required": false
},
{
"name": "payment_rate_manager",
"type": "str",
"description": "if not given the default payment rate manager will be used. supported alternative rate managers: 'only-free'",
"is_required": false
}
],
"returns": "(str) Success/Fail message or (dict) decoded data"
@@ -820,12 +783,6 @@
],
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
},
{
"name": "commands",
"description": "Return a list of available commands",
"arguments": [],
"returns": "(list) list of available commands"
},
{
"name": "file_delete",
"description": "Delete a LBRY file",
@@ -975,14 +932,32 @@
"description": "get file with matching claim name",
"is_required": false
},
{
"name": "blobs_in_stream<blobs_in_stream>",
"type": "int",
"description": "get file with matching blobs in stream",
"is_required": false
},
{
"name": "blobs_remaining",
"type": "int",
"description": "amount of remaining blobs to download",
"is_required": false
},
{
"name": "sort",
"type": "str",
"description": "sort by any property, like 'file_name' or 'metadata.author'; to specify direction append ',asc' or ',desc'",
"description": "field to sort by (one of the above filter fields)",
"is_required": false
},
{
"name": "comparison",
"type": "str",
"description": "logical comparision, (eq | ne | g | ge | l | le)",
"is_required": false
}
],
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]"
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes_lower_bound': (int) lower bound file size in bytes,\n 'total_bytes': (int) file upper bound size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'blobs_remaining': (int) total blobs remaining to download,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]\n}"
},
{
"name": "file_reflect",
@@ -1083,19 +1058,6 @@
],
"returns": "(dict) Dictionary containing information about the stream\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status,\n 'claim_id': (str) claim id,\n 'outpoint': (str) claim outpoint string,\n 'txid': (str) claim txid,\n 'nout': (int) claim nout,\n 'metadata': (dict) claim metadata,\n 'channel_claim_id': (str) None if claim is not signed\n 'channel_name': (str) None if claim is not signed\n 'claim_name': (str) claim name\n }"
},
{
"name": "help",
"description": "Return a useful message for an API command",
"arguments": [
{
"name": "command",
"type": "str",
"description": "command to retrieve documentation for",
"is_required": false
}
],
"returns": "(str) Help message"
},
{
"name": "peer_list",
"description": "Get peers for blob hash",
@@ -1107,31 +1069,18 @@
"is_required": true
},
{
"name": "timeout",
"name": "search_bottom_out_limit",
"type": "int",
"description": "peer search timeout in seconds",
"description": "the number of search probes in a row that don't find any new peers before giving up and returning",
"is_required": false
}
],
"returns": "(list) List of contact dictionaries {'host': <peer ip>, 'port': <peer port>, 'node_id': <peer node id>}"
"returns": "(list) List of contact dictionaries {'address': <peer ip>, 'udp_port': <dht port>, 'tcp_port': <peer port>,\n 'node_id': <peer node id>}"
},
{
"name": "peer_ping",
"description": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.",
"arguments": [
{
"name": "address",
"type": "str",
"description": "ip address of the peer",
"is_required": false
},
{
"name": "port",
"type": "int",
"description": "udp port of the peer",
"is_required": false
}
],
"arguments": [],
"returns": "(str) pong, or {'error': <error message>} if an error is encountered"
},
{
@@ -1222,12 +1171,6 @@
"description": "whether the content is nsfw",
"is_required": false
},
{
"name": "sources",
"type": "str",
"description": "{'lbry_sd_hash': sd_hash} specifies sd hash of file",
"is_required": false
},
{
"name": "channel_name",
"type": "str",
@@ -1276,53 +1219,22 @@
},
{
"name": "resolve",
"description": "Resolve given LBRY URIs",
"description": "Get the claim that a URL refers to.",
"arguments": [
{
"name": "force",
"type": "bool",
"description": "force refresh and ignore cache",
"is_required": false
},
{
"name": "uri",
"type": "str",
"description": "uri to resolve",
"is_required": true
},
{
"name": "uris",
"type": "list",
"description": "uris to resolve",
"name": "urls",
"type": "str, list",
"description": "one or more urls to resolve",
"is_required": false
}
],
"returns": "Dictionary of results, keyed by uri\n '<uri>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
},
{
"name": "resolve_name",
"description": "Resolve stream info from a LBRY name",
"arguments": [
{
"name": "name",
"type": "str",
"description": "the name to resolve",
"is_required": true
},
{
"name": "force",
"type": "bool",
"description": "force refresh and do not check cache",
"is_required": false
}
],
"returns": "(dict) Metadata dictionary from name claim, None if the name is not\n resolvable"
"returns": "Dictionary of results, keyed by url\n '<url>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the url resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the url resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the url resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
},
{
"name": "routing_table_get",
"description": "Get DHT routing information",
"arguments": [],
"returns": "(dict) dictionary containing routing and contact information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"port\": (int) peer udp port\n \"node_id\": (str) peer node id,\n \"blobs\": (list) blob hashes announced by peer\n }\n ]\n },\n \"contacts\": (list) contact node ids,\n \"blob_hashes\": (list) all of the blob hashes stored by peers in the list of buckets,\n \"node_id\": (str) the local dht node id\n }"
"returns": "(dict) dictionary containing routing and peer information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"udp_port\": (int) peer udp port,\n \"tcp_port\": (int) peer tcp port,\n \"node_id\": (str) peer node id,\n }\n ]\n },\n \"node_id\": (str) the local dht node id\n }"
},
{
"name": "settings_get",
@@ -1333,131 +1245,21 @@
{
"name": "settings_set",
"description": "Set daemon settings",
"arguments": [
{
"name": "download_directory",
"type": "str",
"description": "path of download directory",
"is_required": false
},
{
"name": "data_rate",
"type": "float",
"description": "0.0001",
"is_required": false
},
{
"name": "download_timeout",
"type": "int",
"description": "180",
"is_required": false
},
{
"name": "peer_port",
"type": "int",
"description": "3333",
"is_required": false
},
{
"name": "max_key_fee",
"type": "dict",
"description": "maximum key fee for downloads, in the format: { 'currency': <currency_symbol>, 'amount': <amount> }. In the CLI, it must be an escaped JSON string Supported currency symbols: LBC, USD, BTC",
"is_required": false
},
{
"name": "disable_max_key_fee",
"type": "bool",
"description": "False",
"is_required": false
},
{
"name": "use_upnp",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "run_reflector_server",
"type": "bool",
"description": "False",
"is_required": false
},
{
"name": "cache_time",
"type": "int",
"description": "150",
"is_required": false
},
{
"name": "reflect_uploads",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "share_usage_data",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "peer_search_timeout",
"type": "int",
"description": "3",
"is_required": false
},
{
"name": "sd_download_timeout",
"type": "int",
"description": "3",
"is_required": false
},
{
"name": "auto_renew_claim_height_delta",
"type": "int",
"description": "0 claims set to expire within this many blocks will be automatically renewed after startup (if set to 0, renews will not be made automatically)",
"is_required": false
}
],
"arguments": [],
"returns": "(dict) Updated dictionary of daemon settings"
},
{
"name": "status",
"description": "Get daemon status",
"arguments": [],
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'database': (bool),\n 'dht': (bool),\n 'exchange_rate_manager': (bool),\n 'hash_announcer': (bool),\n 'peer_protocol_server': (bool),\n 'stream_manager': (bool),\n 'upnp': (bool),\n 'wallet': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'stream_manager': {\n 'managed_files': (int) count of files in the stream manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
},
{
"name": "stop",
"description": "Stop lbrynet",
"description": "Stop lbrynet API server.",
"arguments": [],
"returns": "(string) Shutdown message"
},
{
"name": "stream_availability",
"description": "Get stream availability for lbry uri",
"arguments": [
{
"name": "uri",
"type": "str",
"description": "check availability for this uri",
"is_required": true
},
{
"name": "search_timeout",
"type": "int",
"description": "how long to search for peers for the blob in the dht",
"is_required": false
},
{
"name": "blob_timeout",
"type": "int",
"description": "how long to try downloading from a peer",
"is_required": false
}
],
"returns": "(dict) {\n 'is_available': <bool>,\n 'did_decode': <bool>,\n 'did_resolve': <bool>,\n 'is_stream': <bool>,\n 'num_blobs_in_stream': <int>,\n 'sd_hash': <str>,\n 'sd_blob_availability': <dict> see `blob_availability`,\n 'head_blob_hash': <str>,\n 'head_blob_availability': <dict> see `blob_availability`,\n 'use_upnp': <bool>,\n 'upnp_redirect_is_set': <bool>,\n 'error': <None> | <str> error message\n }"
},
{
"name": "stream_cost_estimate",
"description": "Get estimated cost for a lbry stream",
@@ -1467,12 +1269,6 @@
"type": "str",
"description": "uri to use",
"is_required": true
},
{
"name": "size",
"type": "float",
"description": "stream size in bytes. if provided an sd blob won't be downloaded.",
"is_required": false
}
],
"returns": "(float) Estimated cost in lbry credits, returns None if uri is not\n resolvable"
@@ -1541,35 +1337,23 @@
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
},
{
"name": "version",
"description": "Get lbry version information",
"arguments": [],
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
},
{
"name": "wallet_prefill_addresses",
"description": "Create new UTXOs, each containing `amount` credits",
"name": "utxo_release",
"description": "When spending a UTXO it is locally locked to prevent double spends;\noccasionally this can result in a UTXO being locked which ultimately\ndid not get spent (failed to broadcast, spend transaction was not\naccepted by blockchain node, etc). This command releases the lock\non all UTXOs in your account.",
"arguments": [
{
"name": "no_broadcast",
"type": "bool",
"description": "whether to broadcast or not",
"name": "account_id",
"type": "str",
"description": "id of the account to query",
"is_required": false
},
{
"name": "num_addresses",
"type": "int",
"description": "num of addresses to create",
"is_required": true
},
{
"name": "amount",
"type": "decimal",
"description": "initial amount in each address",
"is_required": true
}
],
"returns": "(dict) the resulting transaction"
"returns": "None"
},
{
"name": "version",
"description": "Get lbrynet API server version information",
"arguments": [],
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
},
{
"name": "wallet_send",
+2 -4
View File
@@ -1409,7 +1409,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1954,7 +1954,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1979,7 +1978,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2586,4 +2584,4 @@ Returns:
</body>
</html>
</html>
+2 -4
View File
@@ -1339,7 +1339,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1786,7 +1786,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1811,7 +1810,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2312,4 +2310,4 @@ Returns:
</body>
</html>
</html>
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.31.0rc3"
__version__ = "0.34.0rc2"
version = tuple(__version__.split('.'))
+31 -21
View File
@@ -1,4 +1,5 @@
import os
import re
import asyncio
import binascii
import logging
@@ -17,8 +18,11 @@ from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
_hexmatch = re.compile("^[a-f,0-9]+$")
def is_valid_hashcharacter(char: str) -> bool:
return char in "0123456789abcdef"
return len(char) == 1 and _hexmatch.match(char)
def is_valid_blobhash(blobhash: str) -> bool:
@@ -29,7 +33,7 @@ def is_valid_blobhash(blobhash: str) -> bool:
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
@@ -66,18 +70,22 @@ class BlobFile:
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=loop)
self.blob_write_lock = asyncio.Lock(loop=loop)
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
if self.file_exists:
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
self.length = length
self.verified.set()
self.finished_writing.set()
self.saved_verified_blob = False
self.blob_completed_callback = blob_completed_callback
@property
def file_exists(self):
return os.path.isfile(self.file_path)
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
try:
error = finished.result()
error = finished.exception()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
@@ -86,37 +94,37 @@ class BlobFile:
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer))
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
log.exception("something else")
raise error
return callback
async def save_verified_blob(self, writer):
async def save_verified_blob(self, writer, verified_bytes: bytes):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(writer.verified_bytes):
if self.get_length() == len(verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(writer.verified_bytes)
write_handle.write(verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
if self.verified.is_set():
return
async with self.blob_write_lock:
if self.verified.is_set():
return
await self.loop.run_in_executor(None, _save_verified)
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if os.path.exists(self.file_path):
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
@@ -130,18 +138,20 @@ class BlobFile:
"""
with open(self.file_path, 'rb') as handle:
return await self.loop.sendfile(writer.transport, handle)
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
async def close(self):
def close(self):
while self.writers:
self.writers.pop().finished.cancel()
async def delete(self):
await self.close()
async with self.blob_write_lock:
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
def delete(self):
self.close()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
self.verified.clear()
self.finished_writing.clear()
self.length = None
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
+31 -30
View File
@@ -1,9 +1,9 @@
import os
import typing
import asyncio
import logging
from sqlite3 import IntegrityError
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
@@ -30,10 +30,21 @@ class BlobFileManager:
self.blobs: typing.Dict[str, BlobFile] = {}
async def setup(self) -> bool:
raw_blob_hashes = await self.get_all_verified_blobs()
self.completed_blob_hashes.update(raw_blob_hashes)
def get_files_in_blob_dir() -> typing.Set[str]:
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
return True
def stop(self):
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
self.completed_blob_hashes.clear()
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if length and self.blobs[blob_hash].length is None:
@@ -52,38 +63,28 @@ class BlobFileManager:
raise Exception("Blob has a length of 0")
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
blobs = [self.get_blob(b) for b in blob_hashes]
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
async def set_should_announce(self, blob_hash: str, should_announce: bool):
now = self.loop.time()
return await self.storage.set_should_announce(blob_hash, now, should_announce)
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
async def get_all_verified_blobs(self) -> typing.List[str]:
blob_hashes = await self.storage.get_all_blob_hashes()
return self.check_completed_blobs(blob_hashes)
async def delete_blobs(self, blob_hashes: typing.List[str]):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
if not blob_hash:
continue
try:
blob = self.get_blob(blob_hash)
await blob.delete()
bh_to_delete_from_db.append(blob_hash)
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
if blob_hash not in self.blobs:
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
if blob_hash in self.blobs:
del self.blobs[blob_hash]
try:
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if str(err) != "FOREIGN KEY constraint failed":
raise err
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
for blob_hash in blob_hashes:
self.delete_blob(blob_hash)
if delete_from_db:
await self.storage.delete_blobs_from_db(blob_hashes)
+4 -8
View File
@@ -18,7 +18,6 @@ class HashBlobWriter:
self.finished.add_done_callback(lambda *_: self.close_handle())
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.verified_bytes = b''
def __del__(self):
if self.buffer is not None:
@@ -46,7 +45,7 @@ class HashBlobWriter:
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_result(InvalidDataError(
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
return
@@ -55,15 +54,12 @@ class HashBlobWriter:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_result(InvalidBlobHashError(
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
return
self.buffer.seek(0)
self.verified_bytes = self.buffer.read()
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(self.buffer.getvalue())
self.close_handle()
if self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(None)
def close_handle(self):
if self.buffer is not None:
+116 -80
View File
@@ -1,6 +1,8 @@
import asyncio
import logging
import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
@@ -19,150 +21,184 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.download_running = asyncio.Event(loop=self.loop)
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._request_lock = asyncio.Lock(loop=self.loop)
self.buf = b''
def handle_data_received(self, data: bytes):
if self.transport.is_closing():
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
def data_received(self, data: bytes):
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
response = BlobResponse.deserialize(data)
response = BlobResponse.deserialize(self.buf + data)
if not response.responses and not self._response_fut.done():
self.buf += data
return
else:
self.buf = b''
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
self._blob_bytes_received += len(response.blob_data)
try:
self.writer.write(response.blob_data)
except IOError as err:
log.error("error downloading blob: %s", err)
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def data_received(self, data):
def _write(self, data):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
else:
data = data
self._blob_bytes_received += len(data)
try:
return self.handle_data_received(data)
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
except (asyncio.TimeoutError) as err: # TODO: is this needed?
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[bool, bool]:
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
try:
self.transport.write(request.serialize())
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("blob not in availability response")
return False, True
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request")
return False, False
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected")
return False, False
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from this peer")
return False, True
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch")
return False, False
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length")
return False, False
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.info(msg)
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
await self.blob.finished_writing.wait()
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
log.info(msg)
return True, True
except asyncio.CancelledError:
return False, True
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
return False, False
finally:
await self.close()
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
async def close(self):
def close(self):
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
if self.blob:
await self.blob.close()
self.download_running.clear()
self._response_fut = None
self.writer = None
self.blob = None
if self.transport:
self.transport.close()
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
if blob.get_is_verified():
return False, True
async with self._request_lock:
try:
if self.download_running.is_set():
log.info("wait for download already running")
await self.download_running.wait()
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self.download_running.set()
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError:
log.error("race happened")
# i'm not sure how to fix this race condition - jack
return False, True
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, False
except asyncio.CancelledError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, True
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
return 0, self.transport
try:
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
# i'm not sure how to fix this race condition - jack
log.exception(e)
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
self.close()
return self._blob_bytes_received, None
except asyncio.CancelledError:
self.close()
raise
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
# log.info("connection made to %s: %s", self.peer_address, transport)
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
self.transport = None
self.loop.create_task(self.close())
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
if blob.get_is_verified():
return False, True
if blob.get_is_verified():
log.info("already verified")
return False, True
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.connection_made(connected_transport)
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
try:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or blob.file_exists:
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return False, False
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return 0, None
+123 -111
View File
@@ -1,10 +1,10 @@
import asyncio
import typing
import logging
from lbrynet import conf
from lbrynet.utils import drain_tasks
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
@@ -13,120 +13,132 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
def drain_into(a: list, b: list):
while a:
b.append(a.pop())
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
"""A single blob downloader"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
class BlobDownloader:
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.lock = asyncio.Lock(loop=self.loop)
self.blob: 'BlobFile' = None
self.blob_queue = asyncio.Queue(loop=self.loop)
self.peer_queue = peer_queue
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.time_since_last_blob = loop.time()
self.blob_download_timeout = config.get('blob_download_timeout')
self.peer_connect_timeout = config.get('peer_connect_timeout')
self.max_connections = config.get('max_connections_per_stream')
def should_race_continue(self, blob: 'BlobFile'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
# the safe net side is that any failure will reset the peer score, triggering the race back
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
# for peer, task in self.active_connections.items():
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
# return False
return not (blob.get_is_verified() or blob.file_exists)
async def _request_blob(self, peer: 'KademliaPeer'):
if self.blob.get_is_verified():
log.info("already verified")
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
if blob.get_is_verified():
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self.lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport
)
if bytes_received == blob.get_length():
self.time_since_last_blob = self.loop.time()
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.connections[peer] = transport
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
self.scores[peer] = rough_speed
def _update_requests(self):
self.new_peer_event.clear()
if self.blob.blob_hash not in self.requested_from:
self.requested_from[self.blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.blob.blob_hash][peer] = task
self.running_download_requests.append(task)
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
async def _accumulate_connections(self, node: 'Node'):
async def new_peer_or_finished(self, blob: 'BlobFile'):
async def get_and_re_add_peers():
try:
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
self.peer_queue.put_nowait(new_peers)
except asyncio.TimeoutError:
pass
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
active_tasks = list(self.active_connections.values())
try:
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
return
except asyncio.CancelledError:
pass
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
self.blob = self.blob_manager.get_blob(blob_hash)
if self.blob.get_is_verified():
return self.blob
accumulator = self.loop.create_task(self._accumulate_connections(node))
self.blob_queue.put_nowait(blob_hash)
try:
while not self.blob.get_is_verified():
if len(self.running_download_requests) < self.max_connections:
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
if self.blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
finally:
if accumulator and not accumulator.done():
accumulator.cancel()
drain_tasks(tasks)
def cleanup_active(self):
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
def clearbanned(self):
now = self.loop.time()
if now - self.time_since_last_blob > 60.0:
return
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
self.peer_queue.put_nowait(forgiven)
for banned_peer in forgiven:
self.ignored.pop(banned_peer)
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
try:
while not blob.get_is_verified():
batch: typing.List['KademliaPeer'] = []
while not self.peer_queue.empty():
batch.extend(self.peer_queue.get_nowait())
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in batch:
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
self.active_connections[peer] = t
await self.new_peer_or_finished(blob)
self.cleanup_active()
if batch:
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
else:
self.clearbanned()
blob.close()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
while self.active_connections:
self.active_connections.popitem()[1].cancel()
def close(self):
self.scores.clear()
self.ignored.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
blob_hash: str) -> 'BlobFile':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try:
return await downloader.download_blob(blob_hash)
finally:
if accumulate_task and not accumulate_task.done():
accumulate_task.cancel()
downloader.close()
+12 -18
View File
@@ -146,32 +146,26 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
# <blob bytes>
# <json><blob bytes>
extra_data = b''
response = None
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
break
return None, response_msg
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
if not isinstance(response, dict):
raise ValueError()
for key in response.keys():
if key not in [
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key]:
raise ValueError()
extra_data = response_msg[curr_pos:]
break
except ValueError:
response = None
if response is None:
extra_data = response_msg
return response, extra_data
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
return response, response_msg[curr_pos:]
return None, response_msg
class BlobRequest:
+10 -4
View File
@@ -57,8 +57,13 @@ class BlobServerProtocol(asyncio.Protocol):
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
sent = await blob.sendfile(self)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
if self.transport:
self.transport.close()
return
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
@@ -72,7 +77,7 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf += data
return
try:
request = BlobRequest.deserialize(data)
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
@@ -95,6 +100,7 @@ class BlobServer:
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -102,7 +108,7 @@ class BlobServer:
async def _start_server():
server = await self.loop.create_server(
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
+71 -64
View File
@@ -16,29 +16,11 @@ log = logging.getLogger(__name__)
NOT_SET = type(str('NOT_SET'), (object,), {})
T = typing.TypeVar('T')
KB = 2 ** 10
MB = 2 ** 20
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
API_ADDRESS = 'lbryapi'
APP_NAME = 'LBRY'
BLOBFILES_DIR = 'blobfiles'
CRYPTSD_FILE_EXTENSION = '.cryptsd'
CURRENCIES = {
'BTC': {'type': 'crypto'},
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
LOG_FILE_NAME = 'lbrynet.log'
LOG_POST_URL = 'https://lbry.io/log-upload'
MAX_BLOB_REQUEST_SIZE = 64 * KB
MAX_HANDSHAKE_SIZE = 64 * KB
MAX_REQUEST_SIZE = 64 * KB
MAX_RESPONSE_INFO_SIZE = 64 * KB
MAX_BLOB_INFOS_TO_REQUEST = 20
PROTOCOL_PREFIX = 'lbry'
SLACK_WEBHOOK = (
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
@@ -471,7 +453,7 @@ class CLIConfig(BaseConfig):
class Config(CLIConfig):
# directories
data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path(
"Directory path to place assembled files downloaded from LBRY.",
@@ -482,60 +464,85 @@ class Config(CLIConfig):
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
# network
use_upnp = Toggle(
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
"ports or have firewall rules you likely want to disable this.", True
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# routing table
split_buckets_under_index = Integer(
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
"use.", 1
)
# claims set to expire within this many blocks will be
# automatically renewed after startup (if set to 0, renews
# will not be made automatically)
auto_renew_claim_height_delta = Integer("", 0)
cache_time = Integer("", 150)
data_rate = Float("points/megabyte", .0001)
delete_blobs_on_remove = Toggle("", True)
dht_node_port = Integer("", 4444)
download_timeout = Float("", 30.0)
blob_download_timeout = Float("", 20.0)
peer_connect_timeout = Float("", 3.0)
node_rpc_timeout = Float("", constants.rpc_timeout)
is_generous_host = Toggle("", True)
announce_head_blobs_only = Toggle("", True)
concurrent_announcers = Integer("", 10)
known_dht_nodes = Servers("", [
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
)
concurrent_blob_announcers = Integer(
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 8,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
)
# reflector settings
reflect_streams = Toggle(
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
previous_names=['reflect_uploads']
)
concurrent_reflector_uploads = Integer(
"Maximum number of streams to upload to a reflector server at a time", 10
)
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.io', 4444), # US EAST
('lbrynet2.lbry.io', 4444), # US WEST
('lbrynet3.lbry.io', 4444), # EU
('lbrynet4.lbry.io', 4444) # ASIA
])
max_connections_per_stream = Integer("", 5)
seek_head_blob_first = Toggle("", True)
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
min_info_rate = Float("points/1000 infos", .02)
min_valuable_hash_rate = Float("points/1000 infos", .05)
min_valuable_info_rate = Float("points/1000 infos", .05)
peer_port = Integer("", 3333)
pointtrader_server = String("", 'http://127.0.0.1:2424')
reflector_port = Integer("", 5566)
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
reflect_uploads = Toggle("", True)
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
reflector_servers = Servers("", [
('reflector.lbry.io', 5566)
])
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
sd_download_timeout = Integer("", 3)
peer_search_timeout = Integer("", 60)
use_upnp = Toggle("", True)
use_keyring = Toggle("", False)
blockchain_name = String("", 'lbrycrd_main')
lbryum_servers = Servers("", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
def __init__(self, **kwargs):
super().__init__(**kwargs)
+3 -1
View File
@@ -19,6 +19,8 @@ class BlobAnnouncer:
self.announce_queue: typing.List[str] = []
async def _announce(self, batch_size: typing.Optional[int] = 10):
if not batch_size:
return
if not self.node.joined.is_set():
await self.node.joined.wait()
blob_hashes = await self.storage.get_blobs_to_announce()
@@ -39,7 +41,7 @@ class BlobAnnouncer:
to_await.append(batch.pop())
if to_await:
await asyncio.gather(*tuple(to_await), loop=self.loop)
await self.storage.update_last_announced_blobs(announced, self.loop.time())
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
if self.running:
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
+2
View File
@@ -6,6 +6,7 @@ hash_length = hash_class().digest_size
hash_bits = hash_length * 8
alpha = 5
k = 8
split_buckets_under_index = 1
replacement_cache_size = 8
rpc_timeout = 5.0
rpc_attempts = 5
@@ -15,6 +16,7 @@ refresh_interval = 3600 # 1 hour
replicate_interval = refresh_interval
data_expiration = 86400 # 24 hours
token_secret_refresh_interval = 300 # 5 minutes
maybe_ping_delay = 300 # 5 minutes
check_refresh_interval = refresh_interval / 5
max_datagram_size = 8192 # 8 KB
rpc_id_length = 20
+62 -41
View File
@@ -1,9 +1,9 @@
import logging
import asyncio
import typing
import socket
import binascii
import contextlib
from lbrynet.utils import resolve_host
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
@@ -20,10 +20,12 @@ log = logging.getLogger(__name__)
class Node:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str):
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
split_buckets_under_index)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
@@ -64,7 +66,7 @@ class Node:
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
@@ -79,10 +81,9 @@ class Node:
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.info("Store to %i peers", len(peers))
log.info(peers)
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
@@ -116,39 +117,50 @@ class Node:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
known_node_addresses = known_node_addresses or []
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
if known_node_urls:
for host, port in known_node_urls:
info = await self.loop.getaddrinfo(
host, 'https',
proto=socket.IPPROTO_TCP,
)
if (info[0][4][0], port) not in known_node_addresses:
known_node_addresses.append((info[0][4][0], port))
futs = []
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
futs.append(peer.ping())
if futs:
await asyncio.wait(futs, loop=self.loop)
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
known_node_addresses.append((address, port))
url_to_addr[address] = host
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
if known_node_addresses:
while not self.protocol.routing_table.get_peers():
success = False
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
self.joined.set()
await peer.ping()
success = True
except asyncio.TimeoutError:
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
round(self.protocol.rpc_timeout, 2))
if success:
break
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
# in the routing table with good peers who are near us
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
@@ -178,10 +190,7 @@ class Node:
async def _add_hashes_from_queue():
while True:
try:
blob_hash = await hash_queue.get()
except asyncio.CancelledError:
break
blob_hash = await hash_queue.get()
peer_generator.add_generator(
self.get_iterative_value_finder(
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
@@ -193,10 +202,6 @@ class Node:
async with peer_generator as junction:
yield junction
await peer_generator.finished.wait()
except asyncio.CancelledError:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
raise
finally:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
@@ -214,13 +219,29 @@ class Node:
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20) -> typing.List['KademliaPeer']:
accumulated: typing.List['KademliaPeer'] = []
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
async with self.peer_search_junction(node_id, max_results=max_results,
bottom_out_limit=bottom_out_limit) as junction:
async for peers in junction:
log.info("peer search: %s", peers)
accumulated.extend(peers)
log.info("junction done")
log.info("context done")
distance = Distance(node_id)
accumulated.sort(key=lambda peer: distance(peer.node_id))
return accumulated[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
async for peers in search_junction:
if peers:
result_queue.put_nowait([
peer for peer in peers
if not (
peer.address == self.protocol.external_ip
and peer.tcp_port == self.protocol.peer_port
)
])
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue()
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
+36 -54
View File
@@ -3,6 +3,8 @@ import asyncio
import logging
import ipaddress
from binascii import hexlify
from functools import lru_cache
from lbrynet.dht import constants
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
@@ -30,37 +32,30 @@ class PeerManager:
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
self._lock = asyncio.Lock(loop=loop)
async def report_failure(self, address: str, udp_port: int):
def report_failure(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
async def report_last_sent(self, address: str, udp_port: int):
def report_last_sent(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_sent[(address, udp_port)] = now
self._last_sent[(address, udp_port)] = now
async def report_last_replied(self, address: str, udp_port: int):
def report_last_replied(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_replied[(address, udp_port)] = now
self._last_replied[(address, udp_port)] = now
async def report_last_requested(self, address: str, udp_port: int):
def report_last_requested(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_requested[(address, udp_port)] = now
self._last_requested[(address, udp_port)] = now
async def clear_token(self, node_id: bytes):
async with self._lock:
self._node_tokens.pop(node_id, None)
def clear_token(self, node_id: bytes):
self._node_tokens.pop(node_id, None)
async def update_token(self, node_id: bytes, token: bytes):
def update_token(self, node_id: bytes, token: bytes):
now = self._loop.time()
async with self._lock:
self._node_tokens[node_id] = (now, token)
self._node_tokens[node_id] = (now, token)
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
ts, token = self._node_tokens.get(node_id, (None, None))
@@ -70,50 +65,37 @@ class PeerManager:
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
return self._last_replied.get((address, udp_port))
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
return self._node_id_mapping.get((address, udp_port))
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
return self._node_id_reverse_mapping.get(node_id)
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
async with self._lock:
addr_tuple = self._node_id_reverse_mapping.get(node_id)
if addr_tuple and addr_tuple in self._node_id_mapping:
return addr_tuple
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
"""
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
This is to handle peers changing addresses and ids while assuring that the we only ever have
one node id / address tuple mapped to each other
"""
async with self._lock:
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
@lru_cache(maxsize=400)
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
return KademliaPeer(self._loop, address, node_id, udp_port)
async def prune(self):
def prune(self): # TODO: periodically call this
now = self._loop.time()
async with self._lock:
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
"""
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
pass
finally:
if exc_type:
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
err = traceback.format_exception(exc_type, exc, tb)
log.error(err)
+84 -119
View File
@@ -1,7 +1,9 @@
import asyncio
from binascii import hexlify
from itertools import chain
import typing
import logging
from lbrynet.utils import drain_tasks
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.distance import Distance
@@ -90,8 +92,8 @@ class IterativeFinder:
self.exclude = exclude or []
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
self.active: typing.List['KademliaPeer'] = []
self.contacted: typing.List[typing.Tuple[str, int]] = []
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
@@ -99,14 +101,12 @@ class IterativeFinder:
self.iteration_queue = asyncio.Queue(loop=self.loop)
self.running_probes: typing.List[asyncio.Task] = []
self.lock = asyncio.Lock(loop=self.loop)
self.running_probes: typing.Set[asyncio.Task] = set()
self.iteration_count = 0
self.bottom_out_count = 0
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
self.finished = asyncio.Event(loop=self.loop)
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
"""
@@ -114,9 +114,16 @@ class IterativeFinder:
"""
raise NotImplementedError()
def search_exhausted(self):
"""
This method ends the iterator due no more peers to contact.
Override to provide last time results.
"""
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindResponse):
"""
Called with a lock after adding peers from an rpc result to the shortlist.
Called after adding peers from an rpc result to the shortlist.
This method is responsible for putting a result for the generator into the Queue
"""
raise NotImplementedError()
@@ -129,9 +136,7 @@ class IterativeFinder:
return []
def _is_closer(self, peer: 'KademliaPeer') -> bool:
if not self.closest_peer:
return True
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
def _update_closest(self):
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
@@ -141,35 +146,28 @@ class IterativeFinder:
self.closest_peer = self.shortlist[-1]
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
async with self.lock:
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.append(peer)
for contact_triple in response.get_close_triples():
addr_tuple = (contact_triple[1], contact_triple[2])
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(
contact_triple[0], contact_triple[1], contact_triple[2]
)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.add(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
try:
response = await self.send_probe(peer)
except asyncio.CancelledError:
return
except asyncio.TimeoutError:
if peer in self.active:
self.active.remove(peer)
self.active.discard(peer)
return
except ValueError as err:
log.warning(str(err))
if peer in self.active:
self.active.remove(peer)
self.active.discard(peer)
return
except RemoteException:
return
@@ -181,31 +179,35 @@ class IterativeFinder:
"""
added = 0
async with self.lock:
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.append((peer.address, peer.udp_port))
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.add((peer.address, peer.udp_port))
t = self.loop.create_task(self._send_probe(peer))
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
if t and t in self.running_probes:
self.running_probes.remove(t)
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.append(t)
added += 1
t.add_done_callback(callback)
self.running_probes.add(t)
added += 1
log.debug("running %d probes", len(self.running_probes))
if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8])
self.search_exhausted()
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
try:
@@ -215,70 +217,40 @@ class IterativeFinder:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration):
if self.running:
drain_tasks(self.running_probes)
self.running = False
self.loop.call_soon(self.aclose)
def _search(self):
self.tasks.append(self.loop.create_task(self._search_task()))
def search(self):
def __aiter__(self):
if self.running:
raise Exception("already running")
self.running = True
self._search()
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
peers = self.loop.create_task(self.iteration_queue.get())
finished = self.loop.create_task(self.finished.wait())
err = None
try:
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
if peers.done():
return peers.result()
raise StopAsyncIteration()
except asyncio.CancelledError as error:
err = error
finally:
if not finished.done() and not finished.cancelled():
finished.cancel()
if not peers.done() and not peers.cancelled():
peers.cancel()
if err:
raise err
def __aiter__(self):
self.search()
return self
async def __anext__(self) -> typing.List['KademliaPeer']:
try:
if self.iteration_count == 0:
initial_results = self.get_initial_result()
if initial_results:
self.iteration_queue.put_nowait(initial_results)
result = await self.next_queue_or_finished()
result = self.get_initial_result() or await self.iteration_queue.get()
else:
result = await self.iteration_queue.get()
if not result:
raise StopAsyncIteration
self.iteration_count += 1
return result
except (asyncio.CancelledError, StopAsyncIteration):
await self.aclose()
self.loop.call_soon(self.aclose)
raise
def aclose(self):
self.running = False
async def _aclose():
async with self.lock:
self.running = False
if not self.finished.is_set():
self.finished.set()
drain_tasks(self.tasks)
drain_tasks(self.running_probes)
while self.delayed_calls:
timer = self.delayed_calls.pop()
if timer:
timer.cancel()
return asyncio.ensure_future(_aclose(), loop=self.loop)
self.iteration_queue.put_nowait(None)
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
task.cancel()
self.tasks.clear()
self.running_probes.clear()
self.delayed_calls.clear()
class IterativeNodeFinder(IterativeFinder):
@@ -295,24 +267,26 @@ class IterativeNodeFinder(IterativeFinder):
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response)
def put_result(self, from_list: typing.List['KademliaPeer']):
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
def search_exhausted(self):
self.put_result(self.active, finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
if to_yield:
for peer in to_yield:
self.yielded_peers.add(peer)
self.iteration_queue.put_nowait(to_yield)
if finish:
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindNodeResponse):
found = response.found and self.key != self.protocol.node_id
if found:
log.info("found")
self.put_result(self.shortlist)
if not self.finished.is_set():
self.finished.set()
return
log.debug("found")
return self.put_result(self.shortlist, finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
@@ -323,16 +297,10 @@ class IterativeNodeFinder(IterativeFinder):
self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.info("max results")
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
self.put_result(self.active, True)
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.debug("max results")
self.put_result(self.active, True)
class IterativeValueFinder(IterativeFinder):
@@ -366,14 +334,11 @@ class IterativeValueFinder(IterativeFinder):
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
return
if self.prev_closest_peer and self.closest_peer:
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
if not self.finished.is_set():
self.finished.set()
return
self.iteration_queue.put_nowait(None)
def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key):
+104 -103
View File
@@ -77,24 +77,23 @@ class KademliaRPC:
response = {
b'token': self.make_token(rpc_contact.compact_ip()),
b'contacts': self.find_node(rpc_contact, key)
}
if self.protocol.protocol_version:
response[b'protocolVersion'] = self.protocol.protocol_version
# get peers we have stored for this blob_exchange
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
peers = []
if has_other_peers:
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
peers = [
peer.compact_address_tcp()
for peer in self.protocol.data_store.get_peers_for_blob(key)
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
]
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
if peers:
response[key] = peers
else:
response[b'contacts'] = self.find_node(rpc_contact, key)
return response
def refresh_token(self): # TODO: this needs to be called periodically
@@ -180,7 +179,7 @@ class RemoteKademliaRPC:
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
)
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
return response.response
@@ -188,59 +187,46 @@ class PingQueue:
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
self._loop = loop
self._protocol = protocol
self._enqueued_contacts: typing.List['KademliaPeer'] = []
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
self._process_task: asyncio.Task = None
self._next_task: asyncio.Future = None
self._next_timer: asyncio.TimerHandle = None
self._lock = asyncio.Lock()
self._running = False
self._running_pings: typing.Set[asyncio.Task] = set()
@property
def running(self):
return self._running
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
delay = constants.check_refresh_interval if delay is None else delay
async with self._lock:
for peer in peers:
if delay and peer not in self._enqueued_contacts:
self._pending_contacts[peer] = self._loop.time() + delay
elif peer not in self._enqueued_contacts:
self._enqueued_contacts.append(peer)
if peer in self._pending_contacts:
del self._pending_contacts[peer]
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
now = self._loop.time()
for peer in peers:
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
self._pending_contacts[peer] = delay + now
async def _process(self):
async def _ping(p: 'KademliaPeer'):
def maybe_ping(self, peer: 'KademliaPeer'):
async def ping_task():
try:
if self._protocol.peer_manager.peer_is_good(p):
await self._protocol.add_peer(p)
if self._protocol.peer_manager.peer_is_good(peer):
if peer not in self._protocol.routing_table.get_peers():
await self._protocol.add_peer(peer)
return
await self._protocol.get_rpc_peer(p).ping()
except TimeoutError:
await self._protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
pass
task = self._loop.create_task(ping_task())
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
self._running_pings.add(task)
async def _process(self): # send up to 1 ping per second
while True:
tasks = []
async with self._lock:
if self._enqueued_contacts or self._pending_contacts:
now = self._loop.time()
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
for k in scheduled:
del self._pending_contacts[k]
if k not in self._enqueued_contacts:
self._enqueued_contacts.append(k)
while self._enqueued_contacts:
peer = self._enqueued_contacts.pop()
tasks.append(self._loop.create_task(_ping(peer)))
if tasks:
await asyncio.wait(tasks, loop=self._loop)
f = self._loop.create_future()
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
await f
enqueued = list(self._pending_contacts.keys())
now = self._loop.time()
for peer in enqueued:
if self._pending_contacts[peer] <= now:
del self._pending_contacts[peer]
self.maybe_ping(peer)
break
await asyncio.sleep(1, loop=self._loop)
def start(self):
assert not self._running
@@ -254,17 +240,14 @@ class PingQueue:
if self._process_task:
self._process_task.cancel()
self._process_task = None
if self._next_task:
self._next_task.cancel()
self._next_task = None
if self._next_timer:
self._next_timer.cancel()
self._next_timer = None
while self._running_pings:
self._running_pings.pop().cancel()
class KademliaProtocol(DatagramProtocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.peer_manager = peer_manager
self.loop = loop
self.node_id = node_id
@@ -279,11 +262,10 @@ class KademliaProtocol(DatagramProtocol):
self.transport: DatagramTransport = None
self.old_token_secret = constants.generate_id()
self.token_secret = constants.generate_id()
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
self.data_store = DictDataStore(self.loop, self.peer_manager)
self.ping_queue = PingQueue(self.loop, self)
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
self.lock = asyncio.Lock(loop=self.loop)
self.rpc_timeout = rpc_timeout
self._split_lock = asyncio.Lock(loop=self.loop)
@@ -317,9 +299,14 @@ class KademliaProtocol(DatagramProtocol):
return args, {}
async def _add_peer(self, peer: 'KademliaPeer'):
for p in self.routing_table.get_peers():
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
self.routing_table.remove_peer(p)
self.routing_table.join_buckets()
bucket_index = self.routing_table.kbucket_index(peer.node_id)
if self.routing_table.buckets[bucket_index].add_peer(peer):
return True
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
if self.routing_table.should_split(bucket_index, peer.node_id):
self.routing_table.split_bucket(bucket_index)
@@ -413,22 +400,32 @@ class KademliaProtocol(DatagramProtocol):
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
)
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
# This is an RPC method request
await self.peer_manager.report_last_requested(address[0], address[1])
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
# only add a requesting contact to the routing table if it has replied to one of our requests
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
self.peer_manager.report_last_requested(address[0], address[1])
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
await self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
# will be added to our routing table if successful
is_good = self.peer_manager.peer_is_good(peer)
if is_good is None:
await self.ping_queue.enqueue_maybe_ping(peer)
self.ping_queue.enqueue_maybe_ping(peer)
# only add a requesting contact to the routing table if it has replied to one of our requests
elif is_good is True:
await self.add_peer(peer)
except ValueError as err:
log.debug("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
await self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
)
except Exception as err:
log.warning("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
@@ -445,11 +442,10 @@ class KademliaProtocol(DatagramProtocol):
peer, df, request = self.sent_messages[response_datagram.rpc_id]
if peer.address != address[0]:
df.set_exception(RemoteException(
f"response from {address[0]}:{address[1]}, "
f"expected {peer.address}:{peer.udp_port}")
f"response from {address[0]}, expected {peer.address}")
)
return
peer.set_id(response_datagram.node_id)
# We got a result from the RPC
if peer.node_id == self.node_id:
df.set_exception(RemoteException("node has our node id"))
@@ -457,8 +453,10 @@ class KademliaProtocol(DatagramProtocol):
elif response_datagram.node_id == self.node_id:
df.set_exception(RemoteException("incoming message is from our node id"))
return
await self.peer_manager.report_last_replied(address[0], address[1])
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
peer.set_id(response_datagram.node_id)
peer.update_udp_port(address[1])
self.peer_manager.report_last_replied(address[0], address[1])
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
df.set_result(response_datagram)
await self.add_peer(peer)
@@ -475,21 +473,23 @@ class KademliaProtocol(DatagramProtocol):
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
if error_datagram.rpc_id in self.sent_messages:
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
if (peer.address, peer.udp_port) != address:
df.set_exception(RemoteException(
f"response from {address[0]}:{address[1]}, "
f"expected {peer.address}:{peer.udp_port}")
)
return
error_msg = f"" \
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
f"Args: {request.args}\n" \
f"Raised: {str(remote_exception)}"
if error_datagram.response not in old_protocol_errors:
if 'Invalid token' in error_msg:
log.debug(error_msg)
elif error_datagram.response not in old_protocol_errors:
log.warning(error_msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
# reject replies coming from a different address than what we sent our request to
if (peer.address, peer.udp_port) != address:
log.error("node id mismatch in reply")
remote_exception = TimeoutError(peer.node_id)
df.set_exception(remote_exception)
return
else:
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
try:
message = decode_datagram(datagram)
except (ValueError, TypeError):
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
@@ -522,12 +522,14 @@ class KademliaProtocol(DatagramProtocol):
response_fut = self.sent_messages[request.rpc_id][1]
try:
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except (asyncio.TimeoutError, RemoteException):
await self.peer_manager.report_failure(peer.address, peer.udp_port)
self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
self.routing_table.remove_peer(peer)
async with self._split_lock:
self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
raise
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
@@ -554,30 +556,29 @@ class KademliaProtocol(DatagramProtocol):
if message.rpc_id in self.sent_messages:
self.sent_messages.pop(message.rpc_id)
async with self.lock:
if isinstance(message, RequestDatagram):
response_fut = self.loop.create_future()
response_fut.add_done_callback(pop_from_sent_messages)
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
try:
self.transport.sendto(data, (peer.address, peer.udp_port))
except OSError as err:
# TODO: handle ENETUNREACH
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), peer.address, peer.udp_port, str(err), err.errno)
if isinstance(message, RequestDatagram):
self.sent_messages[message.rpc_id][1].set_exception(err)
else:
raise err
if isinstance(message, RequestDatagram):
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
response_fut = self.loop.create_future()
response_fut.add_done_callback(pop_from_sent_messages)
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
try:
self.transport.sendto(data, (peer.address, peer.udp_port))
except OSError as err:
# TODO: handle ENETUNREACH
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), peer.address, peer.udp_port, str(err), err.errno)
if isinstance(message, RequestDatagram):
self.sent_messages[message.rpc_id][1].set_exception(err)
else:
raise err
if isinstance(message, RequestDatagram):
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
elif isinstance(message, ErrorDatagram):
await self.peer_manager.report_failure(peer.address, peer.udp_port)
self.peer_manager.report_failure(peer.address, peer.udp_port)
def change_token(self):
self.old_token_secret = self.token_secret
@@ -601,7 +602,7 @@ class KademliaProtocol(DatagramProtocol):
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
raise ValueError(res)
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, True
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
@@ -609,7 +610,7 @@ class KademliaProtocol(DatagramProtocol):
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
await self.peer_manager.clear_token(peer.node_id)
self.peer_manager.clear_token(peer.node_id)
else:
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
+22 -8
View File
@@ -29,6 +29,7 @@ class KBucket:
self.range_max = range_max
self.peers: typing.List['KademliaPeer'] = []
self._node_id = node_id
self._distance_to_self = Distance(node_id)
def add_peer(self, peer: 'KademliaPeer') -> bool:
""" Add contact to _contact list in the right order. This will move the
@@ -48,7 +49,14 @@ class KBucket:
self.peers.remove(peer)
self.peers.append(peer)
return True
elif len(self.peers) < constants.k:
else:
for i in range(len(self.peers)):
p = self.peers[i]
if p.node_id == peer.node_id:
self.peers.remove(p)
self.peers.append(peer)
return True
if len(self.peers) < constants.k:
self.peers.append(peer)
return True
else:
@@ -130,7 +138,7 @@ class KBucket:
if not.
@rtype: bool
"""
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
return self.range_min <= self._distance_to_self(key) < self.range_max
def __len__(self) -> int:
return len(self.peers)
@@ -155,10 +163,12 @@ class TreeRoutingTable:
that paper.
"""
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
split_buckets_under_index: int = constants.split_buckets_under_index):
self._loop = loop
self._peer_manager = peer_manager
self._parent_node_id = parent_node_id
self._split_buckets_under_index = split_buckets_under_index
self.buckets: typing.List[KBucket] = [
KBucket(
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
@@ -170,7 +180,7 @@ class TreeRoutingTable:
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
if bucket_index < self._split_buckets_under_index:
return True
contacts = self.get_peers()
distance = Distance(self._parent_node_id)
@@ -183,8 +193,6 @@ class TreeRoutingTable:
exclude = [self._parent_node_id]
if sender_node_id:
exclude.append(sender_node_id)
if key in exclude:
exclude.remove(key)
count = count or constants.k
distance = Distance(key)
contacts = self.get_peers()
@@ -238,11 +246,15 @@ class TreeRoutingTable:
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
return random_id.to_bytes(constants.hash_length, 'big')
return Distance(
self._parent_node_id
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
return Distance(self._parent_node_id)(
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
).to_bytes(constants.hash_length, 'big')
def split_bucket(self, old_bucket_index: int) -> None:
""" Splits the specified k-bucket into two new buckets which together
@@ -269,6 +281,8 @@ class TreeRoutingTable:
old_bucket.remove_peer(contact)
def join_buckets(self):
if len(self.buckets) == 1:
return
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
if not to_pop:
return
+1 -2
View File
@@ -67,8 +67,7 @@ class RequestDatagram(KademliaDatagramBase):
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
if rpc_id and len(rpc_id) != constants.rpc_id_length:
raise ValueError("invalid rpc id length")
elif not rpc_id:
rpc_id = constants.generate_id()[:constants.rpc_id_length]
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(from_node_id) != constants.hash_bits // 8:
raise ValueError("invalid node id")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
+1 -2
View File
@@ -28,8 +28,7 @@ class DownloadTimeoutError(Exception):
class DownloadDataTimeout(Exception):
def __init__(self, download):
super().__init__('Failed to download data blobs for sd hash '
'{} within timeout'.format(download))
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
self.download = download
+29 -2
View File
@@ -1,4 +1,7 @@
import os
import sys
import signal
import pathlib
import json
import asyncio
import argparse
@@ -216,12 +219,19 @@ def get_argument_parser():
return main
def ensure_directory_exists(path: str):
if not os.path.isdir(path):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
def main(argv=None):
argv = argv or sys.argv[1:]
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
ensure_directory_exists(directory)
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
@@ -238,17 +248,23 @@ def main(argv=None):
)
file_handler.setFormatter(default_formatter)
log.addHandler(file_handler)
logging.getLogger('torba').addHandler(file_handler)
if not args.quiet:
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
log.addHandler(handler)
logging.getLogger('torba').addHandler(handler)
logging.getLogger('torba').setLevel(logging.INFO)
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
loop = asyncio.get_event_loop()
if args.verbose:
log.setLevel(logging.DEBUG)
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
@@ -257,10 +273,21 @@ def main(argv=None):
log.addHandler(loggly_handler)
daemon = Daemon(conf)
loop = asyncio.get_event_loop()
started = False
def __exit():
if started:
daemon.stop_event.set()
else:
raise GracefulExit()
try:
loop.add_signal_handler(signal.SIGINT, __exit)
loop.add_signal_handler(signal.SIGTERM, __exit)
except NotImplementedError:
pass # Not implemented on Windows
try:
loop.run_until_complete(daemon.start())
loop.run_forever()
started = True
loop.run_until_complete(daemon.stop_event.wait())
except (GracefulExit, KeyboardInterrupt):
pass
finally:
+46 -79
View File
@@ -1,11 +1,9 @@
import os
import asyncio
import aiohttp
import logging
import math
import binascii
import typing
import socket
from hashlib import sha256
from types import SimpleNamespace
import base58
@@ -18,7 +16,6 @@ import lbrynet.schema
from lbrynet import utils
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.server import BlobServer
@@ -54,25 +51,6 @@ async def gather_dict(tasks: dict):
)))
async def get_external_ip(): # used if upnp is disabled or non-functioning
try:
async with aiohttp.ClientSession() as session:
async with session.get("https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
pass
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
info = await loop.getaddrinfo(
url, 'https',
proto=socket.IPPROTO_TCP,
)
return info[0][4][0]
class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT
@@ -116,7 +94,7 @@ class DatabaseComponent(Component):
from lbrynet.extras.daemon.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
await asyncio.get_event_loop().run_in_executor(
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
@@ -157,40 +135,31 @@ class HeadersComponent(Component):
}
async def fetch_headers_from_s3(self):
def collector(d, h_file):
h_file.write(d)
local_size = float(h_file.tell())
final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
got_406 = response.status == 406 # our file is bigger
final_size_after_download = response.content_length + local_header_size
if got_406:
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
data = await response.read()
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.content_length)
with open(self.headers_file, "a+b") as headers_file:
collector(data, headers_file)
else:
with open(self.headers_file, "wb") as headers_file:
collector(data, headers_file)
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
return
if response.content_length % HEADER_SIZE != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while True:
chunk = await response.content.read(512)
if not chunk:
break
fd.write(chunk)
self._headers_progress_percent = math.ceil(
float(fd.tell()) / float(final_size_after_download) * 100
)
log.info("fetched headers from s3, now verifying integrity after download.")
self._check_header_file_integrity()
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
@@ -331,9 +300,7 @@ class BlobComponent(Component):
return await self.blob_manager.setup()
async def stop(self):
while self.blob_manager and self.blob_manager.blobs:
_, blob = self.blob_manager.blobs.popitem()
await blob.close()
self.blob_manager.stop()
async def get_status(self):
count = 0
@@ -359,7 +326,7 @@ class DHTComponent(Component):
async def get_status(self):
return {
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
}
@@ -376,12 +343,12 @@ class DHTComponent(Component):
async def start(self):
log.info("start the dht")
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await get_external_ip()
external_ip = await utils.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
@@ -389,13 +356,15 @@ class DHTComponent(Component):
asyncio.get_event_loop(),
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.dht_node_port,
internal_udp_port=self.conf.udp_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index
)
self.dht_node.start(
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
)
log.info("Started the dht")
@@ -419,7 +388,7 @@ class HashAnnouncerComponent(Component):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
self.hash_announcer.start(self.conf.concurrent_announcers)
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
log.info("Started blob announcer")
async def stop(self):
@@ -462,17 +431,13 @@ class StreamManagerComponent(Component):
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
self.conf.peer_connect_timeout, [
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
for url, port in self.conf.reflector_servers
], self.conf.reflector_servers
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
)
await self.stream_manager.start()
log.info('Done setting up file manager')
async def stop(self):
await self.stream_manager.stop()
self.stream_manager.stop()
class PeerProtocolServerComponent(Component):
@@ -492,10 +457,10 @@ class PeerProtocolServerComponent(Component):
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
peer_port = self.conf.tcp_port
address = await wallet.get_unused_address()
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
self.blob_server.start_server(peer_port, interface='0.0.0.0')
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
await self.blob_server.started_listening.wait()
async def stop(self):
@@ -508,8 +473,8 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self._int_peer_port = self.conf.tcp_port
self._int_dht_node_port = self.conf.udp_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
@@ -548,7 +513,7 @@ class UPnPComponent(Component):
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
external_ip = await get_external_ip()
external_ip = await utils.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
self.external_ip = external_ip
@@ -599,7 +564,7 @@ class UPnPComponent(Component):
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await get_external_ip()
self.external_ip = await utils.get_external_ip()
return
success = False
await self._maintain_redirects()
@@ -614,7 +579,9 @@ class UPnPComponent(Component):
else:
log.error("failed to setup upnp")
if self.component_manager.analytics_manager:
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
success, await self.get_status()
)
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self):
@@ -640,7 +607,7 @@ class ExchangeRateManagerComponent(Component):
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
def __init__(self, component_manager):
Component.__init__(self, component_manager)
super().__init__(component_manager)
self.exchange_rate_manager = ExchangeRateManager()
@property
+228 -151
View File
@@ -2,25 +2,27 @@ import os
import asyncio
import logging
import json
import time
import inspect
import typing
import aiohttp
import base58
import random
from urllib.parse import urlencode, quote
from typing import Callable, Optional, List
from binascii import hexlify, unhexlify
from copy import deepcopy
from traceback import format_exc
from aiohttp import web
from functools import wraps
from torba.client.wallet import Wallet
from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
from lbrynet import __version__, utils
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.blob_exchange.downloader import download_blob
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
from lbrynet.extras import system_info
from lbrynet.extras.daemon import analytics
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
@@ -29,6 +31,7 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
from lbrynet.extras.daemon.mime_types import guess_media_type
from lbrynet.extras.daemon.undecorated import undecorated
from lbrynet.extras.wallet.account import Account as LBCAccount
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
@@ -249,12 +252,14 @@ class Daemon(metaclass=JSONRPCServerType):
self._node_id = None
self._installation_id = None
self.session_id = base58.b58encode(utils.generate_id()).decode()
self.analytics_manager = analytics.Manager(conf, self.installation_id, self.session_id)
self.analytics_manager = analytics.AnalyticsManager(conf, self.installation_id, self.session_id)
self.component_manager = component_manager or ComponentManager(
conf, analytics_manager=self.analytics_manager,
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
self.stop_event = asyncio.Event()
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
app = web.Application()
@@ -367,6 +372,18 @@ class Daemon(metaclass=JSONRPCServerType):
if not os.path.isdir(self.conf.download_dir):
os.makedirs(self.conf.download_dir)
async def update_connection_status(self):
connected = await utils.async_check_connection()
self._connection_status = (self.component_manager.loop.time(), connected)
async def get_connection_status(self) -> str:
if self._connection_status[0] + 300 > self.component_manager.loop.time():
if not self._connection_status[1]:
await self.update_connection_status()
else:
await self.update_connection_status()
return CONNECTION_STATUS_CONNECTED if self._connection_status[1] else CONNECTION_STATUS_NETWORK
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
@@ -401,7 +418,7 @@ class Daemon(metaclass=JSONRPCServerType):
self.ensure_wallet_dir()
self.ensure_download_dir()
if not self.analytics_manager.is_started:
self.analytics_manager.start()
await self.analytics_manager.start()
self.component_startup_task = asyncio.create_task(self.component_manager.start())
await self.component_startup_task
@@ -474,6 +491,9 @@ class Daemon(metaclass=JSONRPCServerType):
if asyncio.iscoroutine(result):
result = await result
return result
except asyncio.CancelledError:
log.info("cancelled API call for: %s", function_name)
raise
except Exception as e: # pylint: disable=broad-except
log.exception("error handling api request")
return JSONRPCError(
@@ -590,6 +610,7 @@ class Daemon(metaclass=JSONRPCServerType):
(string) Shutdown message
"""
log.info("Shutting down lbrynet daemon")
self.stop_event.set()
return "Shutting down"
async def jsonrpc_status(self):
@@ -609,19 +630,16 @@ class Daemon(metaclass=JSONRPCServerType):
'is_running': (bool),
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
'wallet': (bool),
'session': (bool),
'dht': (bool),
'hash_announcer': (bool),
'stream_identifier': (bool),
'file_manager': (bool),
'blob_manager': (bool),
'blockchain_headers': (bool),
'peer_protocol_server': (bool),
'reflector': (bool),
'upnp': (bool),
'database': (bool),
'dht': (bool),
'exchange_rate_manager': (bool),
'hash_announcer': (bool),
'peer_protocol_server': (bool),
'stream_manager': (bool),
'upnp': (bool),
'wallet': (bool),
},
'connection_status': {
'code': (str) connection status code,
@@ -648,8 +666,8 @@ class Daemon(metaclass=JSONRPCServerType):
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
},
'file_manager': {
'managed_files': (int) count of files in the file manager,
'stream_manager': {
'managed_files': (int) count of files in the stream manager,
},
'upnp': {
'aioupnp_version': (str),
@@ -664,7 +682,8 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
connection_code = CONNECTION_STATUS_NETWORK
connection_code = await self.get_connection_status()
response = {
'installation_id': self.installation_id,
'is_running': all(self.component_manager.get_components_status().values()),
@@ -761,7 +780,10 @@ class Daemon(metaclass=JSONRPCServerType):
Set daemon settings
Usage:
settings_set <key> <value>
settings_set (<key>) (<value>)
Options:
None
Returns:
(dict) Updated dictionary of daemon settings
@@ -823,12 +845,12 @@ class Daemon(metaclass=JSONRPCServerType):
amount = self.get_dewies_or_error("amount", amount)
if not amount:
raise NullFundsError
elif amount < 0:
if amount < 0:
raise NegativeFundsError()
if address and claim_id:
raise Exception("Given both an address and a claim id")
elif not address and not claim_id:
if not address and not claim_id:
raise Exception("Not given an address or a claim id")
if address:
@@ -1060,6 +1082,7 @@ class Daemon(metaclass=JSONRPCServerType):
change_made = True
if change_made:
account.modified_on = time.time()
self.default_wallet.save()
result = account.to_dict()
@@ -1218,7 +1241,7 @@ class Daemon(metaclass=JSONRPCServerType):
amount = self.get_dewies_or_error("amount", amount)
if not amount:
raise NullFundsError
elif amount < 0:
if amount < 0:
raise NegativeFundsError()
for address in addresses:
@@ -1229,6 +1252,68 @@ class Daemon(metaclass=JSONRPCServerType):
await self.analytics_manager.send_credits_sent()
return result
SYNC_DOC = """
Wallet synchronization.
"""
@requires("wallet")
def jsonrpc_sync_hash(self):
"""
Deterministic hash of the wallet.
Usage:
sync hash
Options:
Returns:
(str) sha256 hash of wallet
"""
return hexlify(self.default_wallet.hash).decode()
@requires("wallet")
def jsonrpc_sync_apply(self, password, data=None, encrypt_password=None):
"""
Apply incoming synchronization data, if provided, and then produce a sync hash and
an encrypted wallet.
Usage:
sync apply <password> [--data=<data>] [--encrypt-password=<encrypt_password>]
Options:
--password=<password> : (str) password to decrypt incoming and encrypt outgoing data
--data=<data> : (str) incoming sync data, if any
--encrypt-password=<encrypt_password> : (str) password to encrypt outgoing data if different
from the decrypt password, used during password changes
Returns:
(map) sync hash and data
"""
if data is not None:
decrypted_data = Wallet.unpack(password, data)
for account_data in decrypted_data['accounts']:
_, _, pubkey = LBCAccount.keys_from_dict(self.ledger, account_data)
account_id = pubkey.address
local_match = None
for local_account in self.default_wallet.accounts:
if account_id == local_account.id:
local_match = local_account
break
if local_match is not None:
local_match.apply(account_data)
else:
new_account = LBCAccount.from_dict(self.ledger, self.default_wallet, account_data)
if self.ledger.network.is_connected:
asyncio.create_task(self.ledger.subscribe_account(new_account))
self.default_wallet.save()
encrypted = self.default_wallet.pack(encrypt_password or password)
return {
'hash': self.jsonrpc_sync_hash(),
'data': encrypted.decode()
}
ADDRESS_DOC = """
Address management.
"""
@@ -1307,8 +1392,9 @@ class Daemon(metaclass=JSONRPCServerType):
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
[--full_status=<full_status>]
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
Options:
--sd_hash=<sd_hash> : (str) get file with matching sd hash
@@ -1321,11 +1407,12 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str) get file with matching claim txid
--nout=<nout> : (int) get file with matching claim nout
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
--channel_name=<channel_name> : (str) get file with matching channel name
--channel_name=<channel_name> : (str) get file with matching channel name
--claim_name=<claim_name> : (str) get file with matching claim name
--sort=<sort_method> : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
Returns:
(list) List of files
@@ -1344,23 +1431,26 @@ class Daemon(metaclass=JSONRPCServerType):
'download_path': (str) download path of file,
'mime_type': (str) mime type of file,
'key': (str) key attached to file,
'total_bytes': (int) file size in bytes,
'total_bytes_lower_bound': (int) lower bound file size in bytes,
'total_bytes': (int) file upper bound size in bytes,
'written_bytes': (int) written size in bytes,
'blobs_completed': (int) number of fully downloaded blobs,
'blobs_in_stream': (int) total blobs on stream,
'blobs_remaining': (int) total blobs remaining to download,
'status': (str) downloader status
'claim_id': (str) None if claim is not found else the claim id,
'outpoint': (str) None if claim is not found else the tx and output,
'txid': (str) None if claim is not found else the transaction id,
'nout': (int) None if claim is not found else the transaction output index,
'outpoint': (str) None if claim is not found else the tx and output,
'metadata': (dict) None if claim is not found else the claim metadata,
'channel_claim_id': (str) None if claim is not found or not signed,
'channel_name': (str) None if claim is not found or not signed,
'claim_name': (str) None if claim is not found else the claim name
},
]
}
"""
sort = sort or 'status'
sort = sort or 'rowid'
comparison = comparison or 'eq'
return [
stream.as_dict() for stream in self.stream_manager.get_filtered_streams(
@@ -1417,31 +1507,29 @@ class Daemon(metaclass=JSONRPCServerType):
return claim_results
@requires(WALLET_COMPONENT)
async def jsonrpc_resolve(self, force=False, uri=None, uris=None):
async def jsonrpc_resolve(self, urls: typing.Union[str, list]):
"""
Resolve given LBRY URIs
Get the claim that a URL refers to.
Usage:
resolve [--force] (<uri> | --uri=<uri>) [<uris>...]
resolve <urls>...
Options:
--force : (bool) force refresh and ignore cache
--uri=<uri> : (str) uri to resolve
--uris=<uris> : (list) uris to resolve
--urls=<urls> : (str, list) one or more urls to resolve
Returns:
Dictionary of results, keyed by uri
'<uri>': {
Dictionary of results, keyed by url
'<url>': {
If a resolution error occurs:
'error': Error message
If the uri resolves to a channel or a claim in a channel:
If the url resolves to a channel or a claim in a channel:
'certificate': {
'address': (str) claim address,
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1457,16 +1545,16 @@ class Daemon(metaclass=JSONRPCServerType):
'value': ClaimDict if decoded, otherwise hex string
}
If the uri resolves to a channel:
If the url resolves to a channel:
'claims_in_channel': (int) number of claims in the channel,
If the uri resolves to a claim:
If the url resolves to a claim:
'claim': {
'address': (str) claim address,
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1485,23 +1573,24 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
uris = tuple(uris or [])
if uri is not None:
uris += (uri,)
if isinstance(urls, str):
urls = [urls]
results = {}
valid_uris = tuple()
for u in uris:
valid_urls = set()
for u in urls:
try:
parse_lbry_uri(u)
valid_uris += (u,)
valid_urls.add(u)
except URIParseError:
results[u] = {"error": "%s is not a valid uri" % u}
results[u] = {"error": "%s is not a valid url" % u}
resolved = await self.wallet_manager.resolve(*tuple(valid_urls))
resolved = await self.wallet_manager.resolve(*valid_uris, check_cache=not force)
for resolved_uri in resolved:
results[resolved_uri] = resolved[resolved_uri]
return results
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT,
@@ -1551,44 +1640,17 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
timeout = timeout if timeout is not None else self.conf.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise Exception("cannot download a channel claim, specify a /path")
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
try:
stream = await self.stream_manager.download_stream_from_uri(
uri, self.exchange_rate_manager, file_name, timeout
)
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(self.exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
fee_address = claim.source_fee.address
outpoint = f"{resolved['txid']}:{resolved['nout']}"
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
if not existing:
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
sd_hash=claim.source_hash))
if existing:
log.info("already have matching stream for %s", uri)
stream = existing[0]
if not stream:
raise DownloadSDTimeout(uri)
except Exception as e:
log.warning("Error downloading %s: %s", uri, str(e))
return {"error": str(e)}
else:
stream = await self.stream_manager.download_stream_from_claim(
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
)
if stream:
return stream.as_dict()
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
@requires(STREAM_MANAGER_COMPONENT)
async def jsonrpc_file_set_status(self, status, **kwargs):
@@ -1618,11 +1680,11 @@ class Daemon(metaclass=JSONRPCServerType):
if not streams:
raise Exception(f'Unable to find a file for {kwargs}')
stream = streams[0]
if status == 'start' and not stream.running and not stream.finished:
stream.downloader.download(self.dht_node)
if status == 'start' and not stream.running:
await self.stream_manager.start_stream(stream)
msg = "Resumed download"
elif status == 'stop' and stream.running:
await stream.stop_download()
await self.stream_manager.stop_stream(stream)
msg = "Stopped download"
else:
msg = (
@@ -1679,8 +1741,9 @@ class Daemon(metaclass=JSONRPCServerType):
return False
else:
for stream in streams:
message = f"Deleted file {stream.file_name}"
await self.stream_manager.delete_stream(stream, delete_file=delete_from_download_dir)
log.info("Deleted file: %s", stream.file_name)
log.info(message)
result = True
return result
@@ -1829,7 +1892,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def jsonrpc_publish(
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
description=None, author=None, language=None, license=None,
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
license_url=None, thumbnail=None, preview=None, nsfw=None,
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
claim_address=None, change_address=None):
"""
@@ -1853,7 +1916,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
[--description=<description>] [--author=<author>] [--language=<language>]
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
[--preview=<preview>] [--nsfw=<nsfw>]
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
[--claim_address=<claim_address>] [--change_address=<change_address>]
@@ -1889,7 +1952,6 @@ class Daemon(metaclass=JSONRPCServerType):
--thumbnail=<thumbnail> : (str) thumbnail url
--preview=<preview> : (str) preview url
--nsfw=<nsfw> : (bool) whether the content is nsfw
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
@@ -1913,7 +1975,9 @@ class Daemon(metaclass=JSONRPCServerType):
"""
try:
parse_lbry_uri(name)
parsed = parse_lbry_uri(name)
if parsed.name != name:
raise Exception("Name given to publish has invalid characters")
except (TypeError, URIParseError):
raise Exception("Invalid name given to publish")
@@ -1929,9 +1993,10 @@ class Daemon(metaclass=JSONRPCServerType):
account = self.get_account_or_default(account_id)
available = await account.get_balance()
existing_claims = []
existing_claims = await account.get_claims(claim_name=name)
if len(existing_claims) > 1:
log.warning("found %i claims for the name %s", len(existing_claims), name)
if amount >= available:
existing_claims = await account.get_claims(claim_name=name)
if len(existing_claims) == 1:
available += existing_claims[0].get_estimator(self.ledger).effective_amount
if amount >= available:
@@ -1990,39 +2055,62 @@ class Daemon(metaclass=JSONRPCServerType):
}
}
# this will be used to verify the format with lbrynet.schema
claim_copy = deepcopy(claim_dict)
if sources is not None:
claim_dict['stream']['source'] = sources
claim_copy['stream']['source'] = sources
elif file_path is not None:
sd_to_delete = None
if file_path:
if not os.path.isfile(file_path):
raise Exception("invalid file path to publish")
if os.path.getsize(file_path) == 0:
raise Exception(f"Cannot publish empty file {file_path}")
if existing_claims:
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
# since the file hasn't yet been made into a stream, we don't have
# a valid Source for the claim when validating the format, we'll use a fake one
claim_copy['stream']['source'] = {
claim_dict['stream']['source'] = {
'version': '_0_0_1',
'sourceType': 'lbry_sd_hash',
'source': '0' * 96,
'contentType': ''
}
elif not existing_claims:
raise Exception("no previous stream to update")
else:
# there is no existing source to use, and a file was not provided to make a new one
raise Exception("no source provided to publish")
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
try:
ClaimDict.load_dict(claim_copy)
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
# the metadata to use in the claim can be serialized by lbrynet.schema
except DecodeError as err:
# there was a problem with a metadata field, raise an error here rather than
# waiting to find out when we go to publish the claim (after having made the stream)
raise Exception(f"invalid publish metadata: {err}")
certificate = None
if channel_id or channel_name:
certificate = await self.get_channel_or_error(
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
)
if file_path:
stream = await self.stream_manager.create_stream(file_path)
claim_dict['stream']['source']['source'] = stream.sd_hash
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
if sd_to_delete:
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
if stream_to_delete:
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
log.info("updating claim to stream generated from %s, deleted previous stream %s",
sd_to_delete[:8], file_path)
else:
log.info("previous stream %s from claim was not saved locally, nothing to delete",
sd_to_delete[:8])
else:
log.info("generated stream from %s for claim", file_path)
else:
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
sd_hash = claim_dict['stream']['source']['source']
log.info("Publish: %s", {
'name': name,
'file_path': file_path,
@@ -2033,35 +2121,14 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel_id,
'channel_name': channel_name
})
from lbrynet.extras.daemon.mime_types import guess_media_type
if file_path:
if not os.path.isfile(file_path):
raise Exception(f"File {file_path} not found")
if os.path.getsize(file_path) == 0:
raise Exception(f"Cannot publish empty file {file_path}")
claim_dict['stream']['source'] = {}
stream = await self.stream_manager.create_stream(file_path)
stream_hash = stream.stream_hash
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
claim_dict['stream']['source']['source'] = stream.sd_hash
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
else:
if not ('source' not in claim_dict['stream'] and existing_claims):
raise Exception("no previous stream to update")
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
tx = await self.wallet_manager.claim_name(
account, name, amount, claim_dict, certificate, claim_address
)
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
if stream_hash:
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
await self.analytics_manager.send_claim_action('publish')
nout = 0
txo = tx.outputs[nout]
@@ -2289,7 +2356,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
sort_claim_results(claims['claims'])
claims['claims'] = sort_claim_results(claims['claims'])
return claims
@requires(WALLET_COMPONENT)
@@ -2327,7 +2394,6 @@ class Daemon(metaclass=JSONRPCServerType):
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2569,8 +2635,7 @@ class Daemon(metaclass=JSONRPCServerType):
(str) Success/Fail message or (dict) decoded data
"""
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
blob = await downloader.get_blob(blob_hash, self.dht_node)
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
if read:
with open(blob.file_path, 'rb') as handle:
return handle.read().decode()
@@ -2591,7 +2656,8 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(str) Success/fail message
"""
if not blob_hash or not is_valid_blobhash(blob_hash):
return f"Invalid blob hash to delete '{blob_hash}'"
streams = self.stream_manager.get_filtered_streams(sd_hash=blob_hash)
if streams:
await self.stream_manager.delete_stream(streams[0])
@@ -2721,7 +2787,7 @@ class Daemon(metaclass=JSONRPCServerType):
else:
blobs = []
if stream_hash:
blobs.extend([b.blob_hash for b in await self.storage.get_blobs_for_stream(stream_hash)])
blobs.extend([b.blob_hash for b in (await self.storage.get_blobs_for_stream(stream_hash))[:-1]])
else:
blobs = list(self.blob_manager.completed_blob_hashes)
if needed:
@@ -2791,7 +2857,19 @@ class Daemon(metaclass=JSONRPCServerType):
(list) list of blobs reflected
"""
raise NotImplementedError()
server, port = kwargs.get('server'), kwargs.get('port')
if server and port:
port = int(port)
else:
server, port = random.choice(self.conf.reflector_servers)
reflected = await asyncio.gather(*[
stream.upload_to_reflector(server, port)
for stream in self.stream_manager.get_filtered_streams(**kwargs)
])
total = []
for reflected_for_stream in reflected:
total.extend(reflected_for_stream)
return total
@requires(DHT_COMPONENT)
async def jsonrpc_peer_ping(self, node_id, address, port):
@@ -2802,23 +2880,22 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
peer_ping (<node_id> | --node_id=<node_id>) (<address> | --address=<address>) (<port> | --port=<port>)
Options:
None
Returns:
(str) pong, or {'error': <error message>} if an error is encountered
"""
peer = None
if node_id and address and port:
peer = self.component_manager.peer_manager.get_peer(address, unhexlify(node_id), udp_port=int(port))
if not peer:
peer = self.component_manager.peer_manager.make_peer(
address, unhexlify(node_id), udp_port=int(port)
)
peer = self.component_manager.peer_manager.get_kademlia_peer(unhexlify(node_id), address,
udp_port=int(port))
try:
return await self.dht_node.protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
return {'error': 'timeout'}
if not peer:
return {'error': 'peer not found'}
try:
result = await peer.ping()
return result.decode()
except TimeoutError:
return {'error': 'ping timeout'}
@requires(DHT_COMPONENT)
def jsonrpc_routing_table_get(self):
+114 -122
View File
@@ -1,13 +1,15 @@
import asyncio
import collections
import logging
import aiohttp
import typing
from lbrynet import utils
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.conf import Config
from lbrynet.extras import system_info
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
@@ -19,102 +21,137 @@ HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
UPNP_SETUP = "UPnP Setup"
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
TIME_TO_FIRST_BYTES = "Time To First Bytes"
log = logging.getLogger(__name__)
class Manager:
def _event_properties(installation_id: str, session_id: str,
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
properties = {
'lbry_id': installation_id,
'session_id': session_id,
}
properties.update(event_properties or {})
return properties
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
total_duration: typing.Optional[float], download_id: str, name: str,
outpoint: str, active_peer_count: int, tried_peers_count: int,
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_download_duration: typing.Optional[float] = None,
error: typing.Optional[str] = None) -> typing.Dict:
return {
"external_ip": external_ip,
"download_id": download_id,
"total_duration": round(total_duration, 4),
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
"error": error,
'name': name,
"outpoint": outpoint,
"node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.reflector_servers) > 0,
"fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count,
"tried_peers_count": tried_peers_count,
"sd_blob_hash": sd_hash,
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
"head_blob_hash": head_blob_hash,
"head_blob_length": head_blob_length,
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
}
def _make_context(platform):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
class AnalyticsManager:
def __init__(self, conf: Config, installation_id: str, session_id: str):
self.conf = conf
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = self._make_context(system_info.get_platform(), 'torba')
self.context = _make_context(system_info.get_platform())
self.installation_id = installation_id
self.session_id = session_id
self.task: asyncio.Task = None
self.external_ip: typing.Optional[str] = None
@property
def is_started(self):
return self.task is not None
def start(self):
async def start(self):
if self._enabled and self.task is None:
self.external_ip = await utils.get_external_ip()
self.task = asyncio.create_task(self.run())
async def run(self):
while True:
await self._send_heartbeat()
await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
async def _post(self, data: typing.Dict):
request_kwargs = {
'method': 'POST',
'url': self.url + endpoint,
'url': self.url + '/track',
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with aiohttp.request(**request_kwargs) as response:
async with utils.aiohttp_request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
async def track(self, event):
async def track(self, event: typing.Dict):
"""Send a single tracking event"""
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post('/track', event)
async def send_new_download_start(self, download_id, name, claim_dict):
await self._send_new_download_stats("start", download_id, name, claim_dict)
async def send_new_download_success(self, download_id, name, claim_dict):
await self._send_new_download_stats("success", download_id, name, claim_dict)
async def send_new_download_fail(self, download_id, name, claim_dict, e):
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
await self.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
await self._post(event)
async def send_upnp_setup_success_fail(self, success, status):
await self.track(
@@ -133,20 +170,33 @@ class Manager:
async def send_server_startup_error(self, message):
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
async def send_download_started(self, id_, name, claim_dict=None):
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
total_duration: typing.Optional[float], download_id: str,
name: str, outpoint: str, found_peers_count: int,
tried_peers_count: int, added_fixed_peers: bool,
fixed_peers_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_duration: typing.Optional[int] = None,
error: typing.Optional[str] = None):
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
)))
async def send_download_finished(self, download_id, name, sd_hash):
await self.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
self._event(
DOWNLOAD_FINISHED, {
'download_id': download_id,
'name': name,
'stream_info': sd_hash
}
)
)
async def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
async def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
async def send_claim_action(self, action):
await self.track(self._event(CLAIM_ACTION, {'action': action}))
@@ -159,69 +209,11 @@ class Manager:
async def _send_heartbeat(self):
await self.track(self._event(HEARTBEAT))
def _event(self, event, event_properties=None):
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'properties': _event_properties(self.installation_id, self.session_id, properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
@@ -1,10 +1,10 @@
import asyncio
import aiohttp
import time
import logging
import json
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
from lbrynet.utils import aiohttp_request
log = logging.getLogger(__name__)
@@ -52,8 +52,8 @@ class MarketFeed:
def is_online(self):
return self._online
async def _make_request(self):
async with aiohttp.request('get', self.url, params=self.params) as response:
async def _make_request(self) -> str:
async with aiohttp_request('get', self.url, params=self.params) as response:
return (await response.read()).decode()
def _handle_response(self, response):
@@ -219,7 +219,7 @@ class ExchangeRateManager:
def convert_currency(self, from_currency, to_currency, amount):
rates = [market.rate for market in self.market_feeds]
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
if from_currency == to_currency:
return amount
+1 -2
View File
@@ -1,5 +1,4 @@
import asyncio
import aiohttp
import json
import logging.handlers
import traceback
@@ -51,7 +50,7 @@ class HTTPSLogglyHandler(logging.Handler):
async def _emit(self, record):
payload = self.format(record)
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
def emit(self, record):
+4 -2
View File
@@ -154,5 +154,7 @@ types_map = {
def guess_media_type(path):
_, ext = os.path.splitext(path)
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
return types_map.get(ext, default)
return types_map.get(
ext.strip().lower(),
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
)
+193 -107
View File
@@ -1,8 +1,10 @@
import os
import logging
import sqlite3
import typing
import asyncio
import binascii
import time
from torba.client.basedatabase import SQLiteMixin
from lbrynet.conf import Config
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
@@ -104,14 +106,104 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
return StoredStreamClaim(*claim_fields)
def _batched_select(transaction, query, parameters):
for start_index in range(0, len(parameters), 900):
current_batch = parameters[start_index:start_index+900]
def _batched_select(transaction, query, parameters, batch_size=900):
for start_index in range(0, len(parameters), batch_size):
current_batch = parameters[start_index:start_index+batch_size]
bind = "({})".format(','.join(['?'] * len(current_batch)))
for result in transaction.execute(query.format(bind), current_batch):
yield result
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", [
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
claim = StoredStreamClaim(stream_hash, *claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
signed_claims[claim.channel_claim_id] = []
signed_claims[claim.channel_claim_id].append(claim)
files.append(
{
"rowid": rowid,
"stream_hash": stream_hash,
"file_name": file_name, # hex
"download_directory": download_dir, # hex
"blob_data_rate": data_rate,
"status": status,
"sd_hash": sd_hash,
"key": stream_key,
"stream_name": stream_name, # hex
"suggested_file_name": suggested_file_name, # hex
"claim": claim
}
)
for claim_name, claim_id in _batched_select(
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
list(signed_claims.keys())):
for claim in signed_claims[claim_id]:
claim.channel_name = claim_name
return files
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
blob_hashes.append((descriptor.sd_hash, ))
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
data_payment_rate: float, status: str) -> int:
transaction.execute(
"insert or replace into file values (?, ?, ?, ?, ?)",
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
)
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
class SQLiteStorage(SQLiteMixin):
CREATE_TABLES_QUERY = """
pragma foreign_keys=on;
@@ -184,11 +276,12 @@ class SQLiteStorage(SQLiteMixin):
);
"""
def __init__(self, conf: Config, path, loop=None):
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
super().__init__(path)
self.conf = conf
self.content_claim_callbacks = {}
self.loop = loop or asyncio.get_event_loop()
self.time_getter = time_getter or time.time
async def run_and_return_one_or_none(self, query, *args):
for row in await self.db.execute_fetchall(query, args):
@@ -202,15 +295,16 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # blob functions # # # # # # # # #
def add_completed_blob(self, blob_hash: str):
log.debug("Adding a completed blob. blob_hash=%s", blob_hash)
return self.db.execute("update blob set status='finished' where blob.blob_hash=?", (blob_hash, ))
def set_should_announce(self, blob_hash: str, next_announce_time: int, should_announce: int):
return self.db.execute(
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
)
def add_completed_blob(self, blob_hash: str, length: int):
def _add_blob(transaction: sqlite3.Connection):
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
(blob_hash, length, 0, 0, "pending", 0, 0)
)
transaction.execute(
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
)
return self.db.run(_add_blob)
def get_blob_status(self, blob_hash: str):
return self.run_and_return_one_or_none(
@@ -247,8 +341,9 @@ class SQLiteStorage(SQLiteMixin):
"select count(*) from blob where status='finished'"
)
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
def _update_last_announced_blobs(transaction: sqlite3.Connection):
last_announced = self.time_getter()
return transaction.executemany(
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
"where blob_hash=?",
@@ -259,7 +354,7 @@ class SQLiteStorage(SQLiteMixin):
def should_single_announce_blobs(self, blob_hashes, immediate=False):
def set_single_announce(transaction):
now = self.loop.time()
now = int(self.time_getter())
for blob_hash in blob_hashes:
if immediate:
transaction.execute(
@@ -274,21 +369,23 @@ class SQLiteStorage(SQLiteMixin):
def get_blobs_to_announce(self):
def get_and_update(transaction):
timestamp = self.loop.time()
if self.conf.announce_head_blobs_only:
timestamp = int(self.time_getter())
if self.conf.announce_head_and_sd_only:
r = transaction.execute(
"select blob_hash from blob "
"where blob_hash is not null and "
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished'",
(timestamp,)
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
else:
r = transaction.execute(
"select blob_hash from blob where blob_hash is not null "
"and next_announce_time<? and status='finished'", (timestamp,)
"and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
blobs = [b[0] for b in r.fetchall()]
return blobs
return [b[0] for b in r.fetchall()]
return self.db.run(get_and_update)
def delete_blobs_from_db(self, blob_hashes):
@@ -296,11 +393,31 @@ class SQLiteStorage(SQLiteMixin):
transaction.executemany(
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
)
return self.db.run(delete_blobs)
return self.db.run_with_foreign_keys_disabled(delete_blobs)
def get_all_blob_hashes(self):
return self.run_and_return_list("select blob_hash from blob")
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
to_update = [
(blob_hash, )
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
if blob_hash not in blob_files
]
transaction.executemany(
"update blob set status='pending' where blob_hash=?",
to_update
)
return {
blob_hash
for blob_hash, in _batched_select(
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
list(blob_files)
)
}
return self.db.run(_sync_blobs)
# # # # # # # # # stream functions # # # # # # # # #
async def stream_exists(self, sd_hash: str) -> bool:
@@ -313,42 +430,20 @@ class SQLiteStorage(SQLiteMixin):
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
return streams is not None
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
return self.run_and_return_one_or_none(
"select rowid from file where stream_hash=?", stream_hash
)
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
def _store_stream(transaction: sqlite3.Connection):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
return self.db.run(store_stream, sd_blob, descriptor)
return self.db.run(_store_stream)
def get_blobs_for_stream(self, stream_hash, only_completed=False):
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
def _get_blobs_for_stream(transaction):
crypt_blob_infos = []
stream_blobs = transaction.execute(
"select blob_hash, position, iv from stream_blob where stream_hash=?", (stream_hash, )
"select blob_hash, position, iv from stream_blob where stream_hash=? "
"order by position asc", (stream_hash, )
).fetchall()
if only_completed:
lengths = transaction.execute(
@@ -370,7 +465,8 @@ class SQLiteStorage(SQLiteMixin):
for blob_hash, position, iv in stream_blobs:
blob_length = blob_length_dict.get(blob_hash, 0)
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
if not blob_hash:
break
return crypt_blob_infos
return self.db.run(_get_blobs_for_stream)
@@ -385,68 +481,58 @@ class SQLiteStorage(SQLiteMixin):
)
def delete_stream(self, descriptor: 'StreamDescriptor'):
def _delete_stream(transaction: sqlite3.Connection):
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
transaction.executemany("delete from blob where blob_hash=?",
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
return self.db.run(_delete_stream)
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
# # # # # # # # # file stuff # # # # # # # # #
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
def save_downloaded_file(self, stream_hash, file_name, download_directory,
data_payment_rate) -> typing.Awaitable[int]:
return self.save_published_file(
stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
status="running"
stream_hash, file_name, download_directory, data_payment_rate, status="running"
)
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
status="finished"):
return self.db.execute(
"insert into file values (?, ?, ?, ?, ?)",
(stream_hash, file_name, download_directory, data_payment_rate, status)
)
status="finished") -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name) -> typing.Dict:
return {
"row_id": rowid,
"stream_hash": stream_hash,
"file_name": file_name,
"download_directory": download_dir,
"blob_data_rate": data_rate,
"status": status,
"sd_hash": sd_hash,
"key": stream_key,
"stream_name": stream_name,
"suggested_file_name": suggested_file_name
}
def _get_all_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
file_infos = list(map(lambda a: _lbry_file_dict(*a), transaction.execute(
"select file.rowid, file.*, stream.* "
"from file inner join stream on file.stream_hash=stream.stream_hash"
).fetchall()))
stream_hashes = [file_info['stream_hash'] for file_info in file_infos]
claim_infos = get_claims_from_stream_hashes(transaction, stream_hashes)
for index in range(len(file_infos)): # pylint: disable=consider-using-enumerate
file_infos[index]['claim'] = claim_infos.get(file_infos[index]['stream_hash'])
return file_infos
results = await self.db.run(_get_all_files)
if results:
return results
return []
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
def change_file_status(self, stream_hash: str, new_status: str):
log.info("update file status %s -> %s", stream_hash, new_status)
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
stream_hash
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
download_directory: str):
def _recover(transaction: sqlite3.Connection):
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
for descriptor, sd_blob in descriptors_and_sds:
content_claim = transaction.execute(
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
).fetchone()
delete_stream(transaction, descriptor) # this will also delete the content claim
store_stream(transaction, sd_blob, descriptor)
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
download_directory, 0.0, 'stopped')
if content_claim:
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
transaction.executemany(
f"update file set download_directory=? where stream_hash=?",
[(download_dir, stream_hash) for stream_hash in stream_hashes]
)
await self.db.run_with_foreign_keys_disabled(_recover)
def get_all_stream_hashes(self):
return self.run_and_return_list("select stream_hash from stream")
@@ -682,7 +768,7 @@ class SQLiteStorage(SQLiteMixin):
if success:
return self.db.execute(
"insert or replace into reflected_stream values (?, ?, ?)",
(sd_hash, reflector_address, self.loop.time())
(sd_hash, reflector_address, self.time_getter())
)
return self.db.execute(
"delete from reflected_stream where sd_hash=? and reflector_address=?",
@@ -694,5 +780,5 @@ class SQLiteStorage(SQLiteMixin):
"select s.sd_hash from stream s "
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
"where r.timestamp is null or r.timestamp < ?",
self.loop.time() - self.conf.auto_re_reflect_interval
int(self.time_getter()) - 86400
)
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.0/lbrycrd-linux.zip'
)
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
+15 -2
View File
@@ -1,6 +1,7 @@
import json
import logging
import binascii
from hashlib import sha256
from lbrynet.schema.validator import validate_claim_id
from torba.client.baseaccount import BaseAccount
@@ -24,6 +25,17 @@ class Account(BaseAccount):
super().__init__(*args, **kwargs)
self.certificates = {}
@property
def hash(self) -> bytes:
h = sha256(json.dumps(self.to_dict(False)).encode())
for cert in sorted(self.certificates.keys()):
h.update(cert.encode())
return h.digest()
def apply(self, d: dict):
super().apply(d)
self.certificates.update(d.get('certificates', {}))
def add_certificate_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
self.certificates[ref.id] = private_key
@@ -176,9 +188,10 @@ class Account(BaseAccount):
account.certificates = d.get('certificates', {})
return account
def to_dict(self):
def to_dict(self, include_certificates=True):
d = super().to_dict()
d['certificates'] = self.certificates
if include_certificates:
d['certificates'] = self.certificates
return d
async def get_details(self, **kwargs):
+3 -2
View File
@@ -74,9 +74,10 @@ def verify_proof(proof, root_hash, name):
if 'txhash' in proof and 'nOut' in proof:
if not verified_value:
raise InvalidProofError("mismatch between proof claim and outcome")
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
if 'txhash' in proof and 'nOut' in proof:
if name != reverse_computed_name[::-1]:
if name != target:
raise InvalidProofError("name did not match proof")
if not name.startswith(reverse_computed_name[::-1]):
if not name.startswith(target):
raise InvalidProofError("name fragment does not match proof")
return True
-1
View File
@@ -260,7 +260,6 @@ class LbryWalletManager(BaseWalletManager):
async def resolve(self, *uris, **kwargs):
page = kwargs.get('page', 0)
page_size = kwargs.get('page_size', 10)
check_cache = kwargs.get('check_cache', False) # TODO: put caching back (was force_refresh parameter)
ledger: MainNetLedger = self.default_account.ledger
results = await ledger.resolve(page, page_size, *uris)
if 'error' not in results:
+14 -6
View File
@@ -369,8 +369,14 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
return {'error': 'name is not claimed'}
if 'proof' in result:
proof_name = name
if 'name' in result:
proof_name = result['name']
name = result['name']
if 'normalized_name' in result:
proof_name = result['normalized_name']
try:
verify_proof(result['proof'], claim_trie_root, name)
verify_proof(result['proof'], claim_trie_root, proof_name)
except InvalidProofError:
return {'error': "Proof was invalid"}
return _parse_proof_result(name, result)
@@ -421,7 +427,8 @@ def _decode_claim_result(claim):
log.warning('Got an invalid claim while parsing, please report: %s', claim)
claim['hex'] = None
claim['value'] = None
claim['error'] = "Failed to parse: missing value"
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
claim['error'] = "Failed to parse: missing value." + backend_message
return claim
try:
decoded = smart_decode(claim['value'])
@@ -445,11 +452,11 @@ def _handle_claim_result(results):
if results['error'] in ['name is not claimed', 'claim not found']:
if 'claim_id' in results:
raise UnknownClaimID(results['claim_id'])
elif 'name' in results:
if 'name' in results:
raise UnknownNameError(results['name'])
elif 'uri' in results:
if 'uri' in results:
raise UnknownURI(results['uri'])
elif 'outpoint' in results:
if 'outpoint' in results:
raise UnknownOutpoint(results['outpoint'])
raise Exception(results['error'])
@@ -486,6 +493,7 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
continue
if winner is None:
winner = claim
elif claim['claim_sequence'] < winner['claim_sequence']:
elif claim['height'] < winner['height'] or \
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
winner = claim
return winner
@@ -80,7 +80,6 @@ class LBRYBlockProcessor(BlockProcessor):
if claim_info.cert_id:
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
self.db.put_claim_info(claim_id, claim_info)
self.db.put_claim_for_name(claim_info.name, claim_id)
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
return claim_id, None
@@ -116,7 +115,6 @@ class LBRYBlockProcessor(BlockProcessor):
if undo_claim_info.cert_id:
cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address)
self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
self.db.put_claim_for_name(undo_claim_info.name, claim_id)
self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
def backup_txs(self, txs):
+14 -47
View File
@@ -13,10 +13,9 @@ class LBRYDB(DB):
def __init__(self, *args, **kwargs):
self.claim_cache = {}
self.claims_for_name_cache = {}
self.claims_signed_by_cert_cache = {}
self.outpoint_to_claim_id_cache = {}
self.claims_db = self.names_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
# stores deletes not yet flushed to disk
self.pending_abandons = {}
super().__init__(*args, **kwargs)
@@ -24,7 +23,6 @@ class LBRYDB(DB):
def close(self):
self.batched_flush_claims()
self.claims_db.close()
self.names_db.close()
self.signatures_db.close()
self.outpoint_to_claim_id_db.close()
self.claim_undo_db.close()
@@ -42,12 +40,10 @@ class LBRYDB(DB):
return
log_reason('closing claim DBs to re-open', for_sync)
self.claims_db.close()
self.names_db.close()
self.signatures_db.close()
self.outpoint_to_claim_id_db.close()
self.claim_undo_db.close()
self.claims_db = self.db_class('claims', for_sync)
self.names_db = self.db_class('names', for_sync)
self.signatures_db = self.db_class('signatures', for_sync)
self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync)
self.claim_undo_db = self.db_class('claim_undo', for_sync)
@@ -60,21 +56,18 @@ class LBRYDB(DB):
def batched_flush_claims(self):
with self.claims_db.write_batch() as claims_batch:
with self.names_db.write_batch() as names_batch:
with self.signatures_db.write_batch() as signed_claims_batch:
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
self.flush_claims(claims_batch, names_batch, signed_claims_batch,
outpoint_batch)
with self.signatures_db.write_batch() as signed_claims_batch:
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch)
def flush_claims(self, batch, names_batch, signed_claims_batch, outpoint_batch):
def flush_claims(self, batch, signed_claims_batch, outpoint_batch):
flush_start = time.time()
write_claim, write_name, write_cert = batch.put, names_batch.put, signed_claims_batch.put
write_claim, write_cert = batch.put, signed_claims_batch.put
write_outpoint = outpoint_batch.put
delete_claim, delete_outpoint, delete_name = batch.delete, outpoint_batch.delete, names_batch.delete
delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete
delete_cert = signed_claims_batch.delete
for claim_id, outpoints in self.pending_abandons.items():
claim = self.get_claim_info(claim_id)
self.remove_claim_for_name(claim.name, claim_id)
if claim.cert_id:
self.remove_claim_from_certificate_claims(claim.cert_id, claim_id)
self.remove_certificate(claim_id)
@@ -86,11 +79,6 @@ class LBRYDB(DB):
write_claim(key, claim)
else:
delete_claim(key)
for name, claims in self.claims_for_name_cache.items():
if not claims:
delete_name(name)
else:
write_name(name, msgpack.dumps(claims))
for cert_id, claims in self.claims_signed_by_cert_cache.items():
if not claims:
delete_cert(cert_id)
@@ -101,15 +89,13 @@ class LBRYDB(DB):
write_outpoint(key, claim_id)
else:
delete_outpoint(key)
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints, {:,d} names '
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints '
'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...'
.format(self.db_height,
len(self.claim_cache), len(self.outpoint_to_claim_id_cache),
len(self.claims_for_name_cache),
len(self.claims_signed_by_cert_cache), len(self.pending_abandons),
time.time() - flush_start))
self.claim_cache = {}
self.claims_for_name_cache = {}
self.claims_signed_by_cert_cache = {}
self.outpoint_to_claim_id_cache = {}
self.pending_abandons = {}
@@ -117,7 +103,6 @@ class LBRYDB(DB):
def assert_flushed(self, flush_data):
super().assert_flushed(flush_data)
assert not self.claim_cache
assert not self.claims_for_name_cache
assert not self.claims_signed_by_cert_cache
assert not self.outpoint_to_claim_id_cache
assert not self.pending_abandons
@@ -142,27 +127,6 @@ class LBRYDB(DB):
key = tx_hash + struct.pack('>I', tx_idx)
return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key)
def get_claims_for_name(self, name):
if name in self.claims_for_name_cache:
return self.claims_for_name_cache[name]
db_claims = self.names_db.get(name)
return msgpack.loads(db_claims) if db_claims else {}
def put_claim_for_name(self, name, claim_id):
self.logger.info("[+] Adding claim {} for name {}.".format(hash_to_hex_str(claim_id), name))
claims = self.get_claims_for_name(name)
claims.setdefault(claim_id, max(claims.values() or [0]) + 1)
self.claims_for_name_cache[name] = claims
def remove_claim_for_name(self, name, claim_id):
self.logger.info("[-] Removing claim from name: {} - {}".format(hash_to_hex_str(claim_id), name))
claims = self.get_claims_for_name(name)
claim_n = claims.pop(claim_id)
for _claim_id, number in claims.items():
if number > claim_n:
claims[_claim_id] = number - 1
self.claims_for_name_cache[name] = claims
def get_signed_claim_ids_by_cert_id(self, cert_id):
if cert_id in self.claims_signed_by_cert_cache:
return self.claims_signed_by_cert_cache[cert_id]
@@ -170,17 +134,20 @@ class LBRYDB(DB):
return msgpack.loads(db_claims, use_list=True) if db_claims else []
def put_claim_id_signed_by_cert_id(self, cert_id, claim_id):
self.logger.info("[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
self.logger.info(msg)
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
certs.append(claim_id)
self.claims_signed_by_cert_cache[cert_id] = certs
def remove_certificate(self, cert_id):
self.logger.info("[-] Removing certificate: {}".format(hash_to_hex_str(cert_id)))
msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))
self.logger.info(msg)
self.claims_signed_by_cert_cache[cert_id] = []
def remove_claim_from_certificate_claims(self, cert_id, claim_id):
self.logger.info("[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
self.logger.info(msg)
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
if claim_id in certs:
certs.remove(claim_id)
+93 -78
View File
@@ -1,4 +1,5 @@
import math
import unicodedata as uda
from binascii import unhexlify, hexlify
from torba.rpc.jsonrpc import RPCError
@@ -6,7 +7,7 @@ from torba.server.hash import hash_to_hex_str
from torba.server.session import ElectrumX
from torba.server import util
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH
from lbrynet.schema.error import URIParseError
from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
from lbrynet.extras.wallet.server.db import LBRYDB
@@ -131,22 +132,29 @@ class LBRYElectrumX(ElectrumX):
claim_ids = self.get_claim_ids_signed_by(certificate_id)
return await self.batched_formatted_claims_from_daemon(claim_ids)
def claimtrie_getclaimssignedbyidminimal(self, certificate_id):
claim_ids = self.get_claim_ids_signed_by(certificate_id)
ret = []
for claim_id in claim_ids:
raw_claim_id = unhexlify(claim_id)[::-1]
info = self.db.get_claim_info(raw_claim_id)
if info:
ret.append({
'claim_id': claim_id,
'height': info.height,
'name': info.name.decode()
})
return ret
def get_claim_ids_signed_by(self, certificate_id):
raw_certificate_id = unhexlify(certificate_id)[::-1]
raw_claim_ids = self.db.get_signed_claim_ids_by_cert_id(raw_certificate_id)
return list(map(hash_to_hex_str, raw_claim_ids))
def get_signed_claims_with_name_for_channel(self, channel_id, name):
claim_ids_for_name = list(self.db.get_claims_for_name(name.encode('ISO-8859-1')).keys())
claim_ids_for_name = set(map(hash_to_hex_str, claim_ids_for_name))
channel_claim_ids = set(self.get_claim_ids_signed_by(channel_id))
return claim_ids_for_name.intersection(channel_claim_ids)
async def claimtrie_getclaimssignedbynthtoname(self, name, n):
n = int(n)
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
if n == sequence:
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim_id))
claim = self.claimtrie_getnthclaimforname(name, n)
if claim and 'claim_id' in claim:
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim['claim_id']))
async def claimtrie_getclaimsintx(self, txid):
# TODO: this needs further discussion.
@@ -161,28 +169,27 @@ class LBRYElectrumX(ElectrumX):
if proof_has_winning_claim(proof):
tx_hash, nout = proof['txhash'], int(proof['nOut'])
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
result['transaction'] = transaction_info['hex']
result['height'] = (self.db.db_height - transaction_info['confirmations']) + 1
result['transaction'] = transaction_info['hex'] # should have never included this (or the call to get it)
raw_claim_id = self.db.get_claim_id_from_outpoint(unhexlify(tx_hash)[::-1], nout)
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
if sequence:
claim_id = hexlify(raw_claim_id[::-1]).decode()
claim_info = await self.daemon.getclaimbyid(claim_id)
if not claim_info or not claim_info.get('value'):
claim_info = await self.slow_get_claim_by_id_using_name(claim_id)
result['claim_sequence'] = sequence
result['claim_id'] = claim_id
supports = self.format_supports_from_daemon(claim_info.get('supports', [])) # fixme: lbrycrd#124
result['supports'] = supports
else:
self.logger.warning('tx has no claims in db: %s %s', tx_hash, nout)
claim_id = hexlify(raw_claim_id[::-1]).decode()
claim = await self.claimtrie_getclaimbyid(claim_id)
result.update(claim)
return result
async def claimtrie_getnthclaimforname(self, name, n):
n = int(n)
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
if n == sequence:
return await self.claimtrie_getclaimbyid(hash_to_hex_str(claim_id))
result = await self.claimtrie_getclaimsforname(name)
if 'claims' in result and len(result['claims']) > n >= 0:
# TODO: revist this after lbrycrd_#209 to see if we can sort by claim_sequence at this point
result['claims'].sort(key=lambda c: (int(c['height']), int(c['nout'])))
result['claims'][n]['claim_sequence'] = n
return result['claims'][n]
async def claimtrie_getpartialmatch(self, name, part):
result = await self.claimtrie_getclaimsforname(name)
if 'claims' in result:
return next(filter(lambda x: x['claim_id'].starts_with(part), result['claims']), None)
async def claimtrie_getclaimsforname(self, name):
claims = await self.daemon.getclaimsforname(name)
@@ -198,37 +205,38 @@ class LBRYElectrumX(ElectrumX):
async def batched_formatted_claims_from_daemon(self, claim_ids):
claims = await self.daemon.getclaimsbyids(claim_ids)
result = []
for claim, claim_id in zip(claims, claim_ids):
for claim in claims:
if claim and claim.get('value'):
result.append(self.format_claim_from_daemon(claim))
else:
recovered_claim = await self.slow_get_claim_by_id_using_name(claim_id)
if recovered_claim:
result.append(self.format_claim_from_daemon(recovered_claim))
return result
def format_claim_from_daemon(self, claim, name=None):
'''Changes the returned claim data to the format expected by lbrynet and adds missing fields.'''
"""Changes the returned claim data to the format expected by lbrynet and adds missing fields."""
if not claim:
return {}
name = name or claim['name']
# this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
# it will be fixed after the lbrycrd upstream merge to v17 is done
# it originated as a fear of terminals not supporting unicode. alas, they all do
if 'name' in claim:
name = claim['name'].encode('ISO-8859-1').decode()
claim_id = claim['claimId']
raw_claim_id = unhexlify(claim_id)[::-1]
if not self.db.get_claim_info(raw_claim_id):
#raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
info = self.db.get_claim_info(raw_claim_id)
if not info:
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
return {}
address = self.db.get_claim_info(raw_claim_id).address.decode()
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
if not sequence:
return {}
supports = self.format_supports_from_daemon(claim.get('supports', [])) # fixme: lbrycrd#124
address = info.address.decode()
supports = self.format_supports_from_daemon(claim.get('supports', []))
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
height = get_from_possible_keys(claim, 'height', 'nHeight')
effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
return {
result = {
"name": name,
"claim_id": claim['claimId'],
"txid": claim['txid'],
@@ -237,12 +245,19 @@ class LBRYElectrumX(ElectrumX):
"depth": self.db.db_height - height,
"height": height,
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
"claim_sequence": sequence, # from index
"address": address, # from index
"supports": supports, # fixme: to be included in lbrycrd#124
"supports": supports,
"effective_amount": effective_amount,
"valid_at_height": valid_at_height # TODO PR lbrycrd to include it
"valid_at_height": valid_at_height
}
if 'claim_sequence' in claim:
# TODO: ensure that lbrycrd #209 fills in this value
result['claim_sequence'] = claim['claim_sequence']
else:
result['claim_sequence'] = -1
if 'normalized_name' in claim:
result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
return result
def format_supports_from_daemon(self, supports):
return [[support['txid'], support['n'], get_from_possible_keys(support, 'amount', 'nAmount')] for
@@ -251,8 +266,6 @@ class LBRYElectrumX(ElectrumX):
async def claimtrie_getclaimbyid(self, claim_id):
self.assert_claim_id(claim_id)
claim = await self.daemon.getclaimbyid(claim_id)
if not claim or not claim.get('value'):
claim = await self.slow_get_claim_by_id_using_name(claim_id)
return self.format_claim_from_daemon(claim)
async def claimtrie_getclaimsbyids(self, *claim_ids):
@@ -279,20 +292,16 @@ class LBRYElectrumX(ElectrumX):
pass
raise RPCError(1, f'{value} should be a claim id hash')
async def slow_get_claim_by_id_using_name(self, claim_id):
# TODO: temporary workaround for a lbrycrd bug on indexing. Should be removed when it gets stable
raw_claim_id = unhexlify(claim_id)[::-1]
claim = self.db.get_claim_info(raw_claim_id)
if claim:
name = claim.name.decode('ISO-8859-1')
claims = await self.daemon.getclaimsforname(name)
for claim in claims['claims']:
if claim['claimId'] == claim_id:
claim['name'] = name
self.logger.warning(
'Recovered a claim missing from lbrycrd index: %s %s', name, claim_id
)
return claim
def normalize_name(self, name):
# this is designed to match lbrycrd; change it here if it changes there
return uda.normalize('NFD', name).casefold()
def claim_matches_name(self, claim, name):
if not name:
return False
if 'normalized_name' in claim:
return self.normalize_name(name) == claim['normalized_name']
return name == claim['name']
async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None):
# TODO: this thing is huge, refactor
@@ -312,8 +321,11 @@ class LBRYElectrumX(ElectrumX):
# TODO: this is also done on the else, refactor
if parsed_uri.claim_id:
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if certificate_info and certificate_info['name'] == parsed_uri.name:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
else:
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name):
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
elif parsed_uri.claim_sequence:
certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
@@ -327,26 +339,28 @@ class LBRYElectrumX(ElectrumX):
if certificate and 'claim_id' not in certificate['result']:
return result
if certificate and not parsed_uri.path:
if certificate:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id)
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel if claim}
elif certificate:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name)
claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(channel_id)
if not parsed_uri.path:
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel}
else:
# making an assumption that there aren't case conflicts on an existing channel
norm_path = self.normalize_name(parsed_uri.path)
result['unverified_claims_for_name'] = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel
if self.normalize_name(claim['name']) == norm_path}
claims_in_channel = {claim['claim_id']: (claim['name'], claim['height'])
for claim in claims}
result['unverified_claims_for_name'] = claims_in_channel
else:
claim = None
if parsed_uri.claim_id:
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if claim_info and claim_info['name'] == parsed_uri.name:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
else:
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
if claim_info and self.claim_matches_name(claim_info, parsed_uri.name):
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
@@ -369,6 +383,7 @@ class LBRYElectrumX(ElectrumX):
'result': certificate}
result['certificate'] = certificate
result['claim'] = claim
return result
async def claimtrie_getvalueforuris(self, block_hash, *uris):
+1 -1
View File
@@ -139,7 +139,7 @@ def get_schema_regex():
protocol = _named("protocol", re.escape(PROTOCOL))
# Define basic building blocks
valid_name_char = "[a-zA-Z0-9\-]" # these characters are the only valid name characters
valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/
name_content = valid_name_char + '+'
name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
+68 -33
View File
@@ -15,13 +15,13 @@ log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(file_name)
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return os.path.join(download_directory, file_name)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
@@ -29,7 +29,9 @@ async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_dir
class StreamAssembler:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
output_file_name: typing.Optional[str] = None):
self.output_file_name = output_file_name
self.loop = loop
self.blob_manager = blob_manager
self.sd_hash = sd_hash
@@ -43,11 +45,11 @@ class StreamAssembler:
self.written_bytes: int = 0
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
if not blob or not self.stream_handle or self.stream_handle.closed:
return False
def _decrypt_and_write():
if self.stream_handle.closed:
return False
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
self.stream_handle.seek(offset)
_decrypted = blob.decrypt(
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
@@ -55,47 +57,80 @@ class StreamAssembler:
self.stream_handle.write(_decrypted)
self.stream_handle.flush()
self.written_bytes += len(_decrypted)
return True
log.debug("decrypted %s", blob.blob_hash[:8])
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
if decrypted:
log.info("decrypted %s", blob.blob_hash[:8])
return
await self.loop.run_in_executor(None, _decrypt_and_write)
return True
async def setup(self):
pass
async def after_got_descriptor(self):
pass
async def after_finished(self):
pass
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
if not os.path.isdir(output_dir):
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
await self.setup()
self.sd_blob = await self.get_blob(self.sd_hash)
await self.blob_manager.blob_completed(self.sd_blob)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
self.sd_blob)
await self.after_got_descriptor()
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
self.output_path = os.path.join(output_dir, self.output_file_name)
if not self.got_descriptor.is_set():
self.got_descriptor.set()
self.output_path = await get_next_available_file_name(self.loop, output_dir,
output_file_name or self.descriptor.suggested_file_name)
self.stream_handle = open(self.output_path, 'wb')
await self.blob_manager.storage.store_stream(
self.sd_blob, self.descriptor
)
await self.blob_manager.blob_completed(self.sd_blob)
written_blobs = None
save_tasks = []
try:
for blob_info in self.descriptor.blobs[:-1]:
while True:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
break
except ValueError as err:
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
self.descriptor.sd_hash, str(err))
continue
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
self.stream_finished_event.set()
with open(self.output_path, 'wb') as stream_handle:
self.stream_handle = stream_handle
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
if blob_info.blob_num != i:
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
return
while self.stream_handle and not self.stream_handle.closed:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
if blob and blob.length != blob_info.length:
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
await self.blob_manager.delete_blobs([blob_info.blob_hash])
continue
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
written_blobs = i
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
break
except FileNotFoundError:
log.debug("stream assembler stopped")
return
except (ValueError, IOError, OSError):
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
self.descriptor.sd_hash)
continue
finally:
self.stream_handle.close()
if written_blobs == len(self.descriptor.blobs) - 2:
log.debug("finished decrypting and assembling stream")
if save_tasks:
await asyncio.wait(save_tasks)
await self.after_finished()
self.stream_finished_event.set()
else:
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
len(self.descriptor.blobs) - 2)
if self.output_path and os.path.isfile(self.output_path):
log.debug("erasing incomplete file assembly: %s", self.output_path)
os.unlink(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
f = asyncio.Future(loop=self.loop)
f.set_result(self.blob_manager.get_blob(blob_hash, length))
return await f
return self.blob_manager.get_blob(blob_hash, length)
+67 -9
View File
@@ -4,6 +4,7 @@ import binascii
import logging
import typing
import asyncio
from collections import OrderedDict
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.blob.blob_info import BlobInfo
@@ -58,6 +59,10 @@ class StreamDescriptor:
self.stream_hash = stream_hash or self.get_stream_hash()
self.sd_hash = sd_hash
@property
def length(self):
return len(self.as_json())
def get_stream_hash(self) -> str:
return self.calculate_stream_hash(
binascii.hexlify(self.stream_name.encode()), self.key.encode(),
@@ -78,15 +83,46 @@ class StreamDescriptor:
[blob_info.as_dict() for blob_info in self.blobs]), sort_keys=True
).encode()
async def make_sd_blob(self):
sd_hash = self.calculate_sd_hash()
sd_data = self.as_json()
sd_blob = BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
def old_sort_json(self) -> bytes:
blobs = []
for b in self.blobs:
blobs.append(OrderedDict(
[('length', b.length), ('blob_num', b.blob_num), ('iv', b.iv)] if not b.blob_hash else
[('length', b.length), ('blob_num', b.blob_num), ('blob_hash', b.blob_hash), ('iv', b.iv)]
))
if not b.blob_hash:
break
return json.dumps(
OrderedDict([
('stream_name', binascii.hexlify(self.stream_name.encode()).decode()),
('blobs', blobs),
('stream_type', 'lbryfile'),
('key', self.key),
('suggested_file_name', binascii.hexlify(self.suggested_file_name.encode()).decode()),
('stream_hash', self.stream_hash),
])
).encode()
def calculate_old_sort_sd_hash(self):
h = get_lbry_hash_obj()
h.update(self.old_sort_json())
return h.hexdigest()
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
old_sort: typing.Optional[bool] = False):
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
if not old_sort:
sd_data = self.as_json()
else:
sd_data = self.old_sort_json()
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
if blob_file_obj:
blob_file_obj.set_length(len(sd_data))
if not sd_blob.get_is_verified():
writer = sd_blob.open_for_writing()
writer.write(sd_data)
await sd_blob.verified.wait()
await sd_blob.close()
sd_blob.close()
return sd_blob
@classmethod
@@ -95,13 +131,19 @@ class StreamDescriptor:
assert os.path.isfile(blob.file_path)
with open(blob.file_path, 'rb') as f:
json_bytes = f.read()
decoded = json.loads(json_bytes.decode())
try:
decoded = json.loads(json_bytes.decode())
except json.JSONDecodeError:
blob.delete()
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
if decoded['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in decoded['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
descriptor = cls(
loop, blob_dir,
binascii.unhexlify(decoded['stream_name']).decode(),
@@ -154,8 +196,8 @@ class StreamDescriptor:
@classmethod
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None
) -> 'StreamDescriptor':
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False) -> 'StreamDescriptor':
blobs: typing.List[BlobInfo] = []
@@ -174,7 +216,7 @@ class StreamDescriptor:
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
blobs
)
sd_blob = await descriptor.make_sd_blob()
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
descriptor.sd_hash = sd_blob.blob_hash
return descriptor
@@ -184,3 +226,19 @@ class StreamDescriptor:
def upper_bound_decrypted_length(self) -> int:
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
@classmethod
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
suggested_file_name: str, key: str,
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
blobs, stream_hash, sd_blob.blob_hash)
if descriptor.calculate_sd_hash() == sd_blob.blob_hash: # first check for a normal valid sd
old_sort = False
elif descriptor.calculate_old_sort_sd_hash() == sd_blob.blob_hash: # check if old field sorting works
old_sort = True
else:
return
await descriptor.make_sd_blob(sd_blob, old_sort)
return descriptor
+78 -205
View File
@@ -2,12 +2,14 @@ import os
import asyncio
import typing
import logging
from lbrynet.utils import drain_tasks, cancel_task
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
@@ -19,217 +21,88 @@ def drain_into(a: list, b: list):
b.append(a.pop())
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
output_file_name: typing.Optional[str] = None,
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
max_connections_per_stream: typing.Optional[int] = 8):
super().__init__(loop, blob_manager, sd_hash)
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.current_blob: 'BlobFile' = None
self.download_task: asyncio.Task = None
self.accumulate_connections_task: asyncio.Task = None
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.output_dir = output_dir or os.getcwd()
class StreamDownloader(StreamAssembler):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
super().__init__(loop, blob_manager, sd_hash, output_file_name)
self.config = config
self.output_dir = output_dir or self.config.download_dir
self.output_file_name = output_file_name
self._lock = asyncio.Lock(loop=self.loop)
self.max_connections_per_stream = max_connections_per_stream
self.fixed_peers = fixed_peers or []
self.blob_downloader: typing.Optional[BlobDownloader] = None
self.search_queue = asyncio.Queue(loop=loop)
self.peer_queue = asyncio.Queue(loop=loop)
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.descriptor: typing.Optional[StreamDescriptor]
self.node: typing.Optional['Node'] = None
self.assemble_task: typing.Optional[asyncio.Task] = None
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
self.fixed_peers_delay: typing.Optional[float] = None
self.added_fixed_peers = False
async def _update_current_blob(self, blob: 'BlobFile'):
async with self._lock:
drain_tasks(self.running_download_requests)
self.current_blob = blob
if not blob.get_is_verified():
self._update_requests()
async def setup(self): # start the peer accumulator and initialize the downloader
if self.blob_downloader:
raise Exception("downloader is already set up")
if self.node:
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.search_queue.put_nowait(self.sd_hash)
async def _request_blob(self, peer: 'KademliaPeer'):
if self.current_blob.get_is_verified():
log.info("already verified")
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
peer.address, peer.tcp_port, self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self._lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
async def after_got_descriptor(self):
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to search")
def _update_requests(self):
self.new_peer_event.clear()
if self.current_blob.blob_hash not in self.requested_from:
self.requested_from[self.current_blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.current_blob.blob_hash][peer] = task
self.running_download_requests.append(task)
async def after_finished(self):
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
self.blob_downloader.close()
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
async with self._lock:
if len(self.running_download_requests) < self.max_connections_per_stream:
# update the running download requests
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
async with self._lock:
if self.current_blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.current_blob
else:
for task in download_tasks:
if task and not task.done():
self.running_download_requests.append(task)
return
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.assemble_task:
self.assemble_task.cancel()
self.assemble_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader = None
if self.stream_handle:
if not self.stream_handle.closed:
self.stream_handle.close()
self.stream_handle = None
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
self.output_path and os.path.isfile(self.output_path):
os.remove(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
blob = self.blob_manager.get_blob(blob_hash, length)
await self._update_current_blob(blob)
if blob.get_is_verified():
return blob
return await self.blob_downloader.download_blob(blob_hash, length)
# the blob must be downloaded
try:
while not self.current_blob.get_is_verified():
if not self.active_connections: # wait for a new connection
await self.new_peer_event.wait()
continue
blob = await self.wait_for_download_or_new_peer()
if blob:
drain_tasks(self.running_download_requests)
return blob
return blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
def add_fixed_peers(self):
async def _add_fixed_peers():
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
async def _accumulate_connections(self, node: 'Node'):
blob_queue = asyncio.Queue(loop=self.loop)
blob_queue.put_nowait(self.sd_hash)
task = asyncio.create_task(self.got_descriptor.wait())
added_peers = asyncio.Event(loop=self.loop)
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
if self.fixed_peers:
def check_added_peers():
if not added_peers.is_set():
self._add_peer_protocols(self.fixed_peers)
log.info("no dht peers for download yet, adding fixed peer")
added_peers.set()
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
def got_descriptor(f):
try:
f.result()
except asyncio.CancelledError:
return
log.info("add head blob hash to peer search")
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
task.add_done_callback(got_descriptor)
try:
async with node.stream_peer_search_junction(blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
if not added_peers.is_set():
added_peers.set()
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
if not self.config.reflector_servers:
return
except asyncio.CancelledError:
pass
finally:
if task and not task.done():
task.cancel()
log.info("cancelled head blob task")
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
add_fixed_peers_timer.cancel()
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()):
self.fixed_peers_delay = 0.0
else:
self.fixed_peers_delay = self.config.fixed_peer_delay
self.loop.create_task(_add_fixed_peers())
async def stop(self):
cancel_task(self.accumulate_connections_task)
self.accumulate_connections_task = None
drain_tasks(self.running_download_requests)
while self.requested_from:
_, peer_task_dict = self.requested_from.popitem()
while peer_task_dict:
peer, task = peer_task_dict.popitem()
try:
cancel_task(task)
except asyncio.CancelledError:
pass
while self.active_connections:
_, client = self.active_connections.popitem()
if client:
await client.close()
log.info("stopped downloader")
async def _download(self):
try:
log.info("download and decrypt stream")
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
log.info(
"downloaded stream %s -> %s", self.sd_hash, self.output_path
)
await self.blob_manager.storage.change_file_status(
self.descriptor.stream_hash, 'finished'
)
except asyncio.CancelledError:
pass
finally:
await self.stop()
def download(self, node: 'Node'):
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
self.download_task = self.loop.create_task(self._download())
def download(self, node: typing.Optional['Node'] = None):
self.node = node
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
self.add_fixed_peers()
+73 -38
View File
@@ -2,13 +2,17 @@ import os
import asyncio
import typing
import logging
import binascii
from lbrynet.utils import generate_id
from lbrynet.extras.daemon.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
from lbrynet.extras.daemon.storage import StoredStreamClaim
if typing.TYPE_CHECKING:
from lbrynet.extras.daemon.storage import StoredStreamClaim
from lbrynet.schema.claim import ClaimDict
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.node import Node
log = logging.getLogger(__name__)
@@ -18,20 +22,29 @@ class ManagedStream:
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None):
self.loop = loop
self.blob_manager = blob_manager
self.rowid = rowid
self.download_directory = download_directory
self.file_name = file_name
self._file_name = file_name
self.descriptor = descriptor
self.downloader = downloader
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self._store_after_finished: asyncio.Task = None
self.fully_reflected = asyncio.Event(loop=self.loop)
self.tx = None
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
@property
def file_name(self) -> typing.Optional[str]:
return self.downloader.output_file_name if self.downloader else self._file_name
@property
def status(self) -> str:
@@ -98,13 +111,23 @@ class ManagedStream:
def sd_hash(self):
return self.descriptor.sd_hash
def as_dict(self) -> typing.Dict:
full_path = os.path.join(self.download_directory, self.file_name)
if not os.path.exists(full_path):
full_path = None
mime_type = guess_media_type(os.path.basename(self.file_name))
@property
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
if self.downloader:
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
def as_dict(self) -> typing.Dict:
full_path = self.full_path if self.output_file_exists else None
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
if self.downloader and self.downloader.written_bytes:
written_bytes = self.downloader.written_bytes
elif full_path:
written_bytes = os.stat(full_path).st_size
@@ -115,6 +138,7 @@ class ManagedStream:
'file_name': self.file_name,
'download_directory': self.download_directory,
'points_paid': 0.0,
'tx': self.tx,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
@@ -128,6 +152,7 @@ class ManagedStream:
'written_bytes': written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
'status': self.status,
'claim_id': self.claim_id,
'txid': self.txid,
@@ -147,50 +172,60 @@ class ManagedStream:
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
)
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
await blob_manager.blob_completed(sd_blob)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
await blob_manager.blob_completed(sd_blob)
for blob in descriptor.blobs[:-1]:
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
status=cls.STATUS_FINISHED)
async def stop_download(self):
def start_download(self, node: typing.Optional['Node']):
self.downloader.download(node)
self.update_status(self.STATUS_RUNNING)
def stop_download(self):
if self.downloader:
await self.downloader.stop()
if not self.finished:
self.update_status(self.STATUS_STOPPED)
self.downloader.stop()
self.downloader = None
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
await self.loop.create_connection(lambda: protocol, host, port)
except ConnectionRefusedError:
return sent
try:
await protocol.send_handshake()
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
try:
sent_sd, needed = await protocol.send_descriptor()
if sent_sd:
sent.append(self.sd_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
for blob_hash in needed:
try:
if not sent_sd and not needed:
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return []
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
if protocol.transport:
protocol.transport.close()
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
return sent
finally:
if protocol.transport:
protocol.transport.close()
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return sent
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
self.stream_claim_info = StoredStreamClaim(
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
+8 -2
View File
@@ -15,6 +15,7 @@ log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
self.descriptor = descriptor
@@ -26,12 +27,14 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
log.info("Connected to reflector")
log.debug("Connected to reflector")
self.connected.set()
def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None
self.connected.clear()
if self.reflected_blobs:
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
def data_received(self, data):
try:
@@ -45,7 +48,7 @@ class StreamReflectorClient(asyncio.Protocol):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
self.pending_request = self.loop.create_task(self.response_queue.get())
return await self.pending_request
finally:
self.pending_request = None
@@ -80,8 +83,11 @@ class StreamReflectorClient(asyncio.Protocol):
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
needed.append(blob.blob_hash)
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
self.reflected_blobs.append(sd_blob.blob_hash)
else:
log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8])
if needed:
log.info("Reflector needs %i blobs for %s", len(needed), sd_blob.blob_hash[:8])
return sent_sd, needed
async def send_blob(self, blob_hash: str):
+12 -9
View File
@@ -37,7 +37,8 @@ class ReflectorServerProtocol(asyncio.Protocol):
try:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob: %s", err)
log.error("error receiving blob: %s", err)
self.transport.close()
return
try:
request = json.loads(data.decode())
@@ -67,28 +68,30 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_sd_blob": True})
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.writer.close_handle()
self.writer = None
except (asyncio.TimeoutError, asyncio.CancelledError):
self.send_response({"received_sd_blob": True})
except asyncio.TimeoutError:
self.send_response({"received_sd_blob": False})
self.transport.close()
finally:
self.incoming.clear()
self.writer.close_handle()
self.writer = None
self.transport.close()
return
else:
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
if self.writer:
self.writer.close_handle()
self.writer = None
self.send_response({"send_sd_blob": False, 'needed': [
blob.blob_hash for blob in self.descriptor.blobs[:-1]
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
]})
return
return
elif self.descriptor:
if 'blob_hash' not in request:
@@ -105,7 +108,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
try:
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_blob": True})
except (asyncio.TimeoutError, asyncio.CancelledError):
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
self.incoming.clear()
self.writer.close_handle()
+400 -169
View File
@@ -4,21 +4,29 @@ import typing
import binascii
import logging
import random
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
DownloadDataTimeout, DownloadSDTimeout
from lbrynet.utils import generate_id
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import ClaimDict
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.decode import smart_decode
from lbrynet.extras.daemon.storage import lbc_to_dewies
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
filter_fields = [
'rowid',
'status',
'file_name',
'sd_hash',
@@ -31,7 +39,9 @@ filter_fields = [
'nout',
'channel_claim_id',
'channel_name',
'full_status'
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream'
]
comparison_operators = {
@@ -45,205 +55,231 @@ comparison_operators = {
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.wallet = wallet
self.storage = storage
self.node = node
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.analytics_manager = analytics_manager
self.streams: typing.Set[ManagedStream] = set()
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
self.resume_downloading_task: asyncio.Task = None
self.re_reflect_task: asyncio.Task = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.fixed_peers = fixed_peers
self.reflector_servers = reflector_servers
self.running_reflector_uploads: typing.List[asyncio.Task] = []
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, smart_decode(claim_info['value']))
async def start_stream(self, stream: ManagedStream) -> bool:
"""
Resume or rebuild a partial or completed stream
"""
if not stream.running and not stream.output_file_exists:
if stream.downloader:
stream.downloader.stop()
stream.downloader = None
# the directory is gone, can happen when the folder that contains a published file is deleted
# reset the download directory to the default and update the file name
if not os.path.isdir(stream.download_directory):
stream.download_directory = self.config.download_dir
stream.downloader = self.make_downloader(
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
)
if stream.status != ManagedStream.STATUS_FINISHED:
await self.storage.change_file_status(stream.stream_hash, 'running')
stream.update_status('running')
stream.start_download(self.node)
try:
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
self.config.download_timeout)
except asyncio.TimeoutError:
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
return False
file_name = os.path.basename(stream.downloader.output_path)
output_dir = os.path.dirname(stream.downloader.output_path)
await self.storage.change_file_download_dir_and_file_name(
stream.stream_hash, output_dir, file_name
)
stream._file_name = file_name
stream.download_directory = output_dir
self.wait_for_stream_finished(stream)
return True
return True
async def stop_stream(self, stream: ManagedStream):
stream.stop_download()
if not stream.finished and stream.output_file_exists:
try:
os.remove(stream.full_path)
except OSError as err:
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
str(err))
if stream.running:
stream.update_status(ManagedStream.STATUS_STOPPED)
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
return StreamDownloader(
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
)
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = []
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
sd_blob = self.blob_manager.get_blob(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash)
descriptor = await StreamDescriptor.recover(
self.blob_manager.blob_dir, sd_blob, stream_hash, stream_name, suggested_file_name, key, blobs
)
if not descriptor:
return
to_restore.append((descriptor, sd_blob))
await asyncio.gather(*[
recover_stream(
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
) for file_info in file_infos
])
if to_restore:
await self.storage.recover_streams(to_restore, self.config.download_dir)
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
claim: typing.Optional['StoredStreamClaim']):
sd_blob = self.blob_manager.get_blob(sd_hash)
if not sd_blob.get_is_verified():
return
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
except InvalidStreamDescriptorError as err:
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
return
if status == ManagedStream.STATUS_RUNNING:
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
else:
downloader = None
stream = ManagedStream(
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
)
self.streams.add(stream)
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
async def load_streams_from_database(self):
infos = await self.storage.get_all_lbry_files()
for file_info in infos:
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
if sd_blob.get_is_verified():
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
downloader = StreamDownloader(
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
)
stream = ManagedStream(
self.loop, self.blob_manager, descriptor,
binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(),
downloader, file_info['status'], file_info['claim']
)
self.streams.add(stream)
to_recover = []
for file_info in await self.storage.get_all_lbry_files():
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_recover.append(file_info)
if to_recover:
log.info("Attempting to recover %i streams", len(to_recover))
await self.recover_streams(to_recover)
to_start = []
for file_info in await self.storage.get_all_lbry_files():
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_start.append(file_info)
log.info("Initializing %i files", len(to_start))
await asyncio.gather(*[
self.add_stream(
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
file_info['claim']
) for file_info in to_start
])
log.info("Started stream manager with %i files", len(self.streams))
async def resume(self):
if not self.node:
log.warning("no DHT node given, cannot resume downloads")
return
await self.node.joined.wait()
resumed = 0
for stream in self.streams:
if stream.status == ManagedStream.STATUS_RUNNING:
resumed += 1
stream.downloader.download(self.node)
self.wait_for_stream_finished(stream)
if resumed:
log.info("resuming %i downloads", resumed)
if self.node:
await self.node.joined.wait()
else:
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
t = [
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
]
if t:
log.info("resuming %i downloads", len(t))
async def reflect_streams(self):
streams = list(self.streams)
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= 10:
await asyncio.gather(*batch)
while True:
if self.config.reflect_streams and self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
batch = []
if batch:
await asyncio.gather(*batch)
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch)
batch = []
if batch:
await asyncio.gather(*batch)
await asyncio.sleep(300, loop=self.loop)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
async def stop(self):
def stop(self):
if self.resume_downloading_task and not self.resume_downloading_task.done():
self.resume_downloading_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
stream = self.streams.pop()
await stream.stop_download()
stream.stop_download()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads:
self.running_reflector_uploads.pop().cancel()
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
if self.reflector_servers:
host, port = random.choice(self.reflector_servers)
self.loop.create_task(stream.upload_to_reflector(host, port))
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers:
host, port = random.choice(self.config.reflector_servers)
task = self.loop.create_task(stream.upload_to_reflector(host, port))
self.running_reflector_uploads.append(task)
task.add_done_callback(
lambda _: None
if task not in self.running_reflector_uploads else self.running_reflector_uploads.remove(task)
)
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
await stream.stop_download()
self.streams.remove(stream)
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor)
if delete_file and stream.output_file_exists:
os.remove(stream.full_path)
blob_hashes = [stream.sd_hash]
for blob_info in stream.descriptor.blobs[:-1]:
blob_hashes.append(blob_info.blob_hash)
for blob_hash in blob_hashes:
blob = self.blob_manager.get_blob(blob_hash)
if blob.get_is_verified():
await blob.delete()
if delete_file:
path = os.path.join(stream.download_directory, stream.file_name)
if os.path.isfile(path):
os.remove(path)
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
try:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
except asyncio.CancelledError:
pass
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
sd_blob_timeout: typing.Optional[float] = 60
) -> typing.Optional[ManagedStream]:
claim = ClaimDict.load_dict(claim_info['value'])
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
try:
downloader.download(node)
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
except (asyncio.TimeoutError, asyncio.CancelledError):
log.info("stream timeout")
await downloader.stop()
log.info("stopped stream")
return
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
await self.blob_manager.storage.save_downloaded_file(
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
data_rate
)
await self.blob_manager.storage.save_content_claim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
)
stored_claim = StoredStreamClaim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
claim_info.get('channel_name')
)
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
stored_claim)
self.streams.add(stream)
try:
await stream.downloader.wrote_bytes_event.wait()
self.wait_for_stream_finished(stream)
return stream
except asyncio.CancelledError:
await downloader.stop()
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None,
sd_blob_timeout: typing.Optional[float] = 60,
fee_amount: typing.Optional[float] = 0.0,
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
claim = ClaimDict.load_dict(claim_info['value'])
if fee_address and fee_amount:
if fee_amount > await self.wallet.default_account.get_balance():
raise Exception("not enough funds")
sd_hash = claim.source_hash.decode()
if sd_hash in self.starting_streams:
return await self.starting_streams[sd_hash]
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
if already_started:
return already_started[0]
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
stream_task = self.loop.create_task(
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
)
try:
await asyncio.wait_for(stream_task, sd_blob_timeout)
stream = await stream_task
self.starting_streams[sd_hash].set_result(stream)
if fee_address and fee_amount:
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
return stream
except (asyncio.TimeoutError, asyncio.CancelledError):
return
finally:
if sd_hash in self.starting_streams:
del self.starting_streams[sd_hash]
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
if streams:
return streams[0]
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
comparison: typing.Optional[str] = None,
@@ -260,6 +296,8 @@ class StreamManager:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in comparison_operators:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by.keys():
if search not in filter_fields:
raise ValueError(f"'{search}' is not a valid search operation")
@@ -268,8 +306,6 @@ class StreamManager:
streams = []
for stream in self.streams:
for search, val in search_by.items():
if search == 'full_status':
continue
if comparison_operators[comparison](getattr(stream, search), val):
streams.append(stream)
break
@@ -280,3 +316,198 @@ class StreamManager:
if reverse:
streams.reverse()
return streams
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
stream.download_id, stream.claim_name, stream.sd_hash
))
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _store_stream(self, downloader: StreamDownloader) -> int:
file_name = os.path.basename(downloader.output_path)
download_directory = os.path.dirname(downloader.output_path)
if not await self.storage.stream_exists(downloader.sd_hash):
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.storage.file_exists(downloader.sd_hash):
return await self.storage.save_downloaded_file(
downloader.descriptor.stream_hash, file_name, download_directory,
0.0
)
else:
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: ClaimDict) -> typing.Tuple[
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
await self.start_stream(existing[0])
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.source_hash.decode())
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
f"download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
await self.start_stream(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
return None, existing_for_claim_id[0]
return None, None
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
download_id: str, outpoint: str, claim: ClaimDict, resolved: typing.Dict,
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
downloader.download(self.node)
await downloader.got_descriptor.wait()
got_descriptor_time.set_result(self.loop.time() - start_time)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, outpoint
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
stream.set_claim(resolved, claim)
await stream.downloader.wrote_bytes_event.wait()
self.streams.add(stream)
return stream
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", self.config.max_key_fee['amount']
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.source_fee.address.decode()
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
file_name)
), timeout
)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
claim.source_hash.decode(), time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
time_to_first_bytes, None if not error else error.__class__.__name__
)
)
if error:
raise error
return stream
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> ManagedStream:
timeout = timeout or self.config.download_timeout
if uri in self.starting_streams:
return await self.starting_streams[uri]
fut = asyncio.Future(loop=self.loop)
self.starting_streams[uri] = fut
try:
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
fut.set_result(stream)
except Exception as err:
fut.set_exception(err)
try:
return await fut
finally:
del self.starting_streams[uri]
+54 -7
View File
@@ -7,8 +7,13 @@ import string
import json
import typing
import asyncio
import ssl
import logging
import ipaddress
import pkg_resources
import contextlib
import certifi
import aiohttp
from lbrynet.schema.claim import ClaimDict
from lbrynet.cryptoutils import get_lbry_hash_obj
@@ -68,13 +73,12 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5):
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
server = socket.gethostbyname(server)
conn = socket.create_connection((server, port), timeout)
conn.close()
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
@@ -83,19 +87,23 @@ def check_connection(server="lbry.io", port=80, timeout=5):
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout)
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except Exception as ex:
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
except Exception as ex:
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
server, port)
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
@@ -136,3 +144,42 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
try:
if ipaddress.ip_address(url):
return url
except ValueError:
pass
loop = asyncio.get_running_loop()
return (await loop.getaddrinfo(
url, port,
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
))[0][4][0]
def get_ssl_context() -> ssl.SSLContext:
return ssl.create_default_context(
purpose=ssl.Purpose.CLIENT_AUTH, capath=certifi.where()
)
@contextlib.asynccontextmanager
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
async with aiohttp.ClientSession() as session:
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
yield response
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
try:
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
return
+53
View File
@@ -0,0 +1,53 @@
import sys
import os
import asyncio
import socket
import ipaddress
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
import logging
log = logging.getLogger("lbrynet")
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
async def main(blob_hash: str, url: str):
conf = Config()
loop = asyncio.get_running_loop()
host_url, port = url.split(":")
try:
host = None
if ipaddress.ip_address(host_url):
host = host_url
except ValueError:
host = None
if not host:
host_info = await loop.getaddrinfo(
host_url, 'https',
proto=socket.IPPROTO_TCP,
)
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
await storage.open()
await blob_manager.setup()
blob = blob_manager.get_blob(blob_hash)
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
conf.blob_download_timeout)
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
f"keep connection: {keep}")
if blob.get_is_verified():
await blob_manager.delete_blobs([blob.blob_hash])
print(f"deleted {blob_hash}")
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
url = 'reflector.lbry.io:5567'
if len(sys.argv) > 2:
url = sys.argv[2]
asyncio.run(main(sys.argv[1], url))
+49 -19
View File
@@ -19,7 +19,8 @@ except ImportError:
AREA_RENAME = {
'api': 'API'
'api': 'API',
'dht': 'DHT'
}
@@ -29,16 +30,20 @@ def get_github():
with open(config_path, 'r') as config_file:
config = json.load(config_file)
return github3.login(token=config['token'])
print('GitHub Credentials')
username = input('username: ')
password = getpass('password: ')
gh = github3.authorize(
username, password, ['repo'], 'lbry release tool',
two_factor_callback=lambda: input('Enter 2FA: ')
)
with open(config_path, 'w') as config_file:
json.dump({'token': gh.token}, config_file)
return github3.login(token=gh.token)
token = os.environ.get("GH_TOKEN")
if not token:
print('GitHub Credentials')
username = input('username: ')
password = getpass('password: ')
gh = github3.authorize(
username, password, ['repo'], 'lbry release tool',
two_factor_callback=lambda: input('Enter 2FA: ')
)
with open(config_path, 'w') as config_file:
json.dump({'token': gh.token}, config_file)
token = gh.token
return github3.login(token=token)
def get_labels(pr, prefix):
@@ -54,6 +59,7 @@ def get_label(pr, prefix):
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
RELEASE_TEXT = "release-text:"
def get_backwards_incompatible(desc: str):
@@ -62,6 +68,12 @@ def get_backwards_incompatible(desc: str):
yield line[len(BACKWARDS_INCOMPATIBLE):]
def get_release_text(desc: str):
for line in desc.splitlines():
if line.startswith(RELEASE_TEXT):
yield line[len(RELEASE_TEXT):]
def get_previous_final(repo, current_release):
assert current_release.rc is not None, "Need an rc to find the previous final release."
previous = None
@@ -139,15 +151,23 @@ def release(args):
previous_release = repo.release_from_tag(current_version.tag)
incompats = []
release_texts = []
unlabeled = []
areas = {}
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry"):
for area_name in get_labels(pr, 'area'):
area = areas.setdefault(area_name, [])
type_label = get_label(pr, "type")
for incompat in get_backwards_incompatible(pr.body):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
if not (args.action == '*-rc' and type_label == 'fixup'):
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
area_labels = list(get_labels(pr, 'area'))
type_label = get_label(pr, 'type')
if area_labels and type_label:
for area_name in area_labels:
for incompat in get_backwards_incompatible(pr.body):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body):
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
if not (args.action == '*-rc' and type_label == 'fixup'):
area = areas.setdefault(area_name, [])
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
else:
unlabeled.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
area_names = list(areas.keys())
area_names.sort()
@@ -156,6 +176,11 @@ def release(args):
w = lambda s: body.write(s+'\n')
w(f'## [{new_version}] - {date.today().isoformat()}')
if release_texts:
w('')
for release_text in release_texts:
w(release_text)
w('')
if incompats:
w('')
w(f'### Backwards Incompatible Changes')
@@ -163,7 +188,7 @@ def release(args):
w(incompat)
for area in area_names:
prs = areas[area]
area = AREA_RENAME.get(area, area.capitalize())
area = AREA_RENAME.get(area.lower(), area.capitalize())
w('')
w(f'### {area}')
for pr in prs:
@@ -171,6 +196,11 @@ def release(args):
print(body.getvalue())
if unlabeled:
print('The following PRs were skipped and not included in changelog:')
for skipped in unlabeled:
print(skipped)
if not args.dry_run:
commit = version_file.update(
+33
View File
@@ -0,0 +1,33 @@
import os
import binascii
import sqlite3
from lbrynet.conf import Config
def main():
conf = Config()
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
cur = db.cursor()
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
update = {}
for stream_hash, file_name, download_directory in files:
try:
binascii.unhexlify(file_name)
except binascii.Error:
try:
binascii.unhexlify(download_directory)
except binascii.Error:
update[stream_hash] = (
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
)
if update:
print(f"repair {len(update)} streams")
for stream_hash, (file_name, download_directory) in update.items():
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
(file_name, download_directory, stream_hash))
db.commit()
db.close()
if __name__ == "__main__":
main()
+4 -1
View File
@@ -18,12 +18,15 @@ def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
print('Build: rc')
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
print('Build: release')
return 'release'
print('Build: qa')
return 'qa'
except subprocess.CalledProcessError:
# if the build doesn't have a tag
print("Couldn't determine build type, defaulting to qa.")
return 'qa'
+54 -39
View File
@@ -7,6 +7,7 @@ import time
from aiohttp import ClientConnectorError
from lbrynet import __version__
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.conf import Config
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.client import daemon_rpc
@@ -45,11 +46,11 @@ async def report_to_slack(output, webhook):
pass
def confidence(times, z):
def confidence(times, z, plus_err=True):
mean = sum(times) / len(times)
standard_dev = (sum(((t - sum(times) / len(times)) ** 2.0 for t in times)) / len(times)) ** 0.5
err = (z * standard_dev) / (len(times) ** 0.5)
return f"{round(mean, 3) + round(err, 3)}s"
return f"{round((mean + err) if plus_err else (mean - err), 3)}"
def variance(times):
@@ -57,31 +58,29 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(api, uri):
async def wait_for_done(conf, uri):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
while True:
files = await api.file_list(claim_name=name)
files = await daemon_rpc(conf, "file_list", claim_name=name)
file = files[0]
if file['status'] in ['finished', 'stopped']:
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
return True, file['blobs_completed'], file['blobs_in_stream']
if last_complete < int(file['blobs_completed']):
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
hang_count = 0
last_complete = int(file['blobs_completed'])
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > 30:
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
if hang_count > 10:
return False, file['blobs_completed'], file['blobs_in_stream']
async def main(start_daemon=True, uris=None):
async def main(uris=None, allow_fees=False):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
daemon = None
try:
await daemon_rpc(conf, 'status')
except (ClientConnectorError, ConnectionError):
@@ -92,15 +91,20 @@ async def main(start_daemon=True, uris=None):
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', uri=name)
resolved = await daemon_rpc(conf, 'resolve', name)
if 'error' not in resolved.get(name, {}):
resolvable.append(name)
print(f"{len(resolvable)}/{len(uris)} are resolvable")
if ("fee" not in resolved[name]['claim']['value']['stream']['metadata']) or allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
first_byte_times = []
downloaded_times = []
failures = []
download_speeds = []
download_successes = []
failed_to_start = []
download_failures = []
for uri in resolvable:
@@ -113,34 +117,44 @@ async def main(start_daemon=True, uris=None):
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
# if downloaded:
# downloaded_times.append((time.time()-start) / downloaded)
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
# else:
# print(f"failed to downlload {uri}, got {msg}")
# download_failures.append(uri)
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except:
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
failures.append(uri)
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
failed_to_start.append(uri)
return
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Tried to start downloading {len(resolvable)} streams from the front page\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}\n" \
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Started {len(first_byte_times)}/{len(resolvable)} streams"
if failures:
nt = '\n\t'
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
if download_failures:
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
print(result)
if daemon:
await daemon.shutdown()
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
# if webhook:
# await report_to_slack(result, webhook)
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
if webhook:
await report_to_slack(result, webhook)
if __name__ == "__main__":
@@ -148,5 +162,6 @@ if __name__ == "__main__":
parser.add_argument("--data_dir")
parser.add_argument("--wallet_dir")
parser.add_argument("--download_directory")
parser.add_argument("--allow_fees", action='store_true')
args = parser.parse_args()
asyncio.run(main())
asyncio.run(main(allow_fees=args.allow_fees))
+1 -2
View File
@@ -5,9 +5,8 @@ rm -rf /tmp/.wine-*
apt-get -qq update
apt-get -qq install -y git
git clone https://github.com/lbryio/torba.git --depth 1
pip install setuptools_scm
git clone https://github.com/lbryio/torba.git --depth 1
cd torba && pip install -e . && cd ..
cd lbry
+12 -10
View File
@@ -23,19 +23,21 @@ setup(
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
},
install_requires=[
'aiohttp',
'aiohttp==3.5.4',
'aioupnp',
'appdirs',
'distro',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
'distro==1.4.0',
'base58==1.0.0',
'jsonrpc',
'cryptography',
'cffi==1.12.1',
'cryptography==2.5',
'protobuf==3.6.1',
'msgpack',
'jsonschema',
'ecdsa',
'msgpack==0.6.1',
'jsonschema==2.6.0',
'ecdsa==0.13',
'torba',
'pyyaml',
'docopt',
'pyyaml==3.13',
'docopt==0.6.2',
],
)
View File
Whitespace-only changes.
@@ -0,0 +1,56 @@
from integration.testcase import CommandTestCase
class AccountManagement(CommandTestCase):
async def test_account_list_set_create_remove_add(self):
# check initial account
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# change account name and gap
account_id = response['lbc_regtest'][0]['id']
self.daemon.jsonrpc_account_set(
account_id=account_id, new_name='test account',
receiving_gap=95, receiving_max_uses=96,
change_gap=97, change_max_uses=98
)
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
self.assertEqual(response['name'], 'test account')
self.assertEqual(
response['address_generator']['receiving'],
{'gap': 95, 'maximum_uses_per_address': 96}
)
self.assertEqual(
response['address_generator']['change'],
{'gap': 97, 'maximum_uses_per_address': 98}
)
# create another account
await self.daemon.jsonrpc_account_create('second account')
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
account_id2 = response['lbc_regtest'][1]['id']
# make new account the default
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
response = await self.daemon.jsonrpc_account_list(show_seed=True)
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
account_seed = response['lbc_regtest'][1]['seed']
# remove account
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# add account
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
# list specific account
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
self.assertEqual(response['name'], 'recreated account')
+224
View File
@@ -0,0 +1,224 @@
import tempfile
from integration.testcase import CommandTestCase
class EpicAdventuresOfChris45(CommandTestCase):
async def test_no_this_is_not_a_test_its_an_adventure(self):
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
# and everything in between; it's also the setting of some record setting
# integration tests.
# Chris45 starts everyday by checking his balance.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '10.0')
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
# enthusiastically. But he is hungry so he goes into the kitchen
# to make himself a spamdwich.
# While making the spamdwich he wonders... has anyone on LBRY
# registered the @spam channel yet? "I should do that!" he
# exclaims and goes back to his computer to do just that!
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# Do we have it locally?
channels = await self.out(self.daemon.jsonrpc_channel_list())
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@spam')
# As the new channel claim travels through the intertubes and makes its
# way into the mempool and then a block and then into the claimtrie,
# Chris doesn't sit idly by: he checks his balance!
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.989893')
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
await self.generate(6)
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
self.assertEqual(result, '8.989893')
# And is the channel resolvable and empty?
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam'))
self.assertIn('lbry://@spam', response)
self.assertIn('certificate', response['lbry://@spam'])
# "What goes well with spam?" ponders Chris...
# "A hovercraft with eels!" he exclaims.
# "That's what goes great with spam!" he further confirms.
# And so, many hours later, Chris is finished writing his epic story
# about eels driving a hovercraft across the wetlands while eating spam
# and decides it's time to publish it to the @spam channel.
with tempfile.NamedTemporaryFile() as file:
file.write(b'blah blah blah...')
file.write(b'[insert long story about eels driving hovercraft]')
file.write(b'yada yada yada!')
file.write(b'the end')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# He quickly checks the unconfirmed balance to make sure everything looks
# correct.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '7.969786')
# Also checks that his new story can be found on the blockchain before
# giving the link to all his friends.
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
self.assertIn('lbry://@spam/hovercraft', response)
self.assertIn('claim', response['lbry://@spam/hovercraft'])
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
await self.generate(5)
# When he comes back he verifies the confirmed balance.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '7.969786')
# As people start reading his story they discover some typos and notify
# Chris who explains in despair "Oh! Noooooos!" but then remembers
# "No big deal! I can update my claim." And so he updates his claim.
with tempfile.NamedTemporaryFile() as file:
file.write(b'blah blah blah...')
file.write(b'[typo fixing sounds being made]')
file.write(b'yada yada yada!')
file.flush()
claim2 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
))
self.assertTrue(claim2['success'])
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
await self.confirm_tx(claim2['tx']['txid'])
# After some soul searching Chris decides that his story needs more
# heart and a better ending. He takes down the story and begins the rewrite.
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
# And now checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
# After abandoning he just waits for his LBCs to be returned to his account
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.9693585')
# Amidst all this Chris receives a call from his friend Ramsey
# who says that it is of utmost urgency that Chris transfer him
# 1 LBC to which Chris readily obliges
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
self.assertIn("txid", result)
await self.confirm_tx(result['txid'])
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
# he received it or not
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
# Chris' balance was correct
self.assertEqual(result, '7.9692345')
# Ramsey too assured him that he had received the 1 LBC and thanks him
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
self.assertEqual(result, '1.0')
# After Chris is done with all the "helping other people" stuff he decides that it's time to
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
with tempfile.NamedTemporaryFile() as file:
file.write(b'Amazingly Original First Line')
file.write(b'Super plot for the grand novel')
file.write(b'Totally un-cliched ending')
file.write(b'**Audience Gasps**')
file.flush()
claim3 = await self.out(self.daemon.jsonrpc_publish(
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
))
self.assertTrue(claim3['success'])
await self.confirm_tx(claim3['tx']['txid'])
await self.generate(5)
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
uri = 'lbry://@spam/fresh-start'
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
))
await self.confirm_tx(tx['txid'])
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
# It obviously did! Because, blockchain baby \O/
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
await self.generate(5)
# Now he also wanted to support the original creator of the Award Winning Novel
# So he quickly decides to send a tip to him
tx = await self.out(
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
await self.confirm_tx(tx['txid'])
# And again checks if it went to the just right place
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
# Which it obviously did. Because....?????
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
await self.generate(5)
# Seeing the ravishing success of his novel Chris adds support to his claim too
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
await self.confirm_tx(tx['txid'])
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
# It did!
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
await self.generate(5)
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
# .......
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
with tempfile.NamedTemporaryFile() as file:
file.write(b'The Whale amd The Bookmark')
file.write(b'I know right? Totally a hit song')
file.write(b'That\'s what goes around for songs these days anyways')
file.flush()
claim4 = await self.out(self.daemon.jsonrpc_publish(
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
))
self.assertTrue(claim4['success'])
await self.confirm_tx(claim4['tx']['txid'])
await self.generate(5)
# He sends the link to Ramsey, all happy and proud
uri = 'lbry://@spam/hit-song'
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
# He them checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve(uri))
self.assertNotIn('claim', response[uri])
+406
View File
@@ -0,0 +1,406 @@
import tempfile
from lbrynet.extras.wallet.transaction import Transaction
from lbrynet.error import InsufficientFundsError
from lbrynet.schema.claim import ClaimDict
from integration.testcase import CommandTestCase
class ClaimCommands(CommandTestCase):
async def craft_claim(self, name, amount_dewies, claim_dict, address):
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
# However, it still happens... So, let's reproduce them.
claim = ClaimDict.load_dict(claim_dict)
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
await self.broadcast(tx)
await self.ledger.wait(tx)
await self.generate(1)
await self.ledger.wait(tx)
return tx
async def test_create_update_and_abandon_claim(self):
await self.assertBalance(self.account, '10.0')
claim = await self.make_claim(amount='2.5') # creates new claim
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['claim_info']), 1)
self.assertEqual(txs[0]['confirmations'], 1)
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
await self.assertBalance(self.account, '7.479893')
await self.make_claim(amount='1.0') # updates previous claim
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['update_info']), 1)
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.0001985')
await self.assertBalance(self.account, '8.9796945')
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
await self.assertBalance(self.account, '9.9795875')
async def test_update_claim_holding_address(self):
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
other_account = self.daemon.get_account_or_error(other_account_id)
other_address = await other_account.receiving.get_or_create_usable_address()
await self.assertBalance(self.account, '10.0')
# create the initial name claim
claim = await self.make_claim()
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
tx = await self.daemon.jsonrpc_claim_send_to_address(
claim['claim_id'], other_address
)
await self.ledger.wait(tx)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
async def test_publishing_checks_all_accounts_for_certificate(self):
account1_id, account1 = self.account.id, self.account
new_account = await self.daemon.jsonrpc_account_create('second account')
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
self.assertTrue(spam_channel['success'])
await self.confirm_tx(spam_channel['tx']['txid'])
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
))
await self.confirm_tx(result['txid'])
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
self.assertTrue(baz_channel['success'])
await self.confirm_tx(baz_channel['tx']['txid'])
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@spam')
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@baz')
# defaults to using all accounts to lookup channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# uses only the specific accounts which contains the channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name,
channel_name='@baz', channel_account_id=[account2_id]
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# fails when specifying account which does not contain channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name,
channel_name='@baz', channel_account_id=[account1_id]
))
async def test_updating_claim_includes_claim_value_in_balance_check(self):
await self.assertBalance(self.account, '10.0')
await self.make_claim(amount='9.0')
await self.assertBalance(self.account, '0.979893')
# update the same claim
await self.make_claim(amount='9.0')
await self.assertBalance(self.account, '0.9796205')
# update the claim a second time but use even more funds
await self.make_claim(amount='9.97')
await self.assertBalance(self.account, '0.009348')
# fails when specifying more than available
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
with self.assertRaisesRegex(
InsufficientFundsError,
"Please lower the bid value, the maximum amount"
" you can specify for this claim is 9.979274."
):
await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '9.98', file_path=file.name
))
async def test_abandoning_claim_at_loss(self):
await self.assertBalance(self.account, '10.0')
claim = await self.make_claim(amount='0.0001')
await self.assertBalance(self.account, '9.979793')
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
await self.assertBalance(self.account, '9.97968399')
async def test_claim_show(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
channel_from_claim_show = await self.out(
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
)
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
channel_from_claim_show = await self.out(
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
)
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
not_a_claim = await self.out(
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
)
self.assertEqual(not_a_claim, 'claim not found')
async def test_claim_list(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(claim['success'])
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
self.assertTrue(claim['success'])
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
self.assertEqual(len(empty['claims']), 0)
async def test_abandoned_channel_with_signed_claims(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(claim['success'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
response = await self.resolve('lbry://on-channel-claim')
self.assertIn('claim', response['lbry://on-channel-claim'])
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
response = await self.resolve(direct_uri)
self.assertIn('claim', response[direct_uri])
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
uri = 'lbry://@abc/on-channel-claim'
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(original_claim['success'])
# resolves normally
response = await self.resolve(uri)
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
# tamper it, invalidating the signature
value = response[uri]['claim']['value'].copy()
value['stream']['metadata']['author'] = 'some troll'
address = response[uri]['claim']['address']
await self.craft_claim('on-channel-claim', 1, value, address)
# it resolves to the now only valid claim under the channel, ignoring the fake one
response = await self.resolve(uri)
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
# it still resolves! but to the older claim
response = await self.resolve(uri)
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
async def test_claim_list_by_channel(self):
self.maxDiff = None
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
await self.ledger.wait(tx)
await self.generate(1)
await self.ledger.wait(tx)
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
claims = []
for j in range(3):
same_height_claims = []
for k in range(3):
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
self.assertTrue(claim['success'])
same_height_claims.append(claim['claim_id'])
await self.on_transaction_dict(claim['tx'])
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
self.assertTrue(claim['success'])
same_height_claims.append(claim['claim_id'])
same_height_claims.sort(key=lambda x: int(x, 16))
claims = same_height_claims + claims
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
self.assertEqual(page_claim_ids, claims)
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
self.assertEqual(page_claim_ids, claims[:6])
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
async def test_regular_supports_and_tip_supports(self):
# account2 will be used to send tips and supports to account1
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
# send account2 5 LBC out of the 10 LBC in account1
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
))
await self.confirm_tx(result['txid'])
# account1 and account2 balances:
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
# create the claim we'll be tipping and supporting
claim = await self.make_claim()
# account1 and account2 balances:
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
)
await self.on_transaction_dict(tip)
await self.generate(1)
await self.on_transaction_dict(tip)
# tips don't affect balance so account1 balance is same but account2 balance went down
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
# verify that the incoming tip is marked correctly as is_tip=True in account1
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['support_info']), 1)
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
self.assertEqual(txs[0]['value'], '1.0')
self.assertEqual(txs[0]['fee'], '0.0')
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = await self.out(
self.daemon.jsonrpc_transaction_list(account2_id)
)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
self.assertEqual(txs2[0]['value'], '-1.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
)
await self.on_transaction_dict(support)
await self.generate(1)
await self.on_transaction_dict(support)
# account2 balance went down ~2
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
self.assertEqual(txs2[0]['value'], '0.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
async def test_normalization_resolution(self):
# this test assumes that the lbrycrd forks normalization at height == 250 on regtest
c1 = await self.make_claim('ΣίσυφοςfiÆ', '0.1')
c2 = await self.make_claim('ΣΊΣΥΦΟσFIæ', '0.2')
r1 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
r2 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
r1c = list(r1.values())[0]['claim']['claim_id']
r2c = list(r2.values())[0]['claim']['claim_id']
self.assertEqual(c1['claim_id'], r1c)
self.assertEqual(c2['claim_id'], r2c)
self.assertNotEqual(r1c, r2c)
await self.generate(50)
head = await self.daemon.jsonrpc_block_show()
self.assertTrue(head['height'] > 250)
r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
r3c = list(r3.values())[0]['claim']['claim_id']
r4c = list(r4.values())[0]['claim']['claim_id']
r3n = list(r3.values())[0]['claim']['name']
r4n = list(r4.values())[0]['claim']['name']
self.assertEqual(c2['claim_id'], r3c)
self.assertEqual(c2['claim_id'], r4c)
self.assertEqual(r3c, r4c)
self.assertEqual(r3n, r4n)
File renamed without changes.
+191
View File
@@ -0,0 +1,191 @@
import asyncio
import logging
import os
from integration.testcase import CommandTestCase
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.error import InsufficientFundsError
class FileCommands(CommandTestCase):
VERBOSITY = logging.WARN
async def test_file_management(self):
await self.make_claim('foo', '0.01')
await self.make_claim('foo2', '0.01')
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
await self.daemon.jsonrpc_get('lbry://foo')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
async def test_download_different_timeouts(self):
claim = await self.make_claim('foo', '0.01')
sd_hash = claim['output']['value']['stream']['source']['source']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
all_except_sd = [
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
]
await self.server.blob_manager.delete_blobs(all_except_sd)
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
self.assertIn('error', resp)
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.server.blob_manager.delete_blobs([sd_hash])
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
self.assertIn('error', resp)
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
async def wait_files_to_complete(self):
while self.daemon.jsonrpc_file_list(status='running'):
await asyncio.sleep(0.01)
async def test_filename_conflicts_management_on_resume_download(self):
await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
file_info = self.daemon.jsonrpc_file_list()[0]
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
await self.daemon.jsonrpc_file_delete(claim_name='foo')
await self.daemon.jsonrpc_get('lbry://foo')
with open(original_path, 'wb') as handle:
handle.write(b'some other stuff was there instead')
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
# check that internal state got through up to the file list API
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
file_info = self.daemon.jsonrpc_file_list()[0]
self.assertEqual(downloader.output_file_name, file_info['file_name'])
# checks if what the API shows is what he have at the very internal level.
self.assertEqual(downloader.output_path, file_info['download_path'])
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
async def test_incomplete_downloads_erases_output_file_on_stop(self):
claim = await self.make_claim('foo', '0.01')
sd_hash = claim['output']['value']['stream']['source']['source']
file_info = self.daemon.jsonrpc_file_list()[0]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
]
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
self.assertNotIn('error', resp)
self.assertTrue(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
self.daemon.stream_manager.stop()
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
async def test_incomplete_downloads_retry(self):
claim = await self.make_claim('foo', '0.01')
sd_hash = claim['output']['value']['stream']['source']['source']
await self.daemon.jsonrpc_file_delete(claim_name='foo')
blobs = await self.server_storage.get_blobs_for_stream(
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
)
all_except_sd_and_head = [
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
]
# backup server blobs
for blob_hash in all_except_sd_and_head:
blob = self.server_blob_manager.get_blob(blob_hash)
os.rename(blob.file_path, blob.file_path + '__')
# erase all except sd blob
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
# start the download
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
self.assertNotIn('error', resp)
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
self.assertEqual('running', self.daemon.jsonrpc_file_list()[0]['status'])
await self.daemon.jsonrpc_file_set_status('stop', claim_name='foo')
# recover blobs
for blob_hash in all_except_sd_and_head:
blob = self.server_blob_manager.get_blob(blob_hash)
os.rename(blob.file_path + '__', blob.file_path)
self.server_blob_manager.blobs.clear()
await self.server_blob_manager.blob_completed(self.server_blob_manager.get_blob(blob_hash))
await self.daemon.jsonrpc_file_set_status('start', claim_name='foo')
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
file_info = self.daemon.jsonrpc_file_list()[0]
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
self.assertEqual('finished', file_info['status'])
async def test_unban_recovers_stream(self):
BlobDownloader.BAN_TIME = .5 # fixme: temporary field, will move to connection manager or a conf
claim = await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
sd_hash = claim['output']['value']['stream']['source']['source']
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
await self.daemon.jsonrpc_file_delete(claim_name='foo')
# backup blob
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
os.rename(missing_blob.file_path, missing_blob.file_path + '__')
self.server_blob_manager.delete_blob(missing_blob_hash)
await self.daemon.jsonrpc_get('lbry://foo')
with self.assertRaises(asyncio.TimeoutError):
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
# restore blob
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
self.server_blob_manager.blobs.clear()
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
await self.server_blob_manager.blob_completed(missing_blob)
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
async def test_paid_download(self):
target_address = await self.blockchain.get_raw_change_address()
# FAIL: beyond available balance
await self.make_claim(
'expensive', '0.01', data=b'pay me if you can',
fee={'currency': 'LBC', 'amount': 11.0, 'address': target_address})
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
response = await self.daemon.jsonrpc_get('lbry://expensive')
self.assertEqual(response['error'], 'fee of 11.0 exceeds max available balance')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
# FAIL: beyond maximum key fee
await self.make_claim(
'maxkey', '0.01', data=b'no pay me, no',
fee={'currency': 'LBC', 'amount': 111.0, 'address': target_address})
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
response = await self.daemon.jsonrpc_get('lbry://maxkey')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
self.assertEqual(response['error'], 'fee of 111.0 exceeds max configured to allow of 50.0')
# PASS: purchase is successful
await self.make_claim(
'icanpay', '0.01', data=b'I got the power!',
fee={'currency': 'LBC', 'amount': 1.0, 'address': target_address})
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
await self.assertBalance(self.account, '9.925679')
response = await self.daemon.jsonrpc_get('lbry://icanpay')
self.assertNotIn('error', response)
await self.ledger.wait(response['tx'])
await self.assertBalance(self.account, '8.925555')
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
# check that the fee was received
starting_balance = await self.blockchain.get_balance()
await self.generate(1)
block_reward_and_claim_fee = 2.0
self.assertEqual(
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
)
@@ -8,6 +8,7 @@ from lbrynet.schema.signature import Signature, NAMED_SECP256K1
from lbrynet.schema.signer import get_signer
from torba.testcase import IntegrationTestCase
from lbrynet.schema.claim import ClaimDict
import lbrynet.extras.wallet
from lbrynet.extras.wallet.transaction import Transaction
from lbrynet.extras.wallet.account import generate_certificate
from lbrynet.extras.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
@@ -45,7 +46,7 @@ example_claim_dict = {
class BasicTransactionTest(IntegrationTestCase):
VERBOSITY = logging.WARN
LEDGER = lbrynet.extras.wallet
async def test_creating_updating_and_abandoning_claim_with_channel(self):
+59
View File
@@ -0,0 +1,59 @@
from integration.testcase import CommandTestCase
class ResolveCommand(CommandTestCase):
async def test_resolve(self):
await self.make_channel('@abc', '0.01')
# resolving a channel @abc
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertIn('certificate', response['lbry://@abc'])
self.assertNotIn('claim', response['lbry://@abc'])
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 0)
await self.make_claim('foo', '0.01', channel_name='@abc')
await self.make_claim('foo2', '0.01', channel_name='@abc')
# resolving a channel @abc with some claims in it
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertIn('certificate', response['lbry://@abc'])
self.assertNotIn('claim', response['lbry://@abc'])
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 2)
# resolving claim foo within channel @abc
response = await self.resolve('lbry://@abc/foo')
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
claim = response['lbry://@abc/foo']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
# resolving claim foo by itself
response = await self.resolve('lbry://foo')
self.assertSetEqual({'lbry://foo'}, set(response))
claim = response['lbry://foo']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
# resolving multiple at once
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
claim = response['lbry://foo2']
self.assertIn('certificate', claim)
self.assertIn('claim', claim)
self.assertEqual(claim['claim']['name'], 'foo2')
self.assertEqual(claim['claim']['channel_name'], '@abc')
self.assertEqual(claim['certificate']['name'], '@abc')
self.assertEqual(claim['claims_in_channel'], 0)
+91
View File
@@ -0,0 +1,91 @@
from unittest import mock
from lbrynet.schema.claim import ClaimDict
from torba.orchstr8.node import WalletNode, SPVNode
from torba.testcase import AsyncioTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon
from lbrynet.extras.wallet import LbryWalletManager, RegTestLedger
from lbrynet.extras.daemon.Components import WalletComponent
from lbrynet.extras.daemon.Components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbrynet.extras.daemon.ComponentManager import ComponentManager
class AccountSynchronization(AsyncioTestCase):
async def asyncSetUp(self):
self.wallet_node = WalletNode(LbryWalletManager, RegTestLedger)
await self.wallet_node.start(
SPVNode(None),
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach absent",
False
)
self.account = self.wallet_node.account
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = False
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('localhost', 50001)]
conf.reflector_servers = []
conf.known_dht_nodes = []
def wallet_maker(component_manager):
self.wallet_component = WalletComponent(component_manager)
self.wallet_component.wallet_manager = self.wallet_node.manager
self.wallet_component._running = True
return self.wallet_component
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
]
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
))
await self.daemon.initialize()
async def asyncTearDown(self):
self.wallet_component._running = False
await self.daemon.stop()
@mock.patch('time.time', mock.Mock(return_value=12345))
def test_sync(self):
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
self.account.modified_on = 123.456
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
self.assertFalse(self.account.certificates)
hash_w_cert = '974721f42dab42657b5911b7caf4af98ce4d3879eea6ac23d50c1d79bc5020ef'
add_cert = (
'czo4MTkyOjE2OjE6qs3JRvS/bhX8p1JD68sqyA2Qhx3EVTqskhqEAwtfAUfUsQqeJ1rtMdRf40vkGKnpt4NT0b'
'XEqb5O+lba4nkLF7vZENhc2zuOrjobCPVbiVHNwOfH56Ayrh1ts5LMcnl5+Mk1BUyGCwXcqEg2KiUkd3YZpiHQ'
'T7WfcODcU6l7IRivb8iawCebZJx9waVyQoqEDKwZUY1i5HA0VLC+s5cV7it1AWbewiyWOQtZdEPzNY44oXLJex'
'SirElQqDqNZyl3Hjy8YqacBbSYoejIRnmXpC9y25keP6hep3f9i1K2HDNwhwns1W1vhuzuO2Gy9+a0JlVm5mwc'
'N2pqO4tCZr6tE3aym2FaSAunOi7QYVFMI6arb9Gvn9P+T+WRiFYfzwDFVR+j5ZPmUDXxHisy5OF163jH61wbBY'
'pPienjlVtDOxoZmA8+AwWXKRdINsRcull9pu7EVCq5yQmrmxoPbLxNh5pRGrBB0JwCCOMIf+KPwS+7Z6dDbiwO'
'2NUpk8USJMTmXmFDCr2B0PJiG6Od2dD2oGN0F7aYZvUuKbqj8eDrJMe/zLbhq47jUjkJFCvtxUioo63ORk1pzH'
'S0/X4/6/95PRSMaXm4DcZ9BdyxR2E/AKc8UN6AL5rrn6quXkC6R3ZhKgN3Si2S9y6EGFsL7dgzX331U08ZviLj'
'NsrG0EKUnf+TGQ42MqnLQBOiO/ZoAwleOzNZnCYOQQ14Mm8y17xUpmdWRDiRKpAOJU22jKnxtqQ='
)
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})
# applying the same diff is idempotent
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})
@@ -0,0 +1,38 @@
from integration.testcase import CommandTestCase
class TransactionCommandsTestCase(CommandTestCase):
async def test_transaction_show(self):
# local tx
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
))
await self.confirm_tx(result['txid'])
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
self.assertEqual(tx.id, result['txid'])
# someone's tx
change_address = await self.blockchain.get_raw_change_address()
sendtxid = await self.blockchain.send_to_address(change_address, 10)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.id, sendtxid)
self.assertEqual(tx.height, -2)
await self.generate(1)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.height, self.ledger.headers.height)
# inexistent
result = await self.daemon.jsonrpc_transaction_show('0'*64)
self.assertFalse(result['success'])
async def test_utxo_release(self):
sendtxid = await self.blockchain.send_to_address(
await self.account.receiving.get_or_create_usable_address(), 1
)
await self.confirm_tx(sendtxid)
await self.assertBalance(self.account, '11.0')
await self.ledger.reserve_outputs(await self.account.get_utxos())
await self.assertBalance(self.account, '0.0')
await self.daemon.jsonrpc_utxo_release()
await self.assertBalance(self.account, '11.0')
+187
View File
@@ -0,0 +1,187 @@
import json
import shutil
import tempfile
import logging
from binascii import unhexlify
import lbrynet.extras.wallet
from lbrynet.schema.claim import ClaimDict
from torba.testcase import IntegrationTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.Components import Component, WalletComponent
from lbrynet.extras.daemon.Components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.stream.reflector.server import ReflectorServer
from lbrynet.blob_exchange.server import BlobServer
class ExchangeRateManager:
def start(self):
pass
def stop(self):
pass
def convert_currency(self, from_currency, to_currency, amount):
return amount
def fee_dict(self):
return {}
class ExchangeRateManagerComponent(Component):
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.exchange_rate_manager = ExchangeRateManager()
@property
def component(self) -> ExchangeRateManager:
return self.exchange_rate_manager
async def start(self):
self.exchange_rate_manager.start()
async def stop(self):
self.exchange_rate_manager.stop()
class CommandTestCase(IntegrationTestCase):
timeout = 180
LEDGER = lbrynet.extras.wallet
MANAGER = LbryWalletManager
VERBOSITY = logging.WARN
async def asyncSetUp(self):
await super().asyncSetUp()
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
logging.getLogger('lbrynet.stream').setLevel(self.VERBOSITY)
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = True
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.known_dht_nodes = []
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
sendtxid = await self.blockchain.send_to_address(address, 10)
await self.confirm_tx(sendtxid)
await self.generate(5)
def wallet_maker(component_manager):
self.wallet_component = WalletComponent(component_manager)
self.wallet_component.wallet_manager = self.manager
self.wallet_component._running = True
return self.wallet_component
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
]
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
exchange_rate_manager=ExchangeRateManagerComponent
))
await self.daemon.initialize()
self.manager.old_db = self.daemon.storage
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_storage = SQLiteStorage(Config(), ':memory:')
await self.server_storage.open()
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.server.start_server(5567, '127.0.0.1')
await self.server.started_listening.wait()
self.reflector = ReflectorServer(self.server_blob_manager)
self.reflector.start_server(5566, '127.0.0.1')
await self.reflector.started_listening.wait()
self.addCleanup(self.reflector.stop_server)
async def asyncTearDown(self):
await super().asyncTearDown()
self.wallet_component._running = False
await self.daemon.stop()
async def confirm_tx(self, txid):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
await self.on_transaction_id(txid)
await self.generate(1)
await self.on_transaction_id(txid)
async def on_transaction_dict(self, tx):
await self.ledger.wait(
self.ledger.transaction_class(unhexlify(tx['hex']))
)
@staticmethod
def get_all_addresses(tx):
addresses = set()
for txi in tx['inputs']:
addresses.add(txi['address'])
for txo in tx['outputs']:
addresses.add(txo['address'])
return list(addresses)
async def generate(self, blocks):
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
async def out(self, awaitable):
""" Converts Daemon API call results (dictionary)
to JSON and then back to a dictionary. """
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!',
channel_name=None, confirm=True, account_id=None, fee=None):
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(self.daemon.jsonrpc_publish(
name, amount, file_path=file.name, channel_name=channel_name, account_id=account_id,
fee=fee
))
self.assertTrue(claim['success'])
if confirm:
await self.on_transaction_dict(claim['tx'])
await self.generate(1)
await self.on_transaction_dict(claim['tx'])
return claim
async def make_channel(self, name='@arena', amount='1.0', confirm=True, account_id=None):
channel = await self.out(self.daemon.jsonrpc_channel_new(name, amount, account_id))
self.assertTrue(channel['success'])
if confirm:
await self.on_transaction_dict(channel['tx'])
await self.generate(1)
await self.on_transaction_dict(channel['tx'])
return channel
async def resolve(self, uri):
return await self.out(self.daemon.jsonrpc_resolve(uri))
Whitespace-only changes.
-804
View File
@@ -1,804 +0,0 @@
import json
import tempfile
import logging
from binascii import unhexlify
from lbrynet.extras.wallet.transaction import Transaction
from lbrynet.error import InsufficientFundsError
from lbrynet.schema.claim import ClaimDict
from torba.testcase import IntegrationTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.Components import WalletComponent
from lbrynet.extras.daemon.Components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbrynet.extras.daemon.ComponentManager import ComponentManager
class CommandTestCase(IntegrationTestCase):
timeout = 180
MANAGER = LbryWalletManager
VERBOSITY = logging.WARN
async def asyncSetUp(self):
await super().asyncSetUp()
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_uploads = False
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('localhost', 50001)]
conf.known_dht_nodes = []
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
sendtxid = await self.blockchain.send_to_address(address, 10)
await self.confirm_tx(sendtxid)
await self.generate(5)
def wallet_maker(component_manager):
self.wallet_component = WalletComponent(component_manager)
self.wallet_component.wallet_manager = self.manager
self.wallet_component._running = True
return self.wallet_component
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
]
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
))
await self.daemon.initialize()
self.manager.old_db = self.daemon.storage
async def asyncTearDown(self):
await super().asyncTearDown()
self.wallet_component._running = False
await self.daemon.stop()
async def confirm_tx(self, txid):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
await self.on_transaction_id(txid)
await self.generate(1)
await self.on_transaction_id(txid)
async def on_transaction_dict(self, tx):
await self.ledger.wait(
self.ledger.transaction_class(unhexlify(tx['hex']))
)
@staticmethod
def get_all_addresses(tx):
addresses = set()
for txi in tx['inputs']:
addresses.add(txi['address'])
for txo in tx['outputs']:
addresses.add(txo['address'])
return list(addresses)
async def generate(self, blocks):
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
async def out(self, awaitable):
""" Converts Daemon API call results (dictionary)
to JSON and then back to a dictionary. """
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
class EpicAdventuresOfChris45(CommandTestCase):
VERBOSITY = logging.WARN
async def test_no_this_is_not_a_test_its_an_adventure(self):
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
# and everything in between; it's also the setting of some record setting
# integration tests.
# Chris45 starts everyday by checking his balance.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '10.0')
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
# enthusiastically. But he is hungry so he goes into the kitchen
# to make himself a spamdwich.
# While making the spamdwich he wonders... has anyone on LBRY
# registered the @spam channel yet? "I should do that!" he
# exclaims and goes back to his computer to do just that!
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# Do we have it locally?
channels = await self.out(self.daemon.jsonrpc_channel_list())
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@spam')
# As the new channel claim travels through the intertubes and makes its
# way into the mempool and then a block and then into the claimtrie,
# Chris doesn't sit idly by: he checks his balance!
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.989893')
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
await self.generate(6)
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
self.assertEqual(result, '8.989893')
# And is the channel resolvable and empty?
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam'))
self.assertIn('lbry://@spam', response)
self.assertIn('certificate', response['lbry://@spam'])
# "What goes well with spam?" ponders Chris...
# "A hovercraft with eels!" he exclaims.
# "That's what goes great with spam!" he further confirms.
# And so, many hours later, Chris is finished writing his epic story
# about eels driving a hovercraft across the wetlands while eating spam
# and decides it's time to publish it to the @spam channel.
with tempfile.NamedTemporaryFile() as file:
file.write(b'blah blah blah...')
file.write(b'[insert long story about eels driving hovercraft]')
file.write(b'yada yada yada!')
file.write(b'the end')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# He quickly checks the unconfirmed balance to make sure everything looks
# correct.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '7.969786')
# Also checks that his new story can be found on the blockchain before
# giving the link to all his friends.
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
self.assertIn('lbry://@spam/hovercraft', response)
self.assertIn('claim', response['lbry://@spam/hovercraft'])
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
await self.generate(5)
# When he comes back he verifies the confirmed balance.
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '7.969786')
# As people start reading his story they discover some typos and notify
# Chris who explains in despair "Oh! Noooooos!" but then remembers
# "No big deal! I can update my claim." And so he updates his claim.
with tempfile.NamedTemporaryFile() as file:
file.write(b'blah blah blah...')
file.write(b'[typo fixing sounds being made]')
file.write(b'yada yada yada!')
file.flush()
claim2 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
))
self.assertTrue(claim2['success'])
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
await self.confirm_tx(claim2['tx']['txid'])
# After some soul searching Chris decides that his story needs more
# heart and a better ending. He takes down the story and begins the rewrite.
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
# And now checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
# After abandoning he just waits for his LBCs to be returned to his account
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.9693585')
# Amidst all this Chris receives a call from his friend Ramsey
# who says that it is of utmost urgency that Chris transfer him
# 1 LBC to which Chris readily obliges
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
self.assertIn("txid", result)
await self.confirm_tx(result['txid'])
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
# he received it or not
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
# Chris' balance was correct
self.assertEqual(result, '7.9692345')
# Ramsey too assured him that he had received the 1 LBC and thanks him
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
self.assertEqual(result, '1.0')
# After Chris is done with all the "helping other people" stuff he decides that it's time to
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
with tempfile.NamedTemporaryFile() as file:
file.write(b'Amazingly Original First Line')
file.write(b'Super plot for the grand novel')
file.write(b'Totally un-cliched ending')
file.write(b'**Audience Gasps**')
file.flush()
claim3 = await self.out(self.daemon.jsonrpc_publish(
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
))
self.assertTrue(claim3['success'])
await self.confirm_tx(claim3['tx']['txid'])
await self.generate(5)
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
uri = 'lbry://@spam/fresh-start'
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
))
await self.confirm_tx(tx['txid'])
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
# It obviously did! Because, blockchain baby \O/
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
await self.generate(5)
# Now he also wanted to support the original creator of the Award Winning Novel
# So he quickly decides to send a tip to him
tx = await self.out(
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
await self.confirm_tx(tx['txid'])
# And again checks if it went to the just right place
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
# Which it obviously did. Because....?????
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
await self.generate(5)
# Seeing the ravishing success of his novel Chris adds support to his claim too
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
await self.confirm_tx(tx['txid'])
# And check if his support showed up
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
# It did!
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
await self.generate(5)
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
# .......
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
with tempfile.NamedTemporaryFile() as file:
file.write(b'The Whale amd The Bookmark')
file.write(b'I know right? Totally a hit song')
file.write(b'That\'s what goes around for songs these days anyways')
file.flush()
claim4 = await self.out(self.daemon.jsonrpc_publish(
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
))
self.assertTrue(claim4['success'])
await self.confirm_tx(claim4['tx']['txid'])
await self.generate(5)
# He sends the link to Ramsey, all happy and proud
uri = 'lbry://@spam/hit-song'
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
# He them checks that the claim doesn't resolve anymore.
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertNotIn('claim', response[uri])
class AccountManagement(CommandTestCase):
VERBOSE = False
async def test_performing_account_management_commands(self):
# check initial account
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# change account name and gap
account_id = response['lbc_regtest'][0]['id']
self.daemon.jsonrpc_account_set(
account_id=account_id, new_name='test account',
receiving_gap=95, receiving_max_uses=96,
change_gap=97, change_max_uses=98
)
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
self.assertEqual(response['name'], 'test account')
self.assertEqual(
response['address_generator']['receiving'],
{'gap': 95, 'maximum_uses_per_address': 96}
)
self.assertEqual(
response['address_generator']['change'],
{'gap': 97, 'maximum_uses_per_address': 98}
)
# create another account
await self.daemon.jsonrpc_account_create('second account')
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
account_id2 = response['lbc_regtest'][1]['id']
# make new account the default
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
response = await self.daemon.jsonrpc_account_list(show_seed=True)
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
account_seed = response['lbc_regtest'][1]['seed']
# remove account
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# add account
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
# list specific account
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
self.assertEqual(response['name'], 'recreated account')
class ClaimManagement(CommandTestCase):
VERBOSITY = logging.WARN
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!', channel_name=None, confirm=True):
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(self.daemon.jsonrpc_publish(
name, amount, file_path=file.name, channel_name=channel_name
))
self.assertTrue(claim['success'])
if confirm:
await self.on_transaction_dict(claim['tx'])
await self.generate(1)
await self.on_transaction_dict(claim['tx'])
return claim
async def craft_claim(self, name, amount_dewies, claim_dict, address):
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
# However, it still happens... So, let's reproduce them.
claim = ClaimDict.load_dict(claim_dict)
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
await self.broadcast(tx)
await self.ledger.wait(tx)
await self.generate(1)
await self.ledger.wait(tx)
return tx
async def test_create_update_and_abandon_claim(self):
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
claim = await self.make_claim(amount='2.5') # creates new claim
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['claim_info']), 1)
self.assertEqual(txs[0]['confirmations'], 1)
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
self.assertEqual('7.479893', await self.daemon.jsonrpc_account_balance())
await self.make_claim(amount='1.0') # updates previous claim
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['update_info']), 1)
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.0001985')
self.assertEqual('8.9796945', await self.daemon.jsonrpc_account_balance())
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
self.assertEqual('9.9795875', await self.daemon.jsonrpc_account_balance())
async def test_update_claim_holding_address(self):
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
other_account = self.daemon.get_account_or_error(other_account_id)
other_address = await other_account.receiving.get_or_create_usable_address()
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
# create the initial name claim
claim = await self.make_claim()
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
tx = await self.daemon.jsonrpc_claim_send_to_address(
claim['claim_id'], other_address
)
await self.ledger.wait(tx)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
async def test_publishing_checks_all_accounts_for_certificate(self):
account1_id, account1 = self.account.id, self.account
new_account = await self.daemon.jsonrpc_account_create('second account')
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
self.assertTrue(spam_channel['success'])
await self.confirm_tx(spam_channel['tx']['txid'])
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
))
await self.confirm_tx(result['txid'])
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
self.assertTrue(baz_channel['success'])
await self.confirm_tx(baz_channel['tx']['txid'])
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@spam')
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
self.assertEqual(len(channels), 1)
self.assertEqual(channels[0]['name'], '@baz')
# defaults to using all accounts to lookup channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# uses only the specific accounts which contains the channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
claim1 = await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name,
channel_name='@baz', channel_account_id=[account2_id]
))
self.assertTrue(claim1['success'])
await self.confirm_tx(claim1['tx']['txid'])
# fails when specifying account which does not contain channel
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '1.0', file_path=file.name,
channel_name='@baz', channel_account_id=[account1_id]
))
async def test_updating_claim_includes_claim_value_in_balance_check(self):
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
await self.make_claim(amount='9.0')
self.assertEqual('0.979893', await self.daemon.jsonrpc_account_balance())
# update the same claim
await self.make_claim(amount='9.0')
self.assertEqual('0.9796205', await self.daemon.jsonrpc_account_balance())
# update the claim a second time but use even more funds
await self.make_claim(amount='9.97')
self.assertEqual('0.009348', await self.daemon.jsonrpc_account_balance())
# fails when specifying more than available
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi!')
file.flush()
with self.assertRaisesRegex(
InsufficientFundsError,
"Please lower the bid value, the maximum amount"
" you can specify for this claim is 9.979274."
):
await self.out(self.daemon.jsonrpc_publish(
'hovercraft', '9.98', file_path=file.name
))
async def test_abandoning_claim_at_loss(self):
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
claim = await self.make_claim(amount='0.0001')
self.assertEqual('9.979793', await self.daemon.jsonrpc_account_balance())
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
self.assertEqual('9.97968399', await self.daemon.jsonrpc_account_balance())
async def test_claim_show(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
channel_from_claim_show = await self.out(
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
)
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
channel_from_claim_show = await self.out(
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
)
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
not_a_claim = await self.out(
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
)
self.assertEqual(not_a_claim, 'claim not found')
async def test_claim_list(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(claim['success'])
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
self.assertTrue(claim['success'])
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
await self.confirm_tx(abandon['tx']['txid'])
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
self.assertEqual(len(empty['claims']), 0)
async def test_abandoned_channel_with_signed_claims(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(claim['success'])
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
self.assertTrue(abandon['success'])
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@abc/on-channel-claim'))
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://on-channel-claim'))
self.assertIn('claim', response['lbry://on-channel-claim'])
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
response = await self.out(self.daemon.jsonrpc_resolve(uri=direct_uri))
self.assertIn('claim', response[direct_uri])
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
uri = 'lbry://@abc/on-channel-claim'
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
self.assertTrue(original_claim['success'])
# resolves normally
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
# tamper it, invalidating the signature
value = response[uri]['claim']['value'].copy()
value['stream']['metadata']['author'] = 'some troll'
address = response[uri]['claim']['address']
await self.craft_claim('on-channel-claim', 1, value, address)
# it resolves to the now only valid claim under the channel, ignoring the fake one
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
# it still resolves! but to the older claim
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertIn('claim', response[uri])
self.assertTrue(response[uri]['claim']['signature_is_valid'])
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
async def test_claim_list_by_channel(self):
self.maxDiff = None
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
await self.ledger.wait(tx)
await self.generate(1)
await self.ledger.wait(tx)
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
claims = []
for j in range(3):
same_height_claims = []
for k in range(3):
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
self.assertTrue(claim['success'])
same_height_claims.append(claim['claim_id'])
await self.on_transaction_dict(claim['tx'])
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
self.assertTrue(claim['success'])
same_height_claims.append(claim['claim_id'])
same_height_claims.sort(key=lambda x: int(x, 16))
claims = same_height_claims + claims
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
self.assertEqual(page_claim_ids, claims)
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
self.assertEqual(page_claim_ids, claims[:6])
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
async def test_regular_supports_and_tip_supports(self):
# account2 will be used to send tips and supports to account1
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
# send account2 5 LBC out of the 10 LBC in account1
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
))
await self.confirm_tx(result['txid'])
# account1 and account2 balances:
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
# create the claim we'll be tipping and supporting
claim = await self.make_claim()
# account1 and account2 balances:
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
# send a tip to the claim using account2
tip = await self.out(
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
)
await self.on_transaction_dict(tip)
await self.generate(1)
await self.on_transaction_dict(tip)
# tips don't affect balance so account1 balance is same but account2 balance went down
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
# verify that the incoming tip is marked correctly as is_tip=True in account1
txs = await self.out(self.daemon.jsonrpc_transaction_list())
self.assertEqual(len(txs[0]['support_info']), 1)
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
self.assertEqual(txs[0]['value'], '1.0')
self.assertEqual(txs[0]['fee'], '0.0')
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = await self.out(
self.daemon.jsonrpc_transaction_list(account2_id)
)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
self.assertEqual(txs2[0]['value'], '-1.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
# send a support to the claim using account2
support = await self.out(
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
)
await self.on_transaction_dict(support)
await self.generate(1)
await self.on_transaction_dict(support)
# account2 balance went down ~2
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
self.assertEqual(txs2[0]['value'], '0.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
class TransactionCommandsTestCase(CommandTestCase):
async def test_transaction_show(self):
# local tx
result = await self.out(self.daemon.jsonrpc_wallet_send(
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
))
await self.confirm_tx(result['txid'])
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
self.assertEqual(tx.id, result['txid'])
# someone's tx
change_address = await self.blockchain.get_raw_change_address()
sendtxid = await self.blockchain.send_to_address(change_address, 10)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.id, sendtxid)
self.assertEqual(tx.height, -2)
await self.generate(1)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.height, self.ledger.headers.height)
# inexistent
result = await self.daemon.jsonrpc_transaction_show('0'*64)
self.assertFalse(result['success'])
async def test_utxo_release(self):
sendtxid = await self.blockchain.send_to_address(
await self.account.receiving.get_or_create_usable_address(), 1
)
await self.confirm_tx(sendtxid)
await self.assertBalance(self.account, '11.0')
await self.ledger.reserve_outputs(await self.account.get_utxos())
await self.assertBalance(self.account, '0.0')
await self.daemon.jsonrpc_utxo_release()
await self.assertBalance(self.account, '11.0')
Loaded 100 of 118 files, more files were not shown because too many files have changed in this diff. Show more