Compare commits

...
466 Commits
Author SHA1 Message Date
Lex Berezhny 3c1fdb5114 v0.35.0 2019-03-29 21:51:05 -04:00
Lex Berezhny 83411acfbd split claim_abandon into stream_abandon and channel_abandon 2019-03-29 21:48:55 -04:00
Lex Berezhny 4230812f82 added integration test to make sure publishing the same stream re-signs it with the original channel 2019-03-29 21:14:20 -04:00
Lex Berezhny 7b57ba96dd fix docopt parsing and stream manager tests 2019-03-29 21:14:20 -04:00
Lex Berezhny 668840921f typo fix 2019-03-29 21:14:20 -04:00
Lex Berezhny a3bd335f7f --languages, --locations fully support all attributes, add BTC 2019-03-29 21:14:20 -04:00
Lex Berezhny 4e6bca1389 support_abandon sends new --keep support to same address as the first support being abandoned 2019-03-29 21:14:20 -04:00
Lex Berezhny c4b15920ff dont show supports to foreign address if the inputs are also foreign 2019-03-29 21:14:20 -04:00
Lex Berezhny 2a55e583e7 added to docstring that a channel_id is a channel claim id 2019-03-29 21:14:20 -04:00
Lex Berezhny 2c4fde06c5 --new-signing-key -> --new_signing_key 2019-03-29 21:14:20 -04:00
Lex Berezhny eca83d7f47 --clear-tags -> --clear_tags 2019-03-29 21:14:20 -04:00
Lex Berezhny 87e818ac39 v0.35.0rc2 2019-03-27 22:20:09 -04:00
Lex Berezhny adab7bd7b1 fix file command tests 2019-03-27 21:57:22 -04:00
Lex Berezhny 750c0e4f4f fix cli test 2019-03-27 21:57:22 -04:00
Lex Berezhny 5f8482bdb7 hash -> sd_hash, encoding fixes, update to new protobufs 2019-03-27 21:57:22 -04:00
Lex Berezhny cd266477fb stop regenerating v1 protobufs 2019-03-27 21:57:22 -04:00
Lex Berezhny 7860b956ff fixes and tests for publish command 2019-03-27 21:57:22 -04:00
Thomas Zarebczan 148481a5ca doh 2019-03-27 21:54:19 -04:00
Thomas Zarebczan 93faa276a5 update BIP32 2019-03-27 21:54:19 -04:00
Victor Shyba 53e67a6e81 fixes from rebase 2019-03-27 10:31:37 -04:00
Victor Shyba 3ad6808d87 remove unused claim_id_hash 2019-03-27 10:31:37 -04:00
Victor Shyba 071ab32ab7 fix hash references 2019-03-27 10:31:37 -04:00
Victor Shyba ab87588250 use torba parsing for addresses too, erase old code 2019-03-27 10:31:37 -04:00
Victor Shyba 22db29ee1d use torba tx parsing on wallet server 2019-03-27 10:31:37 -04:00
Lex Berezhny 66bf6c61f0 v0.35.0rc1 2019-03-25 23:57:59 -04:00
Lex Berezhny 9abf40d21f stream_type fix 2019-03-25 23:45:10 -04:00
Lex Berezhny ea4444edbe fix internal transaction api test 2019-03-25 22:54:08 -04:00
Lex Berezhny 51bb8fd283 pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 183e3ac1c7 claim search 2019-03-25 22:54:08 -04:00
Lex Berezhny 8087b457c6 reorganized commands and finished support_abandon with --keep 2019-03-25 22:54:08 -04:00
Lex Berezhny fedb7c23d0 bring back blocks_behind in wallet status 2019-03-25 22:54:08 -04:00
Lex Berezhny e2964f9728 test cli fix, take 2 2019-03-25 22:54:08 -04:00
Lex Berezhny e0cea60c87 fix cli test 2019-03-25 22:54:08 -04:00
Lex Berezhny c708c1db07 audio/video/image stream types 2019-03-25 22:54:08 -04:00
Lex Berezhny 1cb696fb5c fix test_claim_show 2019-03-25 22:54:08 -04:00
Lex Berezhny 7cd0213eb9 fixed chris45 again 2019-03-25 22:54:08 -04:00
Lex Berezhny 2007c48af3 fixed tests for setting claim values 2019-03-25 22:54:08 -04:00
Lex Berezhny 50721786d2 skip test_claim_list for now 2019-03-25 22:54:08 -04:00
Lex Berezhny 6d7eb259cb test normalization fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny cc5852a9b9 update pyyaml due to security alert 2019-03-25 22:54:08 -04:00
Lex Berezhny 0592cee567 claim integration test fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny 814eaca524 cli tests fixed 2019-03-25 22:54:08 -04:00
Lex Berezhny e06916f5ae fixing unit tests 2019-03-25 22:54:08 -04:00
Victor Shyba d61e1e9d8c fix proto v1 integration test 2019-03-25 22:54:08 -04:00
Lex Berezhny 25436893bc pylint and a few other fixes 2019-03-25 22:54:08 -04:00
Lex Berezhny 5b9d41fd5c regeneate script after modifying protobufs 2019-03-25 22:54:08 -04:00
Victor Shyba b1720d1659 fix setting sd hash 2019-03-25 22:54:08 -04:00
Lex Berezhny 4b3a6ab875 big support tests passes 2019-03-25 22:54:08 -04:00
Lex Berezhny 67dcede2d8 lots of stuff deleted 2019-03-25 22:54:08 -04:00
Victor Shyba 0872a6fea3 partial test_claim_list_by_channel 2019-03-25 22:54:08 -04:00
Victor Shyba a3aed3219f fix test_abandoned_channel_with_signed_claims 2019-03-25 22:54:08 -04:00
Lex Berezhny af73e1a2f5 chris45 passing again 2019-03-25 22:54:08 -04:00
Lex Berezhny 55bbf5aa74 progress on chris45 2019-03-25 22:54:08 -04:00
Lex Berezhny c1fbb02bfe dont validate inputs in json_response_encoder 2019-03-25 22:54:08 -04:00
Lex Berezhny d47575e8e0 progress 2019-03-25 22:54:08 -04:00
Victor Shyba 8fc4f4aa73 cleanup recent test changes 2019-03-25 22:54:08 -04:00
Victor Shyba fe9aa666e9 fix name is not part of old sigs 2019-03-25 22:54:08 -04:00
Victor Shyba a9e0eeba7e add tests and fix verification of old signatures 2019-03-25 22:54:08 -04:00
Victor Shyba 0830917afb delete unused test method 2019-03-25 22:54:08 -04:00
Victor Shyba 1ed086e90f bring back test for picking oldest on dupe claims 2019-03-25 22:54:08 -04:00
Victor Shyba 56db631626 fix signed channel tests, WIP on tampering 2019-03-25 22:54:08 -04:00
Victor Shyba aa029955ee fix tx api tests 2019-03-25 22:54:08 -04:00
Victor Shyba f2236eb7a4 resolve: check if there is a cert before fetching a tx 2019-03-25 22:54:08 -04:00
Lex Berezhny df649341cd json output formatting fix for Claim 2019-03-25 22:54:08 -04:00
Lex Berezhny a3c3143868 fixed remaining references to lbrynet.extras.wallet -> lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny d0201fe7c7 base integration test class should default to VERBOSITY=WARN, not DEBUG 2019-03-25 22:54:08 -04:00
Lex Berezhny 664a2a324d fix import error on stream unit test 2019-03-25 22:54:08 -04:00
Lex Berezhny ea09aca175 pylint fixes and ClaimDict->Claim 2019-03-25 22:54:08 -04:00
Lex Berezhny ca92fddecf pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 8259d7c680 txo.claim_hash and other fixes related to proto3 refactor 2019-03-25 22:54:08 -04:00
Victor Shyba 6653da2973 all 2019-03-25 22:54:08 -04:00
Lex Berezhny e74998209d for victor 2019-03-25 22:54:08 -04:00
Lex Berezhny 2bea0f697a pylint 2019-03-25 22:54:08 -04:00
Lex Berezhny 1d68bef6f2 client side pretty much done 2019-03-25 22:54:08 -04:00
Lex Berezhny 22bacc907b moved test_channel_signing.py from tests/unit/schema to tests/unit/wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 1e8ad900c7 renamed test_signing.py to test_channel_signing.py 2019-03-25 22:54:08 -04:00
Lex Berezhny cd2b535afb moved claim signing to TXO 2019-03-25 22:54:08 -04:00
Lex Berezhny 9dfe7bbcf6 imports updated after moving lbrynet.extras.wallet to lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 120618ffd5 updated imports due to lbrynet.extras.wallet -> lbrynet.wallet move 2019-03-25 22:54:08 -04:00
Lex Berezhny e8abcbb06e moved lbrynet.extras.wallet -> lbrynet.wallet 2019-03-25 22:54:08 -04:00
Lex Berezhny 1ec8f0b0b4 wip 2019-03-25 22:54:08 -04:00
Lex Berezhny 323694fb12 removed address.py 2019-03-25 22:54:08 -04:00
Lex Berezhny d5e37acd8f minor Fee changes 2019-03-25 22:54:08 -04:00
Lex Berezhny 129a886f2b + fee.amount 2019-03-25 22:54:08 -04:00
Lex Berezhny cd15230a92 getting close to a nice model 2019-03-25 22:54:08 -04:00
Lex Berezhny c1b7fdc68e deleted bunch of stuff 2019-03-25 22:54:08 -04:00
Lex Berezhny 9fd67aaad6 removed legacy_schema_v1 2019-03-25 22:54:08 -04:00
Lex Berezhny ce9dc1836e moved schema/current_schema/claim.py into schema/claim.py 2019-03-25 22:54:08 -04:00
Lex Berezhny 2e557d3a36 converted everyting in schema/legacy/* into schema/jsonschema.py 2019-03-25 22:54:08 -04:00
Lex Berezhny 5df05cb37e make schema.types a module 2019-03-25 22:54:08 -04:00
Lex Berezhny 326b62dc57 moved proto2/proto3 to types/v1 and types/v2 respectively 2019-03-25 22:54:08 -04:00
Lex Berezhny 25cc7ba284 updated schema to latest types: Claim uses oneof type for Channel/Stream 2019-03-25 22:54:08 -04:00
Victor Shyba 550f064302 baseschema 2019-03-25 22:54:08 -04:00
Victor Shyba 11086b93f1 migrate other fields 2019-03-25 22:54:08 -04:00
Victor Shyba df5662136d migrate cert + adapt to latest types repo 2019-03-25 22:54:08 -04:00
Victor Shyba 291d65070d move common schema values to constants.py 2019-03-25 22:54:08 -04:00
Victor Shyba 9b64ee33dc fix imports: schema.schema -> schema.legacy_schema_v1 2019-03-25 22:54:08 -04:00
Victor Shyba 5404d26f2f schema.schema -> schema.legacy_schema_v1 2019-03-25 22:54:08 -04:00
Victor Shyba 26ade1f228 bump merged proto3 2019-03-25 22:54:08 -04:00
Victor Shyba b49f1f1b54 fix imports 2019-03-25 22:54:08 -04:00
Victor Shyba 3bafa2ad22 regenerate protos 2019-03-25 22:54:08 -04:00
Victor Shyba 378ec44f3c update make to generate proto2 and proto3 2019-03-25 22:54:08 -04:00
Victor Shyba 9d0d17b422 rename proto proto2 2019-03-25 22:54:08 -04:00
Victor Shyba f7c8145283 fix travis not to build wallet on PR 2019-03-19 12:22:00 -04:00
Lex Berezhny 0de17730ca v0.34.0 2019-03-15 16:21:29 -04:00
Jack Robison 58743ba19b Merge pull request #1987 from lbryio/more-dht-tests
improve dht unit tests
2019-03-15 15:38:51 -04:00
Jack Robison f2fefbe287 add lbrynet.dht.blob_announcer unit tests 2019-03-15 14:29:02 -04:00
Jack Robison 60a24f0e6e add lbrynet.dht.protocol.data_store unit tests 2019-03-15 12:44:55 -04:00
Jack Robison 664f91bfab add lbrynet.dht.protocol.distance unit tests 2019-03-15 12:44:41 -04:00
Jack Robison 44f92a271f move dht tests to reflect the real directory structure 2019-03-15 12:01:21 -04:00
Jack Robison 6565ca8558 improve lbrynet.dht.serialization unit tests 2019-03-14 18:45:56 -04:00
Jack Robison 3b0ba3e534 v0.34.0rc2 2019-03-14 16:03:39 -04:00
Jack Robison 8106eb1fa5 Merge pull request #1986 from lbryio/fix-timeout-no-peers
fix default download timeout not being set
2019-03-14 15:47:12 -04:00
Jack Robison c9b2829227 fix/test fixed_peer_delay and added_fixed_peers 2019-03-14 15:08:26 -04:00
Jack Robison 4f2f8b16f3 fix pending downloads not being removed 2019-03-14 13:52:00 -04:00
Jack Robison 2ab2b846b1 fix default download timeout not being set 2019-03-14 13:07:11 -04:00
Jack Robison 2df7284c29 Merge pull request #1985 from lbryio/fix-ttfb-analytics
fix the time to the head blob including the time to the sd blob in analytics
2019-03-14 11:17:37 -04:00
Jack Robison ab5d71ff61 fix the time to the head blob including the time to the sd blob in analytics 2019-03-13 16:24:23 -04:00
Lex Berezhny 99b289f135 v0.34.0rc1 2019-03-12 20:29:10 -04:00
Lex Berezhny 6db02e4362 show build type setting when building 2019-03-12 20:23:10 -04:00
Lex Berezhny 98d8b7d463 sync uses new apply APIs in torba 2019-03-12 20:22:41 -04:00
Lex Berezhny 96f1883fc4 check modified_on 2019-03-12 20:22:41 -04:00
Lex Berezhny a914de155a working sync 2019-03-12 20:22:41 -04:00
Lex Berezhny 2dee0ff0bf wip account manifest command 2019-03-12 20:22:41 -04:00
Jack Robison 993cb43e5b fix tests
-fix cleanup of a failed download
2019-03-12 20:22:01 -04:00
Jack Robison 5d212a0f82 time to first bytes analytics 2019-03-12 20:22:01 -04:00
Jack Robison 0cc602ca53 Merge pull request #1978 from lbryio/sigint
handle sigint
2019-03-12 15:06:08 -04:00
Victor Shyba f390ff144d handle sigint 2019-03-12 13:43:12 -04:00
Jack Robison b266cde511 Merge pull request #1980 from lbryio/batched-channel-name-initialization
use _batched_select to populate channel names when initializing files
2019-03-12 13:41:58 -04:00
Jack Robison d0a1541d0a fix test 2019-03-12 13:00:43 -04:00
Jack Robison 4cbde1b08f use _batched_select to populate channel names when initializing files 2019-03-12 09:58:19 -04:00
Lex Berezhny c6740d8d70 v0.33.0 2019-03-08 20:00:45 -05:00
Lex Berezhny 46358142ed v0.33.0rc1 2019-03-06 22:18:04 -05:00
Lex Berezhny f0f651fc77 rely on torba cffi explicit version 2019-03-05 09:32:43 -05:00
Lex Berezhny 5d428138ed try older cffi on windows build 2019-03-05 09:32:43 -05:00
Brannon King 8d9da61c7d reverted temporary code for full local stack from 2019-03-05 09:32:43 -05:00
Brannon King c326a9382c changed to pull down the newer lbrycrd 2019-03-05 09:32:43 -05:00
Brannon King 5bdd87c904 removed name-to-claim lookup table
added unit test for Greek and made it work


pylint: revert bad move


revert claim_sequence changes


fixed broken test
2019-03-05 09:32:43 -05:00
Jack Robison ad134fe8bb v0.32.4 2019-03-01 17:24:55 -05:00
Jack Robison 2f3c616acd v0.32.4rc1 2019-03-01 16:33:59 -05:00
Jack Robison 3caa2a5039 Merge pull request #1968 from lbryio/analytics
restore old download analytics
2019-03-01 16:08:18 -05:00
Jack Robison e38a0f7a95 logging 2019-03-01 15:26:45 -05:00
Jack Robison 9de3f3ff09 don't block on sending download analytics events 2019-03-01 15:22:19 -05:00
Jack Robison df8934d472 restore old download analytics and update tests 2019-03-01 14:50:06 -05:00
Jack Robison be4a96c1b3 Merge pull request #1965 from lbryio/certifi
fix ssl errors, use certifi to make ssl contexts
2019-02-28 13:14:17 -05:00
Jack Robison d97d978961 use certifi for all operating systems 2019-02-28 12:45:56 -05:00
Jack Robison f52e44b56a typing 2019-02-28 12:42:23 -05:00
Jack Robison 10d73fe480 use certifi 2019-02-28 12:40:11 -05:00
Jack Robison 4fd2d994f2 Merge pull request #1964 from lbryio/update-aiohttp
update aiohttp and pylint
2019-02-28 12:38:17 -05:00
Jack Robison d95903d219 update aiohttp and pylint 2019-02-28 12:04:07 -05:00
Jack Robison 591e66bfbb Merge pull request #1952 from lbryio/no_trap
stop trapping CancelledError
2019-02-27 11:06:02 -05:00
Victor Shyba e047748961 log dropped api request instead of stack trace 2019-02-26 17:42:45 -05:00
Victor Shyba 9f071bbe36 pylint 2019-02-26 17:42:45 -05:00
Victor Shyba 2d8ebe25ed refactor so we can stop trapping CancelledError everywhere 2019-02-26 17:42:45 -05:00
Victor Shyba 7a96e742f2 stop trapping CancelledError 2019-02-26 17:42:45 -05:00
Jack Robison 13c7026b20 Merge pull request #1961 from lbryio/pin-requirements
Pin requirements
2019-02-26 17:42:13 -05:00
Jack Robison 876a3c68dd pin requirements 2019-02-26 17:20:16 -05:00
Jack Robison 573617d6b6 Merge pull request #1957 from lbryio/dht_memory
add LRU to KademliaPeer creation
2019-02-26 10:10:33 -05:00
Victor Shyba e8c9abb944 add LRU to KademliaPeer creation 2019-02-26 09:42:32 -05:00
Jack Robison 8cee627dfa Merge pull request #1959 from lbryio/pin-jsonschema
pin jsonschema version to 2.6.0
2019-02-26 09:41:20 -05:00
Jack Robison ab4a5cee01 pin jsonschema version 2019-02-26 09:23:41 -05:00
Lex Berezhny 5bb0a07fae pip with git does not appear to work in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 5ca594b9fd upgrade pip in wine 2019-02-22 20:52:42 -05:00
Lex Berezhny 3910964df1 plyvel no longer installed unless platform is explicitely linux in torba/setup.py 2019-02-22 20:52:42 -05:00
Jack Robison faa1935203 Merge pull request #1946 from lbryio/file-type-fix
lowercase/stripped comparison for known types
2019-02-22 13:07:09 -05:00
Jack Robison ba518e2a0e update test 2019-02-22 11:12:31 -05:00
Thomas Zarebczan 2a82ff7cc5 Update mime_types.py 2019-02-22 11:11:10 -05:00
Jack Robison 673b958020 v0.32.3 2019-02-21 19:38:14 -05:00
Jack Robison 3f4ed0a896 Merge pull request #1951 from lbryio/fix_invalid_blob_decode_startup
Fix invalid blob decode startup
2019-02-21 19:36:02 -05:00
Victor Shyba f49578cdbc fix integration test 2019-02-21 21:14:51 -03:00
Victor Shyba b834f29634 nullify length on blob delete 2019-02-21 21:00:28 -03:00
Victor Shyba 61132312dd test nullify length on blob delete 2019-02-21 21:00:20 -03:00
Victor Shyba fcdd9295b4 delete upon failing to decode JSON from sd blob 2019-02-21 20:53:21 -03:00
Victor Shyba edb5083a5e test that corrupted sd blobs gets delete if fails to decode 2019-02-21 20:53:04 -03:00
Victor Shyba 4e7d88311f raise proper exception on undecode-able blobs 2019-02-21 20:42:51 -03:00
Victor Shyba aa40337ded test that invalid blobs raise proper exception 2019-02-21 20:42:36 -03:00
Jack Robison 690e2e7e0c v0.32.3rc1 2019-02-21 18:37:19 -05:00
Jack Robison 0c67469321 Merge pull request #1950 from lbryio/fix-slow-ping-queue-shutdown
Fix slow ping queue shutdown
2019-02-21 18:12:22 -05:00
Jack Robison fc64d8d1a8 fix slow ping queue shutdown 2019-02-21 17:44:02 -05:00
Jack Robison 7fe9513857 v0.32.2 2019-02-21 14:46:36 -05:00
Jack Robison e76107c318 v0.32.2rc3 2019-02-21 14:28:13 -05:00
Jack Robison 3868dea819 don't join buckets if there is only one 2019-02-21 14:18:48 -05:00
Jack Robison c52541197f Merge pull request #1949 from lbryio/fix-join-buckets
fix cases where buckets could be unjoined after removing a peer
2019-02-21 14:08:34 -05:00
Jack Robison 03cd6b7200 fix cases where buckets could be unjoined after removing a peer 2019-02-21 13:53:35 -05:00
Jack Robison 6915a7a553 v0.32.2rc2 2019-02-21 13:16:32 -05:00
Jack Robison 7ad29b4bfb Merge pull request #1948 from lbryio/duplicates-in-routing
Test and fix edge cases for adding peers to routing
2019-02-21 13:11:34 -05:00
Jack Robison 4294bf742d simplify/fix ping queue 2019-02-21 13:00:23 -05:00
Jack Robison 57ad9f1c52 test_add_peer_after_handle_request 2019-02-21 11:34:25 -05:00
Jack Robison e260de59f1 fix updating/setting contact triple 2019-02-21 10:50:11 -05:00
Jack Robison 154702e6dd fix KBucket.add_peer 2019-02-21 10:50:11 -05:00
Jack Robison d06d998c81 improve KBucket test_add_peer 2019-02-21 10:50:11 -05:00
Jack Robison 8aff860bfe Merge pull request #1945 from lbryio/fix_delete_log
Fix delete log
2019-02-21 10:14:29 -05:00
Victor Shyba d7ba90bbea update ttfb script 2019-02-20 17:10:36 -03:00
Victor Shyba 59325ad8c1 fix delete log 2019-02-20 17:04:08 -03:00
Jack Robison 1f72e27272 improve time_to_first_byte.py 2019-02-20 17:04:08 -03:00
Jack Robison aa41b8b558 Merge pull request #1944 from lbryio/fix-seed-node-startup
fix seed node startup
2019-02-20 15:03:47 -05:00
Jack Robison 2496e7ed44 improve time_to_first_byte.py 2019-02-20 14:43:40 -05:00
Jack Robison ee12985135 dont ping outselves 2019-02-20 14:15:31 -05:00
Jack Robison 4c7f357441 v0.32.2rc1 2019-02-20 12:12:04 -05:00
Jack Robison 8944a36291 Merge pull request #1942 from lbryio/split_buckets_under_index
add split_buckets_under_index config setting for seed nodes
2019-02-20 12:00:46 -05:00
Jack Robison cb5d428d12 add split_buckets_under_index config setting for seed nodes 2019-02-20 11:25:50 -05:00
Jack Robison f175e8826c Merge pull request #1939 from lbryio/missing_await
refactor file_name and full_path as it can be None, fix missing await, stop using blob_manager.storage on stream_manager and use your own
2019-02-20 10:44:35 -05:00
Victor Shyba 38ed212c01 refactor, dont call storage from blob_manager + missing await 2019-02-19 21:45:14 -03:00
Victor Shyba d035c13883 file_path can be None, add helper to check if stream output file exists instead 2019-02-19 21:45:14 -03:00
Jack Robison 82ce2ccee6 update docs 2019-02-19 17:27:31 -05:00
Jack Robison 2cf9b2dd9e Merge pull request #1938 from lbryio/fix-kbucket-indexing
fix KBucket.key_in_range
2019-02-19 15:03:16 -05:00
Jack Robison fcbdaa336f fix KBucket.key_in_range
The range_min and range_max of a KBucket where mistakenly bounding the node id of a peer to be added rather than the distance from the local node id to the peer node id
2019-02-19 13:57:11 -05:00
Lex Berezhny 3ae6137edb v0.32.1 2019-02-18 22:56:35 -05:00
Lex Berezhny 0fdd2ae178 added tx to JSON response from get 2019-02-18 22:53:11 -05:00
Victor Shyba 8c5007fe07 assert amount transferred to correct account 2019-02-18 22:53:11 -05:00
Victor Shyba d2150cf142 fix balance check 2019-02-18 22:53:11 -05:00
Victor Shyba ce2bb22929 add tests for paid downloads 2019-02-18 22:53:11 -05:00
Lex Berezhny 1ce3adb6a0 reduce integration test verbosity 2019-02-18 21:45:37 -05:00
Lex Berezhny c2dc1f3c1d v0.32.1rc1 2019-02-18 17:17:20 -05:00
Victor Shyba b36c22e2f4 unban after 10 seconds, give up after 60 2019-02-18 17:16:10 -05:00
Victor Shyba 71b66ab337 test stall unban 2019-02-18 17:16:10 -05:00
Lex Berezhny 461c3c8408 downloading header from S3 correctly tracks progress 2019-02-18 17:15:40 -05:00
Thomas Zarebczan 03396c1993 remove invalid reflector_component test config 2019-02-18 17:10:42 -05:00
Jack Robison fa7e9923f4 Merge pull request #1932 from lbryio/fix-1930
Fix blob_list for sd_hash including stream terminator
2019-02-18 14:41:00 -05:00
Jack Robison 3f7e466c97 return failure message for invalid blob hash in blob_delete 2019-02-18 12:18:56 -05:00
Jack Robison 4483380352 fix terminator blob hash (None) being included when running blob_list for an sd hash 2019-02-18 12:12:14 -05:00
Lex Berezhny 4a0a6b0243 v0.32.0 2019-02-16 20:17:41 -05:00
Thomas Zarebczan a003e794ba listen on internal port instead of upnp redirect 2019-02-16 18:55:58 -05:00
Lex Berezhny 051fabc180 v0.32.0rc3 2019-02-16 15:07:07 -05:00
Jack Robison ca686041e7 Merge pull request #1925 from lbryio/fix-extension
fix application/x-ext mime types
2019-02-15 21:11:05 -05:00
Jack Robison fc450eb164 add TestMimeTypes, pylint 2019-02-15 20:42:55 -05:00
Thomas Zarebczan 652d7cbd24 consider . extension 2019-02-15 20:40:55 -05:00
Thomas Zarebczan 43b372c0fe fix extension - remove dot
The rest of our extensions don't have a dot. Also, strip off any characters and lowercase it to be safe.
2019-02-15 20:40:55 -05:00
Jack Robison eaacf7e034 Merge pull request #1926 from lbryio/fix_unfair_ban
Fix p2p ignoring good peers
2019-02-15 20:39:49 -05:00
Victor Shyba 576da1bc30 do not keep trying when blob is being written 2019-02-15 21:20:06 -03:00
Victor Shyba 5c217f1544 resovle reflector IP before adding it 2019-02-15 21:20:06 -03:00
Jack Robison cfbf0dbe0f Merge pull request #1919 from lbryio/sync-blobs-on-startup
Sync blobs on startup
2019-02-15 18:40:33 -05:00
Jack Robison fd081496c0 fix blocking on data being written for resumed streams 2019-02-15 17:35:46 -05:00
Jack Robison 32b4405a4c logging
fix https://github.com/lbryio/lbry/issues/1881
2019-02-15 17:26:28 -05:00
Jack Robison 0d558bd552 fix default file list sorting 2019-02-15 17:26:24 -05:00
Jack Robison ca835f3f80 fix 1883 2019-02-15 16:26:13 -05:00
Jack Robison dbb6ba6241 recover streams with missing sd blobs, handle previous sd blob bugs
-test download and recover stream with old key sorting
2019-02-15 15:21:29 -05:00
Jack Robison a228d20137 move store_stream and store_file to standalone functions 2019-02-15 15:21:07 -05:00
Jack Robison f1b60e3ef2 fix get_blobs_for_stream 2019-02-15 15:21:07 -05:00
Jack Robison 39737c790f sync blob files in database with those in directory on startup 2019-02-15 15:21:07 -05:00
Jack Robison dba4a09d38 Merge pull request #1922 from lbryio/supress_invalid_token_error_to_debug
supress invalid token error to debug
2019-02-15 15:19:58 -05:00
Victor Shyba 918122e7cf supress invalid token error to debug 2019-02-15 16:31:47 -03:00
Victor Shyba a49f479a5f move wait to where its needed on assembler 2019-02-15 10:26:05 -05:00
Victor Shyba 320aabbd70 delete duplicated method accident from rebase 2019-02-15 10:26:05 -05:00
Jack Robison 19fdcad5af Merge pull request #1905 from lbryio/faster-file-delete
Faster file delete
2019-02-14 17:23:55 -05:00
Jack Robison 1734e08297 fix test 2019-02-14 16:34:37 -05:00
Jack Robison 9d89f9b834 use executemany instead of batched_operation 2019-02-14 15:57:24 -05:00
Jack Robison 56c41b2fea fix integration test
-cancel running reflector uploads from publishes upon shutdown
-refactor blob.delete to be non-async
-delete blobs synchronously
2019-02-14 15:43:26 -05:00
Jack Robison 9c8593d88e fix file_delete being slow 2019-02-14 12:33:48 -05:00
Jack Robison 85f41887fd don't create BlobFile object when deleting if it doesn't already exist 2019-02-14 12:28:42 -05:00
Mark Michon 8eede6dfc7 Update LICENSE
Updating license to 2019
2019-02-14 08:47:26 -05:00
Victor Shyba 0d8c37a63b resolve uri->url on ttfb 2019-02-14 00:30:24 -05:00
Lex Berezhny d2ba3fdcfb v0.32.0rc2 2019-02-14 00:18:21 -05:00
Victor Shyba be5ab0f307 test get retry 2019-02-14 00:10:44 -05:00
Victor Shyba 04f70d1f31 stop when data timeout 2019-02-14 00:10:44 -05:00
Victor Shyba 60ba07cc17 add tests for incomplete files, update existing one 2019-02-14 00:10:44 -05:00
Victor Shyba dbc8cfe5ae delete incomplete files on assembler stop 2019-02-14 00:10:44 -05:00
Victor Shyba d91e8f5996 revert import to original one + remove print 2019-02-13 23:33:49 -05:00
Victor Shyba b6df2b9d28 missing None for optional field + pylint 2019-02-13 23:33:49 -05:00
Victor Shyba 5c1ab42277 resume should put the stream into wait as done usually on get calls 2019-02-13 23:33:49 -05:00
Victor Shyba bbdc96e762 file_name as a property of managed stream, to reflect internal changes 2019-02-13 23:33:49 -05:00
Victor Shyba fc0c949187 properly update file name on downloader/assembler 2019-02-13 23:33:49 -05:00
Victor Shyba 3fce8e4023 tests resuming downloads with conflicting file names 2019-02-13 23:33:49 -05:00
Victor Shyba 37bb765a2e resume downloads if DHT disabled 2019-02-13 23:33:49 -05:00
hackrush 21ca4f60f7 Stop the daemon on calling lbrynet stop 2019-02-13 14:22:23 -05:00
Victor Shyba 119d498bce apply a timeout on test wait so if it hangs it fails fast 2019-02-13 13:10:27 -05:00
Victor Shyba 8108aa39c1 verify all blobs got transferred 2019-02-13 13:10:27 -05:00
Victor Shyba 0a068011d8 fix chunked response parsing 2019-02-13 13:10:27 -05:00
Victor Shyba 1193d2cea4 adds tests for chunked responses 2019-02-13 13:10:27 -05:00
Lex Berezhny 76ecb5d50d renamed resolve argument url to urls 2019-02-13 11:55:12 -05:00
Lex Berezhny dbc1164562 v0.32.0rc1 2019-02-12 21:35:41 -05:00
Victor Shyba e2170ec83d fix download timeout types 2019-02-12 20:58:31 -05:00
Victor Shyba 266e777655 add integration test for different download errors 2019-02-12 20:58:31 -05:00
Lex Berezhny 68905d82b9 pass conf instead of data_dir to migrate_db, fixes #1898 2019-02-12 18:10:34 -05:00
Lex Berezhny 396c49f7c6 passing get, file list and file delete tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 5e509c9fd6 testing get 2019-02-12 11:54:00 -05:00
Lex Berezhny e6b6a3f55e dedicated transaction commands tests 2019-02-12 11:54:00 -05:00
Lex Berezhny 019af119d9 moved test_cli.py up one level 2019-02-12 11:54:00 -05:00
Lex Berezhny 58da2f16c2 use absolute imports instead of relative 2019-02-12 11:54:00 -05:00
Lex Berezhny 8aa50aa14f cleaned up resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny 65c6c83b51 reusable command testcase 2019-02-12 11:54:00 -05:00
Lex Berezhny f497989871 split out claim commands 2019-02-12 11:54:00 -05:00
Lex Berezhny 5366779b1b moved all other tests out of test_chris45.py and into dedicated files 2019-02-12 11:54:00 -05:00
Lex Berezhny a3f59f041a dedicated integration tests for account commands 2019-02-12 11:54:00 -05:00
Lex Berezhny d560b0c1cd dedicated integration tests for resolve command 2019-02-12 11:54:00 -05:00
Lex Berezhny e94229121a explicitly set LEDGER 2019-02-12 11:54:00 -05:00
Lex Berezhny 097291a498 moved wallet integration tests up one level to simplify 2019-02-12 11:54:00 -05:00
Jack Robison afdc44ca13 Merge pull request #1894 from lbryio/fix-max-key-fee
fix max_key_fee not being followed
2019-02-11 16:27:29 -05:00
Jack Robison 6732b302bb logging 2019-02-11 15:31:31 -05:00
Jack Robison c9d54bb049 fix max_key_fee 2019-02-11 15:26:13 -05:00
Alex Grintsvayg f65e70df2f better resolve description 2019-02-11 12:34:57 -05:00
Jack Robison 4541e754dd Merge pull request #1892 from lbryio/batch-query-files
Improve startup time
2019-02-08 20:43:52 -05:00
Jack Robison 9f89c62e8a download full streams in time_to_first_byte.py 2019-02-08 20:14:10 -05:00
Jack Robison 6386815a29 fix reflect_streams not being followed 2019-02-08 20:13:26 -05:00
Jack Robison 0c93090d2c faster is_valid_hashcharacter 2019-02-08 19:57:26 -05:00
Jack Robison 45404b6ad4 faster is_valid_blobhash 2019-02-08 19:54:59 -05:00
Jack Robison aeb2891baa faster population of channel names 2019-02-08 19:51:13 -05:00
Jack Robison 8abfdc82b2 faster blob manager startup 2019-02-08 17:37:35 -05:00
Jack Robison da9b4e317a limit blobs to announce 2019-02-08 16:29:55 -05:00
Jack Robison e414cc5c48 whitespace 2019-02-08 16:11:56 -05:00
Jack Robison 939edb2f18 logging
-call loop.set_debug when using --verbose
2019-02-08 16:06:38 -05:00
Jack Robison 868110a6f2 populate stream manager with a single batch query 2019-02-08 16:06:35 -05:00
Jack Robison 869a8b712b Merge pull request #1889 from lbryio/download_stuc
Refactor peer scoring and maintain connections to fix stuck downloads, remove locking from ping queue
2019-02-08 15:10:13 -05:00
Victor Shyba 9ecec52412 check lock acquired to prevent races 2019-02-08 04:39:28 -03:00
Victor Shyba 8b25acff6e do not wait blob to be written, log errors from races 2019-02-08 04:32:53 -03:00
Victor Shyba e2b06677b5 defer db insert during downloads 2019-02-08 03:45:23 -03:00
Victor Shyba af1619ebfb disable race limiting for now 2019-02-08 03:32:38 -03:00
Victor Shyba 3352e0e4f4 score peers by speed 2019-02-08 03:08:08 -03:00
Victor Shyba 141d68a2cd fix write race condition 2019-02-08 02:39:29 -03:00
Victor Shyba c06ec6cd69 manage connections, reusing them when possible 2019-02-08 02:27:58 -03:00
Victor Shyba 1be5dce30e stop racing during long streams 2019-02-08 01:28:03 -03:00
Victor Shyba 73da223ae1 conf: give up first contact earlier, but let the transfer go longer 2019-02-08 01:05:58 -03:00
Victor Shyba 2d7eb83518 change score calculation, wait for active peers too, simplify peer sorting/keeping 2019-02-08 01:04:38 -03:00
Victor Shyba ad03f91d24 let cancelled errors flow 2019-02-08 01:03:13 -03:00
Victor Shyba 5586a226c2 bypass parser during download 2019-02-07 21:19:31 -03:00
Victor Shyba a616582733 wait on connection tasks 2019-02-07 20:11:57 -03:00
Victor Shyba 43ac928f0b remove dht locks 2019-02-07 20:11:57 -03:00
Lex Berezhny a121110743 v0.31.0 2019-02-06 14:28:40 -05:00
Jack Robison 8bc5bb4e14 Merge pull request #1873 from lbryio/fix_incomplete_downloads_assembly
test incomplete streams, respect real length, delete on incomplete as…
2019-02-06 14:26:40 -05:00
Victor Shyba 3a15ff4bcd test incomplete streams, respect real length, delete on incomplete assembly 2019-02-06 15:56:06 -03:00
Jack Robison 3508da4993 Merge pull request #1866 from lbryio/non-async-close-blob
refactor blob.close to be non-async, speed up deleting blobs and streams
2019-02-06 13:52:37 -05:00
Jack Robison dfdc8eda00 improve test, fix stream_finished_event 2019-02-06 13:32:50 -05:00
Jack Robison 58f6cb71c6 test stream manager 2019-02-06 13:08:57 -05:00
Jack Robison ae11c5bb4b fix setting finished event 2019-02-06 12:18:24 -05:00
Jack Robison f22b5da170 fix raised error for a peer address mismatch 2019-02-06 12:11:22 -05:00
Jack Robison c1c6d5bc99 fix deleting partial downloads when stopped and previous streams when updating a publish 2019-02-06 11:43:05 -05:00
Jack Robison 0e972ec2ae refactor BlobFile.close to be non async 2019-02-06 11:00:57 -05:00
Jack Robison f8dc0f298e Merge pull request #1868 from lbryio/fix_error_response
format error dict on get
2019-02-06 10:37:31 -05:00
Victor Shyba 8b5803c945 format error dict on get 2019-02-06 11:55:12 -03:00
Lex Berezhny 1314f75219 release script shows PRs that did not get included in changelog 2019-02-06 09:51:09 -05:00
Jack Robison be90ec51e4 release-text 2019-02-06 09:00:59 -05:00
Jack Robison e06c903731 Merge pull request #1867 from lbryio/stream-descriptor-validation
Add stream descriptor tests
2019-02-05 21:43:45 -05:00
Jack Robison 76c8447a56 add stream descriptor tests 2019-02-05 21:25:31 -05:00
Jack Robison 82b53e108c Merge pull request #1865 from lbryio/fix-delete-previous-stream
Fix deleting incomplete downloads when they are stopped
2019-02-05 15:43:55 -05:00
Jack Robison 7c8ad0fe30 fix deleting incomplete downloads upon stop
fix deleting previous streams (not assembled/published files) for a claim being updated
2019-02-05 15:33:08 -05:00
Jack Robison 073ffdd049 proto and type args 2019-02-05 13:36:25 -05:00
Jack Robison 7fe211d134 fix resolve_host 2019-02-05 13:31:57 -05:00
Jack Robison 43d424978d Merge pull request #1862 from lbryio/fix_announce
adds announce tests on stream creation/assemble, refactor calls to set announce
2019-02-05 12:58:56 -05:00
Victor Shyba a4ced4dbd2 set verified after callback 2019-02-05 13:17:00 -03:00
Victor Shyba c6b910a5f3 use time_getter on storage 2019-02-05 13:14:59 -03:00
Victor Shyba 63d1ec3c28 fixes from jack comments 2019-02-05 13:01:12 -03:00
Victor Shyba c31e61586d tests: fix value, compare to blobs len 2019-02-05 13:01:12 -03:00
Victor Shyba 7b8f42e6d0 remove dead set_should_announce code 2019-02-05 13:01:12 -03:00
Victor Shyba 2c275efa48 remove redundant calls to should announce 2019-02-05 13:01:12 -03:00
Victor Shyba 30846f932b test that creating a stream marks sd and head to announce 2019-02-05 13:01:12 -03:00
Victor Shyba 6ead932ccb announces naturally, removes redundant calls 2019-02-05 13:01:12 -03:00
Victor Shyba e9b58577dd tests: verify completed and to announce after assembling streams 2019-02-05 13:01:12 -03:00
Victor Shyba f766dbefe0 announce and set completed after inserts 2019-02-05 13:01:12 -03:00
Jack Robison f8a7d2ebe2 add release_texts to release.py 2019-02-05 09:58:42 -05:00
Jack Robison af43b21ef6 Merge pull request #1860 from lbryio/fix-announce
Fix setting blob completed and next announce time
2019-02-04 18:39:10 -05:00
Jack Robison 0685bc5689 include torba in log file 2019-02-04 18:25:47 -05:00
Jack Robison 8cda3143f2 fix setting should_announce and blob status 2019-02-04 18:22:44 -05:00
Jack Robison 5af351a234 reflector logging 2019-02-04 17:33:25 -05:00
Jack Robison b9934a7eb7 Merge pull request #1859 from lbryio/fix_invalid_data_client
handle and log if downloading from bad sendfile implementations
2019-02-04 17:19:57 -05:00
Victor Shyba b82d2c3120 handle and log if downloading from bad sendfile implementations 2019-02-04 19:13:31 -03:00
Jack Robison 8e607fcc2a Merge pull request #1858 from lbryio/fix-blob-sendfile
Fix blob sendfile
2019-02-04 17:12:50 -05:00
Jack Robison 2bce7cecfa fix sendfile on windows 2019-02-04 16:48:48 -05:00
Jack Robison e141cc9e22 Merge pull request #1857 from lbryio/handle-invalid-data-and-fix-cancel-blob-request
Handle invalid data and fix cancel blob request
2019-02-04 16:47:35 -05:00
Jack Robison 8883587cc7 logging 2019-02-04 16:19:27 -05:00
Jack Robison c85991704d catch invalid data error 2019-02-04 16:19:27 -05:00
Jack Robison 377442d9f0 fix request_blob cancellation 2019-02-04 16:19:24 -05:00
Jack Robison ad30b7df6d Merge pull request #1855 from lbryio/findvalue-always-include-close-contacts
Always return close peers from findValue
2019-02-04 15:30:17 -05:00
Jack Robison f0b49fc153 always return close peers from findValue 2019-02-04 15:07:42 -05:00
Jack Robison 6033399f05 Merge pull request #1854 from lbryio/no-self-queries
fix trying downloading from oneself, prune querying peer from findValue results
2019-02-04 15:00:13 -05:00
Jack Robison ae1162f5b0 don't try downloading from oneself
-don't return the querying peer in results for findValue
2019-02-04 14:50:53 -05:00
Jack Robison 7d7d4c35c3 Merge pull request #1852 from lbryio/close_protocol_reflector
refactor reflect to close protocol
2019-02-04 14:42:18 -05:00
Victor Shyba ad71a9a89d refactor reflect to close protocol 2019-02-04 13:34:18 -03:00
Jack Robison 13b3e4ba8e v0.31.0rc5 2019-02-04 10:22:43 -05:00
Jack Robison 43799168ba Merge pull request #1850 from lbryio/resume
Properly resume streams and fixup with statement
2019-02-04 10:18:31 -05:00
Victor Shyba 736a949cb5 resume streams on startup 2019-02-03 20:49:09 -03:00
Victor Shyba 11bb1cc7ee use with clause to avoid leaking fds 2019-02-03 20:41:47 -03:00
Jack Robison dbce49e567 Merge pull request #1849 from lbryio/fix-connection-status
Fix connection status
2019-02-03 17:00:14 -05:00
Jack Robison dcd78bd86e always delete file from downloads if download was not complete 2019-02-03 16:43:40 -05:00
Jack Robison e5af041627 fix connection status message 2019-02-03 16:20:52 -05:00
Jack Robison 73f4e954e5 Merge pull request #1848 from lbryio/fix-announce-time
Fix setting blob announce time
2019-02-03 16:18:33 -05:00
Jack Robison 0a027cf2de logging 2019-02-03 15:47:19 -05:00
Jack Robison 3f98b16800 fix blob announce time, use time.time instead of loop.time 2019-02-03 15:47:15 -05:00
Jack Robison bbadece75d remove unused packages from extension-pkg-whitelist in pylintrc 2019-02-03 15:02:43 -05:00
Jack Robison 8f5a8ab8ba fix peer_ping 2019-02-02 02:02:19 -05:00
Jack Robison 91e581422c v0.31.0rc4 2019-02-02 00:30:57 -05:00
Jack Robison b54d7185a1 Merge pull request #1847 from lbryio/re-reflect
add re-reflect task to stream manager, add concurrent_reflector_uploads to config
2019-02-01 23:11:06 -05:00
Jack Robison f2968aab22 add re-reflect task to stream manager, add concurrent_reflector_uploads to config 2019-02-01 22:59:41 -05:00
Jack Robison 4e27eaf8e6 Merge pull request #1846 from lbryio/create-directories-if-dont-exist
create data. download, and wallet directories if they don't exist on startup
2019-02-01 22:57:27 -05:00
Jack Robison ccbe8aa0eb create data. download, and wallet directories if they don't exist on startup 2019-02-01 22:45:28 -05:00
Jack Robison 478f79353a fix bytes error 2019-02-01 22:20:11 -05:00
Jack Robison c3b0151943 fix dht status error during startup 2019-02-01 22:12:07 -05:00
Jack Robison ba818354af Merge pull request #1844 from lbryio/reassemble-when-deleted-from-downloads
Reassemble when deleted from downloads
2019-02-01 21:59:36 -05:00
Jack Robison e96b75a0d0 fix file/download paths
move download-from-uri logic into stream manager
2019-02-01 21:40:35 -05:00
Jack Robison efe4afd09e fix blob announce time 2019-02-01 21:28:59 -05:00
Jack Robison c7cb6822cc logging 2019-02-01 21:28:59 -05:00
Jack Robison c75665d3f0 update file_list docs, add blobs_remaining to file list args and to file dict result 2019-02-01 21:28:59 -05:00
Jack Robison 744375b2c0 re-assemble file / resume downloads 2019-02-01 21:28:59 -05:00
Jack Robison c1b4a012ec Merge pull request #1845 from lbryio/stuck_peers_fixup
fix active peers not being removed after closing + parsing of data stream
2019-02-01 21:28:20 -05:00
Victor Shyba 40da5b6cb6 rework data parser 2019-02-01 20:09:14 -03:00
Victor Shyba b36cf3e36c remove closing peers from active connections 2019-02-01 20:09:14 -03:00
Jack Robison 4e3b529aec Merge pull request #1840 from lbryio/close-transport-on-sendfile-fail
close blob server transport on sendfile failing
2019-02-01 15:13:07 -05:00
Jack Robison 0a08f445a4 close blob server transport on sendfile failing 2019-02-01 15:11:37 -05:00
Jack Robison 3eac99c509 Merge pull request #1841 from lbryio/general_fixes_downloader
General fixes downloader
2019-02-01 15:10:36 -05:00
Victor Shyba f5bf1463af fixes from review 2019-02-01 17:09:37 -03:00
Victor Shyba ca4a55ef28 set verified event earlier and remove stop awaits 2019-02-01 16:05:14 -03:00
Victor Shyba 832be0758b let cancel propagate 2019-02-01 16:05:14 -03:00
Victor Shyba 25050fdeeb fixes for writers (check inside lock and dont hold verified bytes) 2019-02-01 16:05:14 -03:00
Jack Robison df0635103e Merge pull request #1839 from lbryio/faster-stream-manager-startup
faster stream manager startup
2019-02-01 14:00:19 -05:00
Jack Robison dabd9cbe1e faster stream manager startup 2019-02-01 13:53:03 -05:00
Jack Robison 42d7fbdbb3 Merge pull request #1838 from lbryio/fix-update-claim-missing-stream
Fix publishing an update where we don't have the previous stream
2019-02-01 13:52:34 -05:00
Jack Robison 2336015f9a fix publish
fixes https://github.com/lbryio/lbry/issues/1826
- remove `sources` argument from `publish`
- only add/update files we have
- delete existing stream if updating with a new one
2019-02-01 13:35:18 -05:00
Jack Robison 5a28128b69 Merge pull request #1835 from lbryio/remove-peer-manager-lock
Remove peer manager lock
2019-01-31 21:04:21 -05:00
Jack Robison 31445c7797 remove lock from PeerManager 2019-01-31 20:57:48 -05:00
Jack Robison 16d0ff8376 Merge pull request #1833 from lbryio/fix-publish-update
Fix publish when making an update with an existing stream
2019-01-31 20:56:34 -05:00
Jack Robison 87f5d1bd4a fix updating claim with existing stream 2019-01-31 20:49:19 -05:00
Jack Robison be6e8744b0 Merge pull request #1834 from lbryio/remove-blob-request-lock
remove blob request asyncio.Lock
2019-01-31 20:46:01 -05:00
Jack Robison 2e9acd1baa remove blob request asyncio.Lock 2019-01-31 20:19:50 -05:00
Jack Robison 607ccefe90 Merge pull request #1831 from lbryio/cleanup-downloader
Download and publish fixes
2019-01-31 18:17:26 -05:00
Jack Robison df86937f4e fix deleting a file mid download 2019-01-31 17:59:42 -05:00
Jack Robison 957d0290df add torba logging 2019-01-31 16:53:54 -05:00
Jack Robison 40b90ce79b fix hanging download 2019-01-31 16:53:27 -05:00
Jack Robison aa7f0845cc catch sendfile errors 2019-01-31 16:10:27 -05:00
Jack Robison 6e69d306de tests 2019-01-31 15:39:23 -05:00
Jack Robison d85d3dfaa4 increase default max_connections_per_download 2019-01-31 15:39:20 -05:00
Jack Robison 2b035009ef handle both urls and ip addresses for fixed nodes and reflector servers 2019-01-31 15:39:20 -05:00
Jack Robison f9fd62c214 add fixed peers immediately if the dht has no peers 2019-01-31 15:39:20 -05:00
Jack Robison e8a1953815 add fixed peers immediately if dht is skipped 2019-01-31 15:39:20 -05:00
Jack Robison 34776e7eb9 fix test 2019-01-31 15:39:20 -05:00
Jack Robison a4419f562f pylint 2019-01-31 15:39:20 -05:00
Jack Robison c0160ab4e9 update time_to_first_byte.py 2019-01-31 15:39:20 -05:00
Jack Robison e60e5b3919 fix updating content claim for a file
-fix file name and download dir not being hex encoded during publish, add repair script
2019-01-31 15:39:20 -05:00
Jack Robison 3589cc9977 call update_reflected_stream upon reflector finishing 2019-01-31 15:39:20 -05:00
Jack Robison 2e978c00b2 prioritize peers to download from 2019-01-31 15:39:20 -05:00
Jack Robison 2335243de9 pylint 2019-01-31 15:39:20 -05:00
Jack Robison 6ad68eabd9 tests 2019-01-31 15:39:20 -05:00
Jack Robison ca5c638124 tests 2019-01-31 15:39:20 -05:00
Jack Robison 262b9a624b update file_reflect 2019-01-31 15:39:20 -05:00
Jack Robison 05915c540a set fully reflected if the server indicates it had all of the stream 2019-01-31 15:39:20 -05:00
Jack Robison 6aef6a80b7 refactor downloader
split peer accumulation out, use Queues instead of locks
2019-01-31 15:39:20 -05:00
Jack Robison 16efe9ba95 add fixed_peer_delay setting 2019-01-31 15:39:20 -05:00
Jack Robison fcda9500fe add accumulate_peers to Node
simplifies collecting peers during a download by giving a queue of hashes to search for and a queue or peers as they are found
2019-01-31 15:39:20 -05:00
Alex Grin 64dec821b7 Update README.md 2019-01-31 14:24:44 -05:00
Jack Robison d5f6e58aad Update README.md 2019-01-31 14:15:38 -05:00
Jack Robison 087b65522e Merge pull request #1830 from lbryio/dht_fixes
Dht fixes
2019-01-31 13:52:00 -05:00
Victor Shyba 5ee4b2173a fix probes bugs, partial DHT refactor 2019-01-31 01:49:52 -03:00
Victor Shyba 3e1c3d1e4b use node_id parameter instead of your own 2019-01-31 01:11:09 -03:00
Victor Shyba 21f2b2dcd7 cast timestamp to integer before querying 2019-01-31 01:09:57 -03:00
Jack Robison ffc7fc0793 fix uncaught exception in BlobServerProtocol when client disconnects during a blob transfer 2019-01-30 12:03:05 -05:00
Jack Robison 15dbe5e54e Merge pull request #1828 from lbryio/fix_chunked
fix chunked requests for server
2019-01-30 10:25:30 -05:00
Victor Shyba 2e633c0ee6 fix chunked requests for server 2019-01-30 12:11:05 -03:00
Jack Robison 933d58d49b logging 2019-01-29 20:47:02 -05:00
Jack Robison 8f7cf0b38f verbose logging 2019-01-29 13:08:11 -05:00
Jack Robison adf378b6e9 Merge pull request #1825 from lbryio/fix-hanging-download
Fix hanging download
2019-01-28 23:44:42 -05:00
Jack Robison 1a4f083e76 skip test 2019-01-28 22:50:06 -05:00
Jack Robison 7bde09dcf8 add download_blob_from_peer.py 2019-01-28 22:41:16 -05:00
Jack Robison 4630fd38aa fix async generator junction tests 2019-01-28 22:39:36 -05:00
Jack Robison 418def56ac fix log 2019-01-28 22:38:17 -05:00
Jack Robison cefe3eb520 fix file_delete 2019-01-28 22:01:43 -05:00
Jack Robison 64dffa306f logging, cleanup 2019-01-28 22:01:43 -05:00
Jack Robison 7d33b4f1f3 fix unawaited task 2019-01-28 22:01:43 -05:00
Jack Robison f507d95198 always add fixed peers if configured after 2s 2019-01-28 22:01:43 -05:00
Jack Robison 2a7d80f8b4 logging 2019-01-28 22:01:43 -05:00
Jack Robison 305034ffb2 Merge pull request #1824 from lbryio/fix-ping-timeout
fix timeout error
2019-01-28 21:53:32 -05:00
Jack Robison bfaebf0085 fix timeout error 2019-01-28 17:20:14 -05:00
Jack Robison a94cf04e87 Merge pull request #1822 from lbryio/update-settings
Update settings
2019-01-28 17:17:56 -05:00
Jack Robison 71f9f8ae9c fix race condition in reflector server 2019-01-28 16:51:11 -05:00
Jack Robison 330862e487 fix https://github.com/lbryio/lbry/issues/1297 2019-01-28 16:00:37 -05:00
Jack Robison 778d3826ab clean up settings and use them 2019-01-28 15:51:27 -05:00
Jack Robison 4a11cf007f remove unused settings and constants 2019-01-27 19:51:44 -05:00
Jack Robison f6e7700c4c Merge pull request #1818 from lbryio/fix-find-value
fix error encoding find value response
2019-01-26 17:26:23 -05:00
Jack Robison 573285a5bb fix error encoding find value response 2019-01-26 17:04:11 -05:00
173 changed files with 11882 additions and 7731 deletions

No files matched your search

+1
View File
@@ -3,6 +3,7 @@
/.tox
/.idea
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
+1 -3
View File
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
miniupnpc,
unqlite
# extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
+2 -14
View File
@@ -9,9 +9,7 @@ jobs:
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
@@ -106,16 +104,6 @@ jobs:
before_install:
- brew upgrade python || true
- brew upgrade python || true
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
@@ -124,7 +112,7 @@ jobs:
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2018 LBRY Inc
Copyright (c) 2015-2019 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
+4 -4
View File
@@ -1,14 +1,14 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
+46 -262
View File
@@ -76,7 +76,7 @@
},
{
"name": "account_decrypt",
"description": "Decrypt an encrypted account, this will remove the wallet password",
"description": "Decrypt an encrypted account, this will remove the wallet password. The account must be unlocked to decrypt it",
"arguments": [
{
"name": "account_id",
@@ -376,31 +376,6 @@
],
"returns": "(bool) true if successful"
},
{
"name": "blob_availability",
"description": "Get blob availability",
"arguments": [
{
"name": "blob_hash",
"type": "str",
"description": "check availability for this blob hash",
"is_required": true
},
{
"name": "search_timeout",
"type": "int",
"description": "how long to search for peers for the blob in the dht",
"is_required": false
},
{
"name": "blob_timeout",
"type": "int",
"description": "how long to try downloading from a peer",
"is_required": false
}
],
"returns": "(dict) {\n \"is_available\": <bool, true if blob is available from a peer from peer list>\n \"reachable_peers\": [\"<ip>:<port>\"],\n \"unreachable_peers\": [\"<ip>:<port>\"]\n }"
},
{
"name": "blob_delete",
"description": "Delete a blob",
@@ -429,18 +404,6 @@
"type": "int",
"description": "timeout in number of seconds",
"is_required": false
},
{
"name": "encoding",
"type": "str",
"description": "by default no attempt at decoding is made, can be set to one of the following decoders: 'json'",
"is_required": false
},
{
"name": "payment_rate_manager",
"type": "str",
"description": "if not given the default payment rate manager will be used. supported alternative rate managers: 'only-free'",
"is_required": false
}
],
"returns": "(str) Success/Fail message or (dict) decoded data"
@@ -820,12 +783,6 @@
],
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
},
{
"name": "commands",
"description": "Return a list of available commands",
"arguments": [],
"returns": "(list) list of available commands"
},
{
"name": "file_delete",
"description": "Delete a LBRY file",
@@ -975,14 +932,32 @@
"description": "get file with matching claim name",
"is_required": false
},
{
"name": "blobs_in_stream<blobs_in_stream>",
"type": "int",
"description": "get file with matching blobs in stream",
"is_required": false
},
{
"name": "blobs_remaining",
"type": "int",
"description": "amount of remaining blobs to download",
"is_required": false
},
{
"name": "sort",
"type": "str",
"description": "sort by any property, like 'file_name' or 'metadata.author'; to specify direction append ',asc' or ',desc'",
"description": "field to sort by (one of the above filter fields)",
"is_required": false
},
{
"name": "comparison",
"type": "str",
"description": "logical comparision, (eq | ne | g | ge | l | le)",
"is_required": false
}
],
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]"
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes_lower_bound': (int) lower bound file size in bytes,\n 'total_bytes': (int) file upper bound size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'blobs_remaining': (int) total blobs remaining to download,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]\n}"
},
{
"name": "file_reflect",
@@ -1083,19 +1058,6 @@
],
"returns": "(dict) Dictionary containing information about the stream\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status,\n 'claim_id': (str) claim id,\n 'outpoint': (str) claim outpoint string,\n 'txid': (str) claim txid,\n 'nout': (int) claim nout,\n 'metadata': (dict) claim metadata,\n 'channel_claim_id': (str) None if claim is not signed\n 'channel_name': (str) None if claim is not signed\n 'claim_name': (str) claim name\n }"
},
{
"name": "help",
"description": "Return a useful message for an API command",
"arguments": [
{
"name": "command",
"type": "str",
"description": "command to retrieve documentation for",
"is_required": false
}
],
"returns": "(str) Help message"
},
{
"name": "peer_list",
"description": "Get peers for blob hash",
@@ -1107,31 +1069,18 @@
"is_required": true
},
{
"name": "timeout",
"name": "search_bottom_out_limit",
"type": "int",
"description": "peer search timeout in seconds",
"description": "the number of search probes in a row that don't find any new peers before giving up and returning",
"is_required": false
}
],
"returns": "(list) List of contact dictionaries {'host': <peer ip>, 'port': <peer port>, 'node_id': <peer node id>}"
"returns": "(list) List of contact dictionaries {'address': <peer ip>, 'udp_port': <dht port>, 'tcp_port': <peer port>,\n 'node_id': <peer node id>}"
},
{
"name": "peer_ping",
"description": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.",
"arguments": [
{
"name": "address",
"type": "str",
"description": "ip address of the peer",
"is_required": false
},
{
"name": "port",
"type": "int",
"description": "udp port of the peer",
"is_required": false
}
],
"arguments": [],
"returns": "(str) pong, or {'error': <error message>} if an error is encountered"
},
{
@@ -1222,12 +1171,6 @@
"description": "whether the content is nsfw",
"is_required": false
},
{
"name": "sources",
"type": "str",
"description": "{'lbry_sd_hash': sd_hash} specifies sd hash of file",
"is_required": false
},
{
"name": "channel_name",
"type": "str",
@@ -1276,53 +1219,22 @@
},
{
"name": "resolve",
"description": "Resolve given LBRY URIs",
"description": "Get the claim that a URL refers to.",
"arguments": [
{
"name": "force",
"type": "bool",
"description": "force refresh and ignore cache",
"is_required": false
},
{
"name": "uri",
"type": "str",
"description": "uri to resolve",
"is_required": true
},
{
"name": "uris",
"type": "list",
"description": "uris to resolve",
"name": "urls",
"type": "str, list",
"description": "one or more urls to resolve",
"is_required": false
}
],
"returns": "Dictionary of results, keyed by uri\n '<uri>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
},
{
"name": "resolve_name",
"description": "Resolve stream info from a LBRY name",
"arguments": [
{
"name": "name",
"type": "str",
"description": "the name to resolve",
"is_required": true
},
{
"name": "force",
"type": "bool",
"description": "force refresh and do not check cache",
"is_required": false
}
],
"returns": "(dict) Metadata dictionary from name claim, None if the name is not\n resolvable"
"returns": "Dictionary of results, keyed by url\n '<url>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the url resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the url resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the url resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
},
{
"name": "routing_table_get",
"description": "Get DHT routing information",
"arguments": [],
"returns": "(dict) dictionary containing routing and contact information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"port\": (int) peer udp port\n \"node_id\": (str) peer node id,\n \"blobs\": (list) blob hashes announced by peer\n }\n ]\n },\n \"contacts\": (list) contact node ids,\n \"blob_hashes\": (list) all of the blob hashes stored by peers in the list of buckets,\n \"node_id\": (str) the local dht node id\n }"
"returns": "(dict) dictionary containing routing and peer information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"udp_port\": (int) peer udp port,\n \"tcp_port\": (int) peer tcp port,\n \"node_id\": (str) peer node id,\n }\n ]\n },\n \"node_id\": (str) the local dht node id\n }"
},
{
"name": "settings_get",
@@ -1333,131 +1245,21 @@
{
"name": "settings_set",
"description": "Set daemon settings",
"arguments": [
{
"name": "download_directory",
"type": "str",
"description": "path of download directory",
"is_required": false
},
{
"name": "data_rate",
"type": "float",
"description": "0.0001",
"is_required": false
},
{
"name": "download_timeout",
"type": "int",
"description": "180",
"is_required": false
},
{
"name": "peer_port",
"type": "int",
"description": "3333",
"is_required": false
},
{
"name": "max_key_fee",
"type": "dict",
"description": "maximum key fee for downloads, in the format: { 'currency': <currency_symbol>, 'amount': <amount> }. In the CLI, it must be an escaped JSON string Supported currency symbols: LBC, USD, BTC",
"is_required": false
},
{
"name": "disable_max_key_fee",
"type": "bool",
"description": "False",
"is_required": false
},
{
"name": "use_upnp",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "run_reflector_server",
"type": "bool",
"description": "False",
"is_required": false
},
{
"name": "cache_time",
"type": "int",
"description": "150",
"is_required": false
},
{
"name": "reflect_uploads",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "share_usage_data",
"type": "bool",
"description": "True",
"is_required": false
},
{
"name": "peer_search_timeout",
"type": "int",
"description": "3",
"is_required": false
},
{
"name": "sd_download_timeout",
"type": "int",
"description": "3",
"is_required": false
},
{
"name": "auto_renew_claim_height_delta",
"type": "int",
"description": "0 claims set to expire within this many blocks will be automatically renewed after startup (if set to 0, renews will not be made automatically)",
"is_required": false
}
],
"arguments": [],
"returns": "(dict) Updated dictionary of daemon settings"
},
{
"name": "status",
"description": "Get daemon status",
"arguments": [],
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'database': (bool),\n 'dht': (bool),\n 'exchange_rate_manager': (bool),\n 'hash_announcer': (bool),\n 'peer_protocol_server': (bool),\n 'stream_manager': (bool),\n 'upnp': (bool),\n 'wallet': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'stream_manager': {\n 'managed_files': (int) count of files in the stream manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
},
{
"name": "stop",
"description": "Stop lbrynet",
"description": "Stop lbrynet API server.",
"arguments": [],
"returns": "(string) Shutdown message"
},
{
"name": "stream_availability",
"description": "Get stream availability for lbry uri",
"arguments": [
{
"name": "uri",
"type": "str",
"description": "check availability for this uri",
"is_required": true
},
{
"name": "search_timeout",
"type": "int",
"description": "how long to search for peers for the blob in the dht",
"is_required": false
},
{
"name": "blob_timeout",
"type": "int",
"description": "how long to try downloading from a peer",
"is_required": false
}
],
"returns": "(dict) {\n 'is_available': <bool>,\n 'did_decode': <bool>,\n 'did_resolve': <bool>,\n 'is_stream': <bool>,\n 'num_blobs_in_stream': <int>,\n 'sd_hash': <str>,\n 'sd_blob_availability': <dict> see `blob_availability`,\n 'head_blob_hash': <str>,\n 'head_blob_availability': <dict> see `blob_availability`,\n 'use_upnp': <bool>,\n 'upnp_redirect_is_set': <bool>,\n 'error': <None> | <str> error message\n }"
},
{
"name": "stream_cost_estimate",
"description": "Get estimated cost for a lbry stream",
@@ -1467,12 +1269,6 @@
"type": "str",
"description": "uri to use",
"is_required": true
},
{
"name": "size",
"type": "float",
"description": "stream size in bytes. if provided an sd blob won't be downloaded.",
"is_required": false
}
],
"returns": "(float) Estimated cost in lbry credits, returns None if uri is not\n resolvable"
@@ -1541,35 +1337,23 @@
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
},
{
"name": "version",
"description": "Get lbry version information",
"arguments": [],
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
},
{
"name": "wallet_prefill_addresses",
"description": "Create new UTXOs, each containing `amount` credits",
"name": "utxo_release",
"description": "When spending a UTXO it is locally locked to prevent double spends;\noccasionally this can result in a UTXO being locked which ultimately\ndid not get spent (failed to broadcast, spend transaction was not\naccepted by blockchain node, etc). This command releases the lock\non all UTXOs in your account.",
"arguments": [
{
"name": "no_broadcast",
"type": "bool",
"description": "whether to broadcast or not",
"name": "account_id",
"type": "str",
"description": "id of the account to query",
"is_required": false
},
{
"name": "num_addresses",
"type": "int",
"description": "num of addresses to create",
"is_required": true
},
{
"name": "amount",
"type": "decimal",
"description": "initial amount in each address",
"is_required": true
}
],
"returns": "(dict) the resulting transaction"
"returns": "None"
},
{
"name": "version",
"description": "Get lbrynet API server version information",
"arguments": [],
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
},
{
"name": "wallet_send",
+2 -4
View File
@@ -1409,7 +1409,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1954,7 +1954,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1979,7 +1978,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2586,4 +2584,4 @@ Returns:
</body>
</html>
</html>
+2 -4
View File
@@ -1339,7 +1339,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1786,7 +1786,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1811,7 +1810,6 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2312,4 +2310,4 @@ Returns:
</body>
</html>
</html>
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.31.0rc3"
__version__ = "0.35.0"
version = tuple(__version__.split('.'))
+31 -21
View File
@@ -1,4 +1,5 @@
import os
import re
import asyncio
import binascii
import logging
@@ -17,8 +18,11 @@ from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
_hexmatch = re.compile("^[a-f,0-9]+$")
def is_valid_hashcharacter(char: str) -> bool:
return char in "0123456789abcdef"
return len(char) == 1 and _hexmatch.match(char)
def is_valid_blobhash(blobhash: str) -> bool:
@@ -29,7 +33,7 @@ def is_valid_blobhash(blobhash: str) -> bool:
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
@@ -66,18 +70,22 @@ class BlobFile:
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=loop)
self.blob_write_lock = asyncio.Lock(loop=loop)
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
if self.file_exists:
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
self.length = length
self.verified.set()
self.finished_writing.set()
self.saved_verified_blob = False
self.blob_completed_callback = blob_completed_callback
@property
def file_exists(self):
return os.path.isfile(self.file_path)
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
try:
error = finished.result()
error = finished.exception()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
@@ -86,37 +94,37 @@ class BlobFile:
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer))
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
log.exception("something else")
raise error
return callback
async def save_verified_blob(self, writer):
async def save_verified_blob(self, writer, verified_bytes: bytes):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(writer.verified_bytes):
if self.get_length() == len(verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(writer.verified_bytes)
write_handle.write(verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
if self.verified.is_set():
return
async with self.blob_write_lock:
if self.verified.is_set():
return
await self.loop.run_in_executor(None, _save_verified)
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if os.path.exists(self.file_path):
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
@@ -130,18 +138,20 @@ class BlobFile:
"""
with open(self.file_path, 'rb') as handle:
return await self.loop.sendfile(writer.transport, handle)
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
async def close(self):
def close(self):
while self.writers:
self.writers.pop().finished.cancel()
async def delete(self):
await self.close()
async with self.blob_write_lock:
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
def delete(self):
self.close()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
self.verified.clear()
self.finished_writing.clear()
self.length = None
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
+31 -30
View File
@@ -1,9 +1,9 @@
import os
import typing
import asyncio
import logging
from sqlite3 import IntegrityError
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
@@ -30,10 +30,21 @@ class BlobFileManager:
self.blobs: typing.Dict[str, BlobFile] = {}
async def setup(self) -> bool:
raw_blob_hashes = await self.get_all_verified_blobs()
self.completed_blob_hashes.update(raw_blob_hashes)
def get_files_in_blob_dir() -> typing.Set[str]:
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
return True
def stop(self):
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
self.completed_blob_hashes.clear()
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if length and self.blobs[blob_hash].length is None:
@@ -52,38 +63,28 @@ class BlobFileManager:
raise Exception("Blob has a length of 0")
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
blobs = [self.get_blob(b) for b in blob_hashes]
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
async def set_should_announce(self, blob_hash: str, should_announce: bool):
now = self.loop.time()
return await self.storage.set_should_announce(blob_hash, now, should_announce)
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
async def get_all_verified_blobs(self) -> typing.List[str]:
blob_hashes = await self.storage.get_all_blob_hashes()
return self.check_completed_blobs(blob_hashes)
async def delete_blobs(self, blob_hashes: typing.List[str]):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
if not blob_hash:
continue
try:
blob = self.get_blob(blob_hash)
await blob.delete()
bh_to_delete_from_db.append(blob_hash)
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
if blob_hash not in self.blobs:
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
if blob_hash in self.blobs:
del self.blobs[blob_hash]
try:
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if str(err) != "FOREIGN KEY constraint failed":
raise err
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
for blob_hash in blob_hashes:
self.delete_blob(blob_hash)
if delete_from_db:
await self.storage.delete_blobs_from_db(blob_hashes)
+4 -8
View File
@@ -18,7 +18,6 @@ class HashBlobWriter:
self.finished.add_done_callback(lambda *_: self.close_handle())
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.verified_bytes = b''
def __del__(self):
if self.buffer is not None:
@@ -46,7 +45,7 @@ class HashBlobWriter:
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_result(InvalidDataError(
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
return
@@ -55,15 +54,12 @@ class HashBlobWriter:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_result(InvalidBlobHashError(
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
return
self.buffer.seek(0)
self.verified_bytes = self.buffer.read()
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(self.buffer.getvalue())
self.close_handle()
if self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(None)
def close_handle(self):
if self.buffer is not None:
+116 -80
View File
@@ -1,6 +1,8 @@
import asyncio
import logging
import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
@@ -19,150 +21,184 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.download_running = asyncio.Event(loop=self.loop)
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._request_lock = asyncio.Lock(loop=self.loop)
self.buf = b''
def handle_data_received(self, data: bytes):
if self.transport.is_closing():
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
def data_received(self, data: bytes):
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
response = BlobResponse.deserialize(data)
response = BlobResponse.deserialize(self.buf + data)
if not response.responses and not self._response_fut.done():
self.buf += data
return
else:
self.buf = b''
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
self._blob_bytes_received += len(response.blob_data)
try:
self.writer.write(response.blob_data)
except IOError as err:
log.error("error downloading blob: %s", err)
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def data_received(self, data):
def _write(self, data):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
else:
data = data
self._blob_bytes_received += len(data)
try:
return self.handle_data_received(data)
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
except (asyncio.TimeoutError) as err: # TODO: is this needed?
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[bool, bool]:
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
try:
self.transport.write(request.serialize())
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("blob not in availability response")
return False, True
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request")
return False, False
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected")
return False, False
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from this peer")
return False, True
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch")
return False, False
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length")
return False, False
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.info(msg)
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
await self.blob.finished_writing.wait()
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
log.info(msg)
return True, True
except asyncio.CancelledError:
return False, True
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
return False, False
finally:
await self.close()
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
async def close(self):
def close(self):
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
if self.blob:
await self.blob.close()
self.download_running.clear()
self._response_fut = None
self.writer = None
self.blob = None
if self.transport:
self.transport.close()
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
if blob.get_is_verified():
return False, True
async with self._request_lock:
try:
if self.download_running.is_set():
log.info("wait for download already running")
await self.download_running.wait()
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self.download_running.set()
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError:
log.error("race happened")
# i'm not sure how to fix this race condition - jack
return False, True
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, False
except asyncio.CancelledError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, True
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
return 0, self.transport
try:
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
# i'm not sure how to fix this race condition - jack
log.exception(e)
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
self.close()
return self._blob_bytes_received, None
except asyncio.CancelledError:
self.close()
raise
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
# log.info("connection made to %s: %s", self.peer_address, transport)
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
self.transport = None
self.loop.create_task(self.close())
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
if blob.get_is_verified():
return False, True
if blob.get_is_verified():
log.info("already verified")
return False, True
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.connection_made(connected_transport)
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
try:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or blob.file_exists:
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return False, False
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return 0, None
+123 -111
View File
@@ -1,10 +1,10 @@
import asyncio
import typing
import logging
from lbrynet import conf
from lbrynet.utils import drain_tasks
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
@@ -13,120 +13,132 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
def drain_into(a: list, b: list):
while a:
b.append(a.pop())
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
"""A single blob downloader"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
class BlobDownloader:
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.lock = asyncio.Lock(loop=self.loop)
self.blob: 'BlobFile' = None
self.blob_queue = asyncio.Queue(loop=self.loop)
self.peer_queue = peer_queue
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.time_since_last_blob = loop.time()
self.blob_download_timeout = config.get('blob_download_timeout')
self.peer_connect_timeout = config.get('peer_connect_timeout')
self.max_connections = config.get('max_connections_per_stream')
def should_race_continue(self, blob: 'BlobFile'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
# the safe net side is that any failure will reset the peer score, triggering the race back
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
# for peer, task in self.active_connections.items():
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
# return False
return not (blob.get_is_verified() or blob.file_exists)
async def _request_blob(self, peer: 'KademliaPeer'):
if self.blob.get_is_verified():
log.info("already verified")
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
if blob.get_is_verified():
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self.lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport
)
if bytes_received == blob.get_length():
self.time_since_last_blob = self.loop.time()
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.connections[peer] = transport
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
self.scores[peer] = rough_speed
def _update_requests(self):
self.new_peer_event.clear()
if self.blob.blob_hash not in self.requested_from:
self.requested_from[self.blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.blob.blob_hash][peer] = task
self.running_download_requests.append(task)
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
async def _accumulate_connections(self, node: 'Node'):
async def new_peer_or_finished(self, blob: 'BlobFile'):
async def get_and_re_add_peers():
try:
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
self.peer_queue.put_nowait(new_peers)
except asyncio.TimeoutError:
pass
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
active_tasks = list(self.active_connections.values())
try:
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
return
except asyncio.CancelledError:
pass
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
self.blob = self.blob_manager.get_blob(blob_hash)
if self.blob.get_is_verified():
return self.blob
accumulator = self.loop.create_task(self._accumulate_connections(node))
self.blob_queue.put_nowait(blob_hash)
try:
while not self.blob.get_is_verified():
if len(self.running_download_requests) < self.max_connections:
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
if self.blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
finally:
if accumulator and not accumulator.done():
accumulator.cancel()
drain_tasks(tasks)
def cleanup_active(self):
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
def clearbanned(self):
now = self.loop.time()
if now - self.time_since_last_blob > 60.0:
return
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
self.peer_queue.put_nowait(forgiven)
for banned_peer in forgiven:
self.ignored.pop(banned_peer)
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
try:
while not blob.get_is_verified():
batch: typing.List['KademliaPeer'] = []
while not self.peer_queue.empty():
batch.extend(self.peer_queue.get_nowait())
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in batch:
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
self.active_connections[peer] = t
await self.new_peer_or_finished(blob)
self.cleanup_active()
if batch:
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
else:
self.clearbanned()
blob.close()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
while self.active_connections:
self.active_connections.popitem()[1].cancel()
def close(self):
self.scores.clear()
self.ignored.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
blob_hash: str) -> 'BlobFile':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try:
return await downloader.download_blob(blob_hash)
finally:
if accumulate_task and not accumulate_task.done():
accumulate_task.cancel()
downloader.close()
+12 -18
View File
@@ -146,32 +146,26 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
# <blob bytes>
# <json><blob bytes>
extra_data = b''
response = None
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
break
return None, response_msg
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
if not isinstance(response, dict):
raise ValueError()
for key in response.keys():
if key not in [
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key]:
raise ValueError()
extra_data = response_msg[curr_pos:]
break
except ValueError:
response = None
if response is None:
extra_data = response_msg
return response, extra_data
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
return response, response_msg[curr_pos:]
return None, response_msg
class BlobRequest:
+10 -4
View File
@@ -57,8 +57,13 @@ class BlobServerProtocol(asyncio.Protocol):
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
sent = await blob.sendfile(self)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
if self.transport:
self.transport.close()
return
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
@@ -72,7 +77,7 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf += data
return
try:
request = BlobRequest.deserialize(data)
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
@@ -95,6 +100,7 @@ class BlobServer:
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -102,7 +108,7 @@ class BlobServer:
async def _start_server():
server = await self.loop.create_server(
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
+71 -68
View File
@@ -16,33 +16,11 @@ log = logging.getLogger(__name__)
NOT_SET = type(str('NOT_SET'), (object,), {})
T = typing.TypeVar('T')
KB = 2 ** 10
MB = 2 ** 20
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
API_ADDRESS = 'lbryapi'
APP_NAME = 'LBRY'
BLOBFILES_DIR = 'blobfiles'
CRYPTSD_FILE_EXTENSION = '.cryptsd'
CURRENCIES = {
'BTC': {'type': 'crypto'},
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
LOG_FILE_NAME = 'lbrynet.log'
LOG_POST_URL = 'https://lbry.io/log-upload'
MAX_BLOB_REQUEST_SIZE = 64 * KB
MAX_HANDSHAKE_SIZE = 64 * KB
MAX_REQUEST_SIZE = 64 * KB
MAX_RESPONSE_INFO_SIZE = 64 * KB
MAX_BLOB_INFOS_TO_REQUEST = 20
PROTOCOL_PREFIX = 'lbry'
SLACK_WEBHOOK = (
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
)
HEADERS_FILE_SHA256_CHECKSUM = (
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
)
@@ -471,7 +449,7 @@ class CLIConfig(BaseConfig):
class Config(CLIConfig):
# directories
data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path(
"Directory path to place assembled files downloaded from LBRY.",
@@ -482,60 +460,85 @@ class Config(CLIConfig):
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
# network
use_upnp = Toggle(
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
"ports or have firewall rules you likely want to disable this.", True
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# routing table
split_buckets_under_index = Integer(
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
"use.", 1
)
# claims set to expire within this many blocks will be
# automatically renewed after startup (if set to 0, renews
# will not be made automatically)
auto_renew_claim_height_delta = Integer("", 0)
cache_time = Integer("", 150)
data_rate = Float("points/megabyte", .0001)
delete_blobs_on_remove = Toggle("", True)
dht_node_port = Integer("", 4444)
download_timeout = Float("", 30.0)
blob_download_timeout = Float("", 20.0)
peer_connect_timeout = Float("", 3.0)
node_rpc_timeout = Float("", constants.rpc_timeout)
is_generous_host = Toggle("", True)
announce_head_blobs_only = Toggle("", True)
concurrent_announcers = Integer("", 10)
known_dht_nodes = Servers("", [
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
)
concurrent_blob_announcers = Integer(
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 8,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
)
# reflector settings
reflect_streams = Toggle(
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
previous_names=['reflect_uploads']
)
concurrent_reflector_uploads = Integer(
"Maximum number of streams to upload to a reflector server at a time", 10
)
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.io', 4444), # US EAST
('lbrynet2.lbry.io', 4444), # US WEST
('lbrynet3.lbry.io', 4444), # EU
('lbrynet4.lbry.io', 4444) # ASIA
])
max_connections_per_stream = Integer("", 5)
seek_head_blob_first = Toggle("", True)
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
min_info_rate = Float("points/1000 infos", .02)
min_valuable_hash_rate = Float("points/1000 infos", .05)
min_valuable_info_rate = Float("points/1000 infos", .05)
peer_port = Integer("", 3333)
pointtrader_server = String("", 'http://127.0.0.1:2424')
reflector_port = Integer("", 5566)
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
reflect_uploads = Toggle("", True)
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
reflector_servers = Servers("", [
('reflector.lbry.io', 5566)
])
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
sd_download_timeout = Integer("", 3)
peer_search_timeout = Integer("", 60)
use_upnp = Toggle("", True)
use_keyring = Toggle("", False)
blockchain_name = String("", 'lbrycrd_main')
lbryum_servers = Servers("", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
def __init__(self, **kwargs):
super().__init__(**kwargs)
+3 -1
View File
@@ -19,6 +19,8 @@ class BlobAnnouncer:
self.announce_queue: typing.List[str] = []
async def _announce(self, batch_size: typing.Optional[int] = 10):
if not batch_size:
return
if not self.node.joined.is_set():
await self.node.joined.wait()
blob_hashes = await self.storage.get_blobs_to_announce()
@@ -39,7 +41,7 @@ class BlobAnnouncer:
to_await.append(batch.pop())
if to_await:
await asyncio.gather(*tuple(to_await), loop=self.loop)
await self.storage.update_last_announced_blobs(announced, self.loop.time())
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
if self.running:
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
+2
View File
@@ -6,6 +6,7 @@ hash_length = hash_class().digest_size
hash_bits = hash_length * 8
alpha = 5
k = 8
split_buckets_under_index = 1
replacement_cache_size = 8
rpc_timeout = 5.0
rpc_attempts = 5
@@ -15,6 +16,7 @@ refresh_interval = 3600 # 1 hour
replicate_interval = refresh_interval
data_expiration = 86400 # 24 hours
token_secret_refresh_interval = 300 # 5 minutes
maybe_ping_delay = 300 # 5 minutes
check_refresh_interval = refresh_interval / 5
max_datagram_size = 8192 # 8 KB
rpc_id_length = 20
+62 -41
View File
@@ -1,9 +1,9 @@
import logging
import asyncio
import typing
import socket
import binascii
import contextlib
from lbrynet.utils import resolve_host
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
@@ -20,10 +20,12 @@ log = logging.getLogger(__name__)
class Node:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str):
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
split_buckets_under_index)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
@@ -64,7 +66,7 @@ class Node:
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
@@ -79,10 +81,9 @@ class Node:
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.info("Store to %i peers", len(peers))
log.info(peers)
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
@@ -116,39 +117,50 @@ class Node:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
known_node_addresses = known_node_addresses or []
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
if known_node_urls:
for host, port in known_node_urls:
info = await self.loop.getaddrinfo(
host, 'https',
proto=socket.IPPROTO_TCP,
)
if (info[0][4][0], port) not in known_node_addresses:
known_node_addresses.append((info[0][4][0], port))
futs = []
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
futs.append(peer.ping())
if futs:
await asyncio.wait(futs, loop=self.loop)
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
known_node_addresses.append((address, port))
url_to_addr[address] = host
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
if known_node_addresses:
while not self.protocol.routing_table.get_peers():
success = False
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
self.joined.set()
await peer.ping()
success = True
except asyncio.TimeoutError:
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
round(self.protocol.rpc_timeout, 2))
if success:
break
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
# in the routing table with good peers who are near us
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
@@ -178,10 +190,7 @@ class Node:
async def _add_hashes_from_queue():
while True:
try:
blob_hash = await hash_queue.get()
except asyncio.CancelledError:
break
blob_hash = await hash_queue.get()
peer_generator.add_generator(
self.get_iterative_value_finder(
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
@@ -193,10 +202,6 @@ class Node:
async with peer_generator as junction:
yield junction
await peer_generator.finished.wait()
except asyncio.CancelledError:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
raise
finally:
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
add_hashes_task.cancel()
@@ -214,13 +219,29 @@ class Node:
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20) -> typing.List['KademliaPeer']:
accumulated: typing.List['KademliaPeer'] = []
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
async with self.peer_search_junction(node_id, max_results=max_results,
bottom_out_limit=bottom_out_limit) as junction:
async for peers in junction:
log.info("peer search: %s", peers)
accumulated.extend(peers)
log.info("junction done")
log.info("context done")
distance = Distance(node_id)
accumulated.sort(key=lambda peer: distance(peer.node_id))
return accumulated[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
async for peers in search_junction:
if peers:
result_queue.put_nowait([
peer for peer in peers
if not (
peer.address == self.protocol.external_ip
and peer.tcp_port == self.protocol.peer_port
)
])
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue()
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
+36 -54
View File
@@ -3,6 +3,8 @@ import asyncio
import logging
import ipaddress
from binascii import hexlify
from functools import lru_cache
from lbrynet.dht import constants
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
@@ -30,37 +32,30 @@ class PeerManager:
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
self._lock = asyncio.Lock(loop=loop)
async def report_failure(self, address: str, udp_port: int):
def report_failure(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
async def report_last_sent(self, address: str, udp_port: int):
def report_last_sent(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_sent[(address, udp_port)] = now
self._last_sent[(address, udp_port)] = now
async def report_last_replied(self, address: str, udp_port: int):
def report_last_replied(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_replied[(address, udp_port)] = now
self._last_replied[(address, udp_port)] = now
async def report_last_requested(self, address: str, udp_port: int):
def report_last_requested(self, address: str, udp_port: int):
now = self._loop.time()
async with self._lock:
self._last_requested[(address, udp_port)] = now
self._last_requested[(address, udp_port)] = now
async def clear_token(self, node_id: bytes):
async with self._lock:
self._node_tokens.pop(node_id, None)
def clear_token(self, node_id: bytes):
self._node_tokens.pop(node_id, None)
async def update_token(self, node_id: bytes, token: bytes):
def update_token(self, node_id: bytes, token: bytes):
now = self._loop.time()
async with self._lock:
self._node_tokens[node_id] = (now, token)
self._node_tokens[node_id] = (now, token)
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
ts, token = self._node_tokens.get(node_id, (None, None))
@@ -70,50 +65,37 @@ class PeerManager:
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
return self._last_replied.get((address, udp_port))
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
return self._node_id_mapping.get((address, udp_port))
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
return self._node_id_reverse_mapping.get(node_id)
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
async with self._lock:
addr_tuple = self._node_id_reverse_mapping.get(node_id)
if addr_tuple and addr_tuple in self._node_id_mapping:
return addr_tuple
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
"""
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
This is to handle peers changing addresses and ids while assuring that the we only ever have
one node id / address tuple mapped to each other
"""
async with self._lock:
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
@lru_cache(maxsize=400)
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
return KademliaPeer(self._loop, address, node_id, udp_port)
async def prune(self):
def prune(self): # TODO: periodically call this
now = self._loop.time()
async with self._lock:
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
"""
@@ -13,11 +13,6 @@ def cancel_task(task: typing.Optional[asyncio.Task]):
task.cancel()
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
for task in tasks:
cancel_task(task)
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
@@ -100,6 +95,6 @@ class AsyncGeneratorJunction:
pass
finally:
if exc_type:
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
err = traceback.format_exception(exc_type, exc, tb)
log.error(err)
+3 -1
View File
@@ -10,11 +10,13 @@ class Distance:
def __init__(self, key: bytes):
if len(key) != constants.hash_length:
raise ValueError("invalid key length: %i" % len(key))
raise ValueError(f"invalid key length: {len(key)}")
self.key = key
self.val_key_one = int.from_bytes(key, 'big')
def __call__(self, key_two: bytes) -> int:
if len(key_two) != constants.hash_length:
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
val_key_two = int.from_bytes(key_two, 'big')
return self.val_key_one ^ val_key_two
+84 -119
View File
@@ -1,7 +1,9 @@
import asyncio
from binascii import hexlify
from itertools import chain
import typing
import logging
from lbrynet.utils import drain_tasks
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.distance import Distance
@@ -90,8 +92,8 @@ class IterativeFinder:
self.exclude = exclude or []
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
self.active: typing.List['KademliaPeer'] = []
self.contacted: typing.List[typing.Tuple[str, int]] = []
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
@@ -99,14 +101,12 @@ class IterativeFinder:
self.iteration_queue = asyncio.Queue(loop=self.loop)
self.running_probes: typing.List[asyncio.Task] = []
self.lock = asyncio.Lock(loop=self.loop)
self.running_probes: typing.Set[asyncio.Task] = set()
self.iteration_count = 0
self.bottom_out_count = 0
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
self.finished = asyncio.Event(loop=self.loop)
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
"""
@@ -114,9 +114,16 @@ class IterativeFinder:
"""
raise NotImplementedError()
def search_exhausted(self):
"""
This method ends the iterator due no more peers to contact.
Override to provide last time results.
"""
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindResponse):
"""
Called with a lock after adding peers from an rpc result to the shortlist.
Called after adding peers from an rpc result to the shortlist.
This method is responsible for putting a result for the generator into the Queue
"""
raise NotImplementedError()
@@ -129,9 +136,7 @@ class IterativeFinder:
return []
def _is_closer(self, peer: 'KademliaPeer') -> bool:
if not self.closest_peer:
return True
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
def _update_closest(self):
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
@@ -141,35 +146,28 @@ class IterativeFinder:
self.closest_peer = self.shortlist[-1]
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
async with self.lock:
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.append(peer)
for contact_triple in response.get_close_triples():
addr_tuple = (contact_triple[1], contact_triple[2])
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(
contact_triple[0], contact_triple[1], contact_triple[2]
)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.add(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
try:
response = await self.send_probe(peer)
except asyncio.CancelledError:
return
except asyncio.TimeoutError:
if peer in self.active:
self.active.remove(peer)
self.active.discard(peer)
return
except ValueError as err:
log.warning(str(err))
if peer in self.active:
self.active.remove(peer)
self.active.discard(peer)
return
except RemoteException:
return
@@ -181,31 +179,35 @@ class IterativeFinder:
"""
added = 0
async with self.lock:
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.append((peer.address, peer.udp_port))
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.add((peer.address, peer.udp_port))
t = self.loop.create_task(self._send_probe(peer))
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
if t and t in self.running_probes:
self.running_probes.remove(t)
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.append(t)
added += 1
t.add_done_callback(callback)
self.running_probes.add(t)
added += 1
log.debug("running %d probes", len(self.running_probes))
if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8])
self.search_exhausted()
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
try:
@@ -215,70 +217,40 @@ class IterativeFinder:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration):
if self.running:
drain_tasks(self.running_probes)
self.running = False
self.loop.call_soon(self.aclose)
def _search(self):
self.tasks.append(self.loop.create_task(self._search_task()))
def search(self):
def __aiter__(self):
if self.running:
raise Exception("already running")
self.running = True
self._search()
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
peers = self.loop.create_task(self.iteration_queue.get())
finished = self.loop.create_task(self.finished.wait())
err = None
try:
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
if peers.done():
return peers.result()
raise StopAsyncIteration()
except asyncio.CancelledError as error:
err = error
finally:
if not finished.done() and not finished.cancelled():
finished.cancel()
if not peers.done() and not peers.cancelled():
peers.cancel()
if err:
raise err
def __aiter__(self):
self.search()
return self
async def __anext__(self) -> typing.List['KademliaPeer']:
try:
if self.iteration_count == 0:
initial_results = self.get_initial_result()
if initial_results:
self.iteration_queue.put_nowait(initial_results)
result = await self.next_queue_or_finished()
result = self.get_initial_result() or await self.iteration_queue.get()
else:
result = await self.iteration_queue.get()
if not result:
raise StopAsyncIteration
self.iteration_count += 1
return result
except (asyncio.CancelledError, StopAsyncIteration):
await self.aclose()
self.loop.call_soon(self.aclose)
raise
def aclose(self):
self.running = False
async def _aclose():
async with self.lock:
self.running = False
if not self.finished.is_set():
self.finished.set()
drain_tasks(self.tasks)
drain_tasks(self.running_probes)
while self.delayed_calls:
timer = self.delayed_calls.pop()
if timer:
timer.cancel()
return asyncio.ensure_future(_aclose(), loop=self.loop)
self.iteration_queue.put_nowait(None)
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
task.cancel()
self.tasks.clear()
self.running_probes.clear()
self.delayed_calls.clear()
class IterativeNodeFinder(IterativeFinder):
@@ -295,24 +267,26 @@ class IterativeNodeFinder(IterativeFinder):
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response)
def put_result(self, from_list: typing.List['KademliaPeer']):
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
def search_exhausted(self):
self.put_result(self.active, finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
if to_yield:
for peer in to_yield:
self.yielded_peers.add(peer)
self.iteration_queue.put_nowait(to_yield)
if finish:
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindNodeResponse):
found = response.found and self.key != self.protocol.node_id
if found:
log.info("found")
self.put_result(self.shortlist)
if not self.finished.is_set():
self.finished.set()
return
log.debug("found")
return self.put_result(self.shortlist, finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
@@ -323,16 +297,10 @@ class IterativeNodeFinder(IterativeFinder):
self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.info("max results")
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
self.put_result(self.active, True)
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.debug("max results")
self.put_result(self.active, True)
class IterativeValueFinder(IterativeFinder):
@@ -366,14 +334,11 @@ class IterativeValueFinder(IterativeFinder):
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
return
if self.prev_closest_peer and self.closest_peer:
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
if not self.finished.is_set():
self.finished.set()
return
self.iteration_queue.put_nowait(None)
def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key):
+104 -103
View File
@@ -77,24 +77,23 @@ class KademliaRPC:
response = {
b'token': self.make_token(rpc_contact.compact_ip()),
b'contacts': self.find_node(rpc_contact, key)
}
if self.protocol.protocol_version:
response[b'protocolVersion'] = self.protocol.protocol_version
# get peers we have stored for this blob_exchange
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
peers = []
if has_other_peers:
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
peers = [
peer.compact_address_tcp()
for peer in self.protocol.data_store.get_peers_for_blob(key)
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
]
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
if peers:
response[key] = peers
else:
response[b'contacts'] = self.find_node(rpc_contact, key)
return response
def refresh_token(self): # TODO: this needs to be called periodically
@@ -180,7 +179,7 @@ class RemoteKademliaRPC:
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
)
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
return response.response
@@ -188,59 +187,46 @@ class PingQueue:
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
self._loop = loop
self._protocol = protocol
self._enqueued_contacts: typing.List['KademliaPeer'] = []
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
self._process_task: asyncio.Task = None
self._next_task: asyncio.Future = None
self._next_timer: asyncio.TimerHandle = None
self._lock = asyncio.Lock()
self._running = False
self._running_pings: typing.Set[asyncio.Task] = set()
@property
def running(self):
return self._running
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
delay = constants.check_refresh_interval if delay is None else delay
async with self._lock:
for peer in peers:
if delay and peer not in self._enqueued_contacts:
self._pending_contacts[peer] = self._loop.time() + delay
elif peer not in self._enqueued_contacts:
self._enqueued_contacts.append(peer)
if peer in self._pending_contacts:
del self._pending_contacts[peer]
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
now = self._loop.time()
for peer in peers:
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
self._pending_contacts[peer] = delay + now
async def _process(self):
async def _ping(p: 'KademliaPeer'):
def maybe_ping(self, peer: 'KademliaPeer'):
async def ping_task():
try:
if self._protocol.peer_manager.peer_is_good(p):
await self._protocol.add_peer(p)
if self._protocol.peer_manager.peer_is_good(peer):
if peer not in self._protocol.routing_table.get_peers():
await self._protocol.add_peer(peer)
return
await self._protocol.get_rpc_peer(p).ping()
except TimeoutError:
await self._protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
pass
task = self._loop.create_task(ping_task())
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
self._running_pings.add(task)
async def _process(self): # send up to 1 ping per second
while True:
tasks = []
async with self._lock:
if self._enqueued_contacts or self._pending_contacts:
now = self._loop.time()
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
for k in scheduled:
del self._pending_contacts[k]
if k not in self._enqueued_contacts:
self._enqueued_contacts.append(k)
while self._enqueued_contacts:
peer = self._enqueued_contacts.pop()
tasks.append(self._loop.create_task(_ping(peer)))
if tasks:
await asyncio.wait(tasks, loop=self._loop)
f = self._loop.create_future()
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
await f
enqueued = list(self._pending_contacts.keys())
now = self._loop.time()
for peer in enqueued:
if self._pending_contacts[peer] <= now:
del self._pending_contacts[peer]
self.maybe_ping(peer)
break
await asyncio.sleep(1, loop=self._loop)
def start(self):
assert not self._running
@@ -254,17 +240,14 @@ class PingQueue:
if self._process_task:
self._process_task.cancel()
self._process_task = None
if self._next_task:
self._next_task.cancel()
self._next_task = None
if self._next_timer:
self._next_timer.cancel()
self._next_timer = None
while self._running_pings:
self._running_pings.pop().cancel()
class KademliaProtocol(DatagramProtocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.peer_manager = peer_manager
self.loop = loop
self.node_id = node_id
@@ -279,11 +262,10 @@ class KademliaProtocol(DatagramProtocol):
self.transport: DatagramTransport = None
self.old_token_secret = constants.generate_id()
self.token_secret = constants.generate_id()
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
self.data_store = DictDataStore(self.loop, self.peer_manager)
self.ping_queue = PingQueue(self.loop, self)
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
self.lock = asyncio.Lock(loop=self.loop)
self.rpc_timeout = rpc_timeout
self._split_lock = asyncio.Lock(loop=self.loop)
@@ -317,9 +299,14 @@ class KademliaProtocol(DatagramProtocol):
return args, {}
async def _add_peer(self, peer: 'KademliaPeer'):
for p in self.routing_table.get_peers():
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
self.routing_table.remove_peer(p)
self.routing_table.join_buckets()
bucket_index = self.routing_table.kbucket_index(peer.node_id)
if self.routing_table.buckets[bucket_index].add_peer(peer):
return True
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
if self.routing_table.should_split(bucket_index, peer.node_id):
self.routing_table.split_bucket(bucket_index)
@@ -413,22 +400,32 @@ class KademliaProtocol(DatagramProtocol):
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
)
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
# This is an RPC method request
await self.peer_manager.report_last_requested(address[0], address[1])
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
# only add a requesting contact to the routing table if it has replied to one of our requests
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
self.peer_manager.report_last_requested(address[0], address[1])
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
await self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
# will be added to our routing table if successful
is_good = self.peer_manager.peer_is_good(peer)
if is_good is None:
await self.ping_queue.enqueue_maybe_ping(peer)
self.ping_queue.enqueue_maybe_ping(peer)
# only add a requesting contact to the routing table if it has replied to one of our requests
elif is_good is True:
await self.add_peer(peer)
except ValueError as err:
log.debug("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
await self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
)
except Exception as err:
log.warning("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
@@ -445,11 +442,10 @@ class KademliaProtocol(DatagramProtocol):
peer, df, request = self.sent_messages[response_datagram.rpc_id]
if peer.address != address[0]:
df.set_exception(RemoteException(
f"response from {address[0]}:{address[1]}, "
f"expected {peer.address}:{peer.udp_port}")
f"response from {address[0]}, expected {peer.address}")
)
return
peer.set_id(response_datagram.node_id)
# We got a result from the RPC
if peer.node_id == self.node_id:
df.set_exception(RemoteException("node has our node id"))
@@ -457,8 +453,10 @@ class KademliaProtocol(DatagramProtocol):
elif response_datagram.node_id == self.node_id:
df.set_exception(RemoteException("incoming message is from our node id"))
return
await self.peer_manager.report_last_replied(address[0], address[1])
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
peer.set_id(response_datagram.node_id)
peer.update_udp_port(address[1])
self.peer_manager.report_last_replied(address[0], address[1])
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
df.set_result(response_datagram)
await self.add_peer(peer)
@@ -475,21 +473,23 @@ class KademliaProtocol(DatagramProtocol):
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
if error_datagram.rpc_id in self.sent_messages:
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
if (peer.address, peer.udp_port) != address:
df.set_exception(RemoteException(
f"response from {address[0]}:{address[1]}, "
f"expected {peer.address}:{peer.udp_port}")
)
return
error_msg = f"" \
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
f"Args: {request.args}\n" \
f"Raised: {str(remote_exception)}"
if error_datagram.response not in old_protocol_errors:
if 'Invalid token' in error_msg:
log.debug(error_msg)
elif error_datagram.response not in old_protocol_errors:
log.warning(error_msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
# reject replies coming from a different address than what we sent our request to
if (peer.address, peer.udp_port) != address:
log.error("node id mismatch in reply")
remote_exception = TimeoutError(peer.node_id)
df.set_exception(remote_exception)
return
else:
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
try:
message = decode_datagram(datagram)
except (ValueError, TypeError):
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
@@ -522,12 +522,14 @@ class KademliaProtocol(DatagramProtocol):
response_fut = self.sent_messages[request.rpc_id][1]
try:
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except (asyncio.TimeoutError, RemoteException):
await self.peer_manager.report_failure(peer.address, peer.udp_port)
self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
self.routing_table.remove_peer(peer)
async with self._split_lock:
self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
raise
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
@@ -554,30 +556,29 @@ class KademliaProtocol(DatagramProtocol):
if message.rpc_id in self.sent_messages:
self.sent_messages.pop(message.rpc_id)
async with self.lock:
if isinstance(message, RequestDatagram):
response_fut = self.loop.create_future()
response_fut.add_done_callback(pop_from_sent_messages)
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
try:
self.transport.sendto(data, (peer.address, peer.udp_port))
except OSError as err:
# TODO: handle ENETUNREACH
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), peer.address, peer.udp_port, str(err), err.errno)
if isinstance(message, RequestDatagram):
self.sent_messages[message.rpc_id][1].set_exception(err)
else:
raise err
if isinstance(message, RequestDatagram):
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
response_fut = self.loop.create_future()
response_fut.add_done_callback(pop_from_sent_messages)
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
try:
self.transport.sendto(data, (peer.address, peer.udp_port))
except OSError as err:
# TODO: handle ENETUNREACH
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), peer.address, peer.udp_port, str(err), err.errno)
if isinstance(message, RequestDatagram):
self.sent_messages[message.rpc_id][1].set_exception(err)
else:
raise err
if isinstance(message, RequestDatagram):
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
elif isinstance(message, ErrorDatagram):
await self.peer_manager.report_failure(peer.address, peer.udp_port)
self.peer_manager.report_failure(peer.address, peer.udp_port)
def change_token(self):
self.old_token_secret = self.token_secret
@@ -601,7 +602,7 @@ class KademliaProtocol(DatagramProtocol):
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
raise ValueError(res)
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, True
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
@@ -609,7 +610,7 @@ class KademliaProtocol(DatagramProtocol):
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
await self.peer_manager.clear_token(peer.node_id)
self.peer_manager.clear_token(peer.node_id)
else:
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
+22 -8
View File
@@ -29,6 +29,7 @@ class KBucket:
self.range_max = range_max
self.peers: typing.List['KademliaPeer'] = []
self._node_id = node_id
self._distance_to_self = Distance(node_id)
def add_peer(self, peer: 'KademliaPeer') -> bool:
""" Add contact to _contact list in the right order. This will move the
@@ -48,7 +49,14 @@ class KBucket:
self.peers.remove(peer)
self.peers.append(peer)
return True
elif len(self.peers) < constants.k:
else:
for i in range(len(self.peers)):
p = self.peers[i]
if p.node_id == peer.node_id:
self.peers.remove(p)
self.peers.append(peer)
return True
if len(self.peers) < constants.k:
self.peers.append(peer)
return True
else:
@@ -130,7 +138,7 @@ class KBucket:
if not.
@rtype: bool
"""
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
return self.range_min <= self._distance_to_self(key) < self.range_max
def __len__(self) -> int:
return len(self.peers)
@@ -155,10 +163,12 @@ class TreeRoutingTable:
that paper.
"""
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
split_buckets_under_index: int = constants.split_buckets_under_index):
self._loop = loop
self._peer_manager = peer_manager
self._parent_node_id = parent_node_id
self._split_buckets_under_index = split_buckets_under_index
self.buckets: typing.List[KBucket] = [
KBucket(
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
@@ -170,7 +180,7 @@ class TreeRoutingTable:
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
if bucket_index < self._split_buckets_under_index:
return True
contacts = self.get_peers()
distance = Distance(self._parent_node_id)
@@ -183,8 +193,6 @@ class TreeRoutingTable:
exclude = [self._parent_node_id]
if sender_node_id:
exclude.append(sender_node_id)
if key in exclude:
exclude.remove(key)
count = count or constants.k
distance = Distance(key)
contacts = self.get_peers()
@@ -238,11 +246,15 @@ class TreeRoutingTable:
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
return random_id.to_bytes(constants.hash_length, 'big')
return Distance(
self._parent_node_id
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
return Distance(self._parent_node_id)(
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
).to_bytes(constants.hash_length, 'big')
def split_bucket(self, old_bucket_index: int) -> None:
""" Splits the specified k-bucket into two new buckets which together
@@ -269,6 +281,8 @@ class TreeRoutingTable:
old_bucket.remove_peer(contact)
def join_buckets(self):
if len(self.buckets) == 1:
return
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
if not to_pop:
return
+26 -29
View File
@@ -65,46 +65,36 @@ class RequestDatagram(KademliaDatagramBase):
@classmethod
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
if rpc_id and len(rpc_id) != constants.rpc_id_length:
raise ValueError("invalid rpc id length")
elif not rpc_id:
rpc_id = constants.generate_id()[:constants.rpc_id_length]
if len(from_node_id) != constants.hash_bits // 8:
raise ValueError("invalid node id")
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
@classmethod
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
if rpc_id and len(rpc_id) != constants.rpc_id_length:
raise ValueError("invalid rpc id length")
if not rpc_id:
rpc_id = constants.generate_id()[:constants.rpc_id_length]
if len(from_node_id) != constants.hash_bits // 8:
raise ValueError("invalid node id")
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(blob_hash) != constants.hash_bits // 8:
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
if not 0 < port < 65536:
raise ValueError(f"invalid port: {port}")
if len(token) != constants.hash_bits // 8:
raise ValueError(f"invalid token length: {len(token)}")
store_args = [blob_hash, token, port, from_node_id, 0]
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
@classmethod
def make_find_node(cls, from_node_id: bytes, key: bytes,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
if rpc_id and len(rpc_id) != constants.rpc_id_length:
raise ValueError("invalid rpc id length")
if not rpc_id:
rpc_id = constants.generate_id()[:constants.rpc_id_length]
if len(from_node_id) != constants.hash_bits // 8:
raise ValueError("invalid node id")
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid key length: {len(key)}")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
@classmethod
def make_find_value(cls, from_node_id: bytes, key: bytes,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
if rpc_id and len(rpc_id) != constants.rpc_id_length:
raise ValueError("invalid rpc id length")
if not rpc_id:
rpc_id = constants.generate_id()[:constants.rpc_id_length]
if len(from_node_id) != constants.hash_bits // 8:
raise ValueError("invalid node id")
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid key length: {len(key)}")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
@@ -148,8 +138,6 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
}
primitive: typing.Dict = bdecode(datagram)
if not isinstance(primitive, dict):
raise ValueError("invalid datagram type")
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
else:
@@ -163,14 +151,19 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
)
def make_compact_ip(address: str):
return reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
def make_compact_ip(address: str) -> bytearray:
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
if len(compact_ip) != 4:
raise ValueError(f"invalid IPv4 length")
return compact_ip
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
compact_ip = make_compact_ip(address)
if not 0 <= port <= 65536:
if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.hash_bits // 8:
raise ValueError(f"invalid node node_id length")
return compact_ip + port.to_bytes(2, 'big') + node_id
@@ -178,4 +171,8 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
address = "{}.{}.{}.{}".format(*compact_address[:4])
port = int.from_bytes(compact_address[4:6], 'big')
node_id = compact_address[6:]
if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.hash_bits // 8:
raise ValueError(f"invalid node node_id length")
return node_id, address, port
+1 -2
View File
@@ -28,8 +28,7 @@ class DownloadTimeoutError(Exception):
class DownloadDataTimeout(Exception):
def __init__(self, download):
super().__init__('Failed to download data blobs for sd hash '
'{} within timeout'.format(download))
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
self.download = download
+29 -2
View File
@@ -1,4 +1,7 @@
import os
import sys
import signal
import pathlib
import json
import asyncio
import argparse
@@ -216,12 +219,19 @@ def get_argument_parser():
return main
def ensure_directory_exists(path: str):
if not os.path.isdir(path):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
def main(argv=None):
argv = argv or sys.argv[1:]
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
ensure_directory_exists(directory)
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
@@ -238,17 +248,23 @@ def main(argv=None):
)
file_handler.setFormatter(default_formatter)
log.addHandler(file_handler)
logging.getLogger('torba').addHandler(file_handler)
if not args.quiet:
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
log.addHandler(handler)
logging.getLogger('torba').addHandler(handler)
logging.getLogger('torba').setLevel(logging.INFO)
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
loop = asyncio.get_event_loop()
if args.verbose:
log.setLevel(logging.DEBUG)
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
@@ -257,10 +273,21 @@ def main(argv=None):
log.addHandler(loggly_handler)
daemon = Daemon(conf)
loop = asyncio.get_event_loop()
started = False
def __exit():
if started:
daemon.stop_event.set()
else:
raise GracefulExit()
try:
loop.add_signal_handler(signal.SIGINT, __exit)
loop.add_signal_handler(signal.SIGTERM, __exit)
except NotImplementedError:
pass # Not implemented on Windows
try:
loop.run_until_complete(daemon.start())
loop.run_forever()
started = True
loop.run_until_complete(daemon.stop_event.wait())
except (GracefulExit, KeyboardInterrupt):
pass
finally:
+52 -89
View File
@@ -1,11 +1,9 @@
import os
import asyncio
import aiohttp
import logging
import math
import binascii
import typing
import socket
from hashlib import sha256
from types import SimpleNamespace
import base58
@@ -14,11 +12,9 @@ from aioupnp import __version__ as aioupnp_version
from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
import lbrynet.schema
from lbrynet import utils
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.server import BlobServer
@@ -26,8 +22,8 @@ from lbrynet.stream.stream_manager import StreamManager
from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network
from lbrynet.wallet import LbryWalletManager
from lbrynet.wallet import Network
log = logging.getLogger(__name__)
@@ -54,25 +50,6 @@ async def gather_dict(tasks: dict):
)))
async def get_external_ip(): # used if upnp is disabled or non-functioning
try:
async with aiohttp.ClientSession() as session:
async with session.get("https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
pass
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
info = await loop.getaddrinfo(
url, 'https',
proto=socket.IPPROTO_TCP,
)
return info[0][4][0]
class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT
@@ -116,7 +93,7 @@ class DatabaseComponent(Component):
from lbrynet.extras.daemon.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
await asyncio.get_event_loop().run_in_executor(
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
@@ -157,40 +134,31 @@ class HeadersComponent(Component):
}
async def fetch_headers_from_s3(self):
def collector(d, h_file):
h_file.write(d)
local_size = float(h_file.tell())
final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
got_406 = response.status == 406 # our file is bigger
final_size_after_download = response.content_length + local_header_size
if got_406:
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
data = await response.read()
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.content_length)
with open(self.headers_file, "a+b") as headers_file:
collector(data, headers_file)
else:
with open(self.headers_file, "wb") as headers_file:
collector(data, headers_file)
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
return
if response.content_length % HEADER_SIZE != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while True:
chunk = await response.content.read(512)
if not chunk:
break
fd.write(chunk)
self._headers_progress_percent = math.ceil(
float(fd.tell()) / float(final_size_after_download) * 100
)
log.info("fetched headers from s3, now verifying integrity after download.")
self._check_header_file_integrity()
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
@@ -283,23 +251,20 @@ class WalletComponent(Component):
async def get_status(self):
if self.wallet_manager and self.running:
local_height = self.wallet_manager.network.get_local_height()
remote_height = self.wallet_manager.network.get_server_height()
local_height = self.wallet_manager.ledger.headers.height
remote_height = await self.wallet_manager.ledger.network.get_server_height()
best_hash = self.wallet_manager.get_best_blockhash()
return {
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
'best_blockhash': best_hash,
'is_encrypted': self.wallet_manager.wallet.use_encryption,
'is_encrypted': self.wallet_manager.use_encryption,
'is_locked': not self.wallet_manager.is_wallet_unlocked,
}
async def start(self):
log.info("Starting torba wallet")
storage = self.component_manager.get_component(DATABASE_COMPONENT)
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
self.wallet_manager.old_db = storage
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
await self.wallet_manager.start()
async def stop(self):
@@ -331,9 +296,7 @@ class BlobComponent(Component):
return await self.blob_manager.setup()
async def stop(self):
while self.blob_manager and self.blob_manager.blobs:
_, blob = self.blob_manager.blobs.popitem()
await blob.close()
self.blob_manager.stop()
async def get_status(self):
count = 0
@@ -359,7 +322,7 @@ class DHTComponent(Component):
async def get_status(self):
return {
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
}
@@ -376,12 +339,12 @@ class DHTComponent(Component):
async def start(self):
log.info("start the dht")
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await get_external_ip()
external_ip = await utils.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
@@ -389,13 +352,15 @@ class DHTComponent(Component):
asyncio.get_event_loop(),
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.dht_node_port,
internal_udp_port=self.conf.udp_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index
)
self.dht_node.start(
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
)
log.info("Started the dht")
@@ -419,7 +384,7 @@ class HashAnnouncerComponent(Component):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
self.hash_announcer.start(self.conf.concurrent_announcers)
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
log.info("Started blob announcer")
async def stop(self):
@@ -462,17 +427,13 @@ class StreamManagerComponent(Component):
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
self.conf.peer_connect_timeout, [
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
for url, port in self.conf.reflector_servers
], self.conf.reflector_servers
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
)
await self.stream_manager.start()
log.info('Done setting up file manager')
async def stop(self):
await self.stream_manager.stop()
self.stream_manager.stop()
class PeerProtocolServerComponent(Component):
@@ -492,10 +453,10 @@ class PeerProtocolServerComponent(Component):
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
peer_port = self.conf.tcp_port
address = await wallet.get_unused_address()
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
self.blob_server.start_server(peer_port, interface='0.0.0.0')
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
await self.blob_server.started_listening.wait()
async def stop(self):
@@ -508,8 +469,8 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self._int_peer_port = self.conf.tcp_port
self._int_dht_node_port = self.conf.udp_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
@@ -548,7 +509,7 @@ class UPnPComponent(Component):
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
external_ip = await get_external_ip()
external_ip = await utils.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
self.external_ip = external_ip
@@ -599,7 +560,7 @@ class UPnPComponent(Component):
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await get_external_ip()
self.external_ip = await utils.get_external_ip()
return
success = False
await self._maintain_redirects()
@@ -614,7 +575,9 @@ class UPnPComponent(Component):
else:
log.error("failed to setup upnp")
if self.component_manager.analytics_manager:
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
success, await self.get_status()
)
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self):
@@ -640,7 +603,7 @@ class ExchangeRateManagerComponent(Component):
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
def __init__(self, component_manager):
Component.__init__(self, component_manager)
super().__init__(component_manager)
self.exchange_rate_manager = ExchangeRateManager()
@property
File diff suppressed because it is too large. Load diff
+114 -122
View File
@@ -1,13 +1,15 @@
import asyncio
import collections
import logging
import aiohttp
import typing
from lbrynet import utils
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.conf import Config
from lbrynet.extras import system_info
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
@@ -19,102 +21,137 @@ HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
UPNP_SETUP = "UPnP Setup"
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
TIME_TO_FIRST_BYTES = "Time To First Bytes"
log = logging.getLogger(__name__)
class Manager:
def _event_properties(installation_id: str, session_id: str,
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
properties = {
'lbry_id': installation_id,
'session_id': session_id,
}
properties.update(event_properties or {})
return properties
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
total_duration: typing.Optional[float], download_id: str, name: str,
outpoint: str, active_peer_count: int, tried_peers_count: int,
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_download_duration: typing.Optional[float] = None,
error: typing.Optional[str] = None) -> typing.Dict:
return {
"external_ip": external_ip,
"download_id": download_id,
"total_duration": round(total_duration, 4),
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
"error": error,
'name': name,
"outpoint": outpoint,
"node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.reflector_servers) > 0,
"fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count,
"tried_peers_count": tried_peers_count,
"sd_blob_hash": sd_hash,
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
"head_blob_hash": head_blob_hash,
"head_blob_length": head_blob_length,
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
}
def _make_context(platform):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
class AnalyticsManager:
def __init__(self, conf: Config, installation_id: str, session_id: str):
self.conf = conf
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = self._make_context(system_info.get_platform(), 'torba')
self.context = _make_context(system_info.get_platform())
self.installation_id = installation_id
self.session_id = session_id
self.task: asyncio.Task = None
self.external_ip: typing.Optional[str] = None
@property
def is_started(self):
return self.task is not None
def start(self):
async def start(self):
if self._enabled and self.task is None:
self.external_ip = await utils.get_external_ip()
self.task = asyncio.create_task(self.run())
async def run(self):
while True:
await self._send_heartbeat()
await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
async def _post(self, data: typing.Dict):
request_kwargs = {
'method': 'POST',
'url': self.url + endpoint,
'url': self.url + '/track',
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with aiohttp.request(**request_kwargs) as response:
async with utils.aiohttp_request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
async def track(self, event):
async def track(self, event: typing.Dict):
"""Send a single tracking event"""
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post('/track', event)
async def send_new_download_start(self, download_id, name, claim_dict):
await self._send_new_download_stats("start", download_id, name, claim_dict)
async def send_new_download_success(self, download_id, name, claim_dict):
await self._send_new_download_stats("success", download_id, name, claim_dict)
async def send_new_download_fail(self, download_id, name, claim_dict, e):
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
await self.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
await self._post(event)
async def send_upnp_setup_success_fail(self, success, status):
await self.track(
@@ -133,20 +170,33 @@ class Manager:
async def send_server_startup_error(self, message):
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
async def send_download_started(self, id_, name, claim_dict=None):
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
total_duration: typing.Optional[float], download_id: str,
name: str, outpoint: str, found_peers_count: int,
tried_peers_count: int, added_fixed_peers: bool,
fixed_peers_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_duration: typing.Optional[int] = None,
error: typing.Optional[str] = None):
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
)))
async def send_download_finished(self, download_id, name, sd_hash):
await self.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
self._event(
DOWNLOAD_FINISHED, {
'download_id': download_id,
'name': name,
'stream_info': sd_hash
}
)
)
async def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
async def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
async def send_claim_action(self, action):
await self.track(self._event(CLAIM_ACTION, {'action': action}))
@@ -159,69 +209,11 @@ class Manager:
async def _send_heartbeat(self):
await self.track(self._event(HEARTBEAT))
def _event(self, event, event_properties=None):
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'properties': _event_properties(self.installation_id, self.session_id, properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
@@ -1,10 +1,11 @@
import asyncio
import aiohttp
import time
import logging
import json
from decimal import Decimal
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
from lbrynet.utils import aiohttp_request
log = logging.getLogger(__name__)
@@ -52,8 +53,8 @@ class MarketFeed:
def is_online(self):
return self._online
async def _make_request(self):
async with aiohttp.request('get', self.url, params=self.params) as response:
async def _make_request(self) -> str:
async with aiohttp_request('get', self.url, params=self.params) as response:
return (await response.read()).decode()
def _handle_response(self, response):
@@ -219,19 +220,19 @@ class ExchangeRateManager:
def convert_currency(self, from_currency, to_currency, amount):
rates = [market.rate for market in self.market_feeds]
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
if from_currency == to_currency:
return amount
for market in self.market_feeds:
if (market.rate_is_initialized() and market.is_online() and
market.rate.currency_pair == (from_currency, to_currency)):
return amount * market.rate.spot
return amount * Decimal(market.rate.spot)
for market in self.market_feeds:
if (market.rate_is_initialized() and market.is_online() and
market.rate.currency_pair[0] == from_currency):
return self.convert_currency(
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
raise CurrencyConversionError(
f'Unable to convert {amount} from {from_currency} to {to_currency}')
+26 -11
View File
@@ -4,9 +4,10 @@ from binascii import hexlify
from datetime import datetime
from json import JSONEncoder
from ecdsa import BadSignatureError
from lbrynet.extras.wallet import MainNetLedger
from lbrynet.extras.wallet.transaction import Transaction, Output
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from lbrynet.schema.claim import Claim
from lbrynet.wallet.ledger import MainNetLedger
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.dewies import dewies_to_lbc
log = logging.getLogger(__name__)
@@ -23,6 +24,22 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, Claim):
claim_dict = obj.to_dict()
if obj.is_stream:
claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
fee = claim_dict['stream'].get('fee', {})
if 'address' in fee:
fee['address'] = obj.stream.fee.address
if 'amount' in fee:
fee['amount'] = obj.stream.fee.amount
if 'languages' in claim_dict['stream']:
claim_dict['stream']['languages'] = obj.stream.langtags
elif obj.is_channel:
claim_dict['channel']['public_key'] = obj.channel.public_key
if 'languages' in claim_dict['channel']:
claim_dict['channel']['languages'] = obj.channel.langtags
return claim_dict
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -43,7 +60,7 @@ class JSONResponseEncoder(JSONEncoder):
'hex': hexlify(tx.raw).decode(),
}
def encode_output(self, txo):
def encode_output(self, txo, check_signature=True):
tx_height = txo.tx_ref.height
best_height = self.ledger.headers.height
output = {
@@ -68,15 +85,13 @@ class JSONResponseEncoder(JSONEncoder):
if txo.script.is_claim_name or txo.script.is_update_claim:
claim = txo.claim
output['value'] = claim.claim_dict
if claim.has_signature:
output['value'] = claim
if claim.is_signed:
output['valid_signature'] = None
if txo.channel is not None:
if check_signature and txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
try:
output['valid_signature'] = claim.validate_signature(
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
)
output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
except BadSignatureError:
output['valid_signature'] = False
except ValueError:
@@ -98,7 +113,7 @@ class JSONResponseEncoder(JSONEncoder):
return output
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
+1 -2
View File
@@ -1,5 +1,4 @@
import asyncio
import aiohttp
import json
import logging.handlers
import traceback
@@ -51,7 +50,7 @@ class HTTPSLogglyHandler(logging.Handler):
async def _emit(self, record):
payload = self.format(record)
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
def emit(self, record):
@@ -3,7 +3,7 @@ import os
import json
import logging
from binascii import hexlify
from lbrynet.schema.decode import smart_decode
from lbrynet.schema.claim import Claim
log = logging.getLogger(__name__)
@@ -260,7 +260,7 @@ def do_migration(conf):
(
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
)
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
+206 -126
View File
@@ -1,13 +1,14 @@
import os
import logging
import sqlite3
import typing
import asyncio
import binascii
import time
from torba.client.basedatabase import SQLiteMixin
from lbrynet.conf import Config
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.decode import smart_decode
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet.schema.claim import Claim
from lbrynet.dht.constants import data_expiration
from lbrynet.blob.blob_info import BlobInfo
@@ -37,7 +38,9 @@ class StoredStreamClaim:
self.claim_name = name
self.amount = amount
self.height = height
self.claim: typing.Optional[ClaimDict] = None if not serialized else smart_decode(serialized)
self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes(
binascii.unhexlify(serialized)
)
self.claim_address = address
self.claim_sequence = claim_sequence
self.channel_claim_id = channel_claim_id
@@ -104,14 +107,104 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
return StoredStreamClaim(*claim_fields)
def _batched_select(transaction, query, parameters):
for start_index in range(0, len(parameters), 900):
current_batch = parameters[start_index:start_index+900]
def _batched_select(transaction, query, parameters, batch_size=900):
for start_index in range(0, len(parameters), batch_size):
current_batch = parameters[start_index:start_index+batch_size]
bind = "({})".format(','.join(['?'] * len(current_batch)))
for result in transaction.execute(query.format(bind), current_batch):
yield result
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
files = []
signed_claims = {}
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name, *claim_args) in _batched_select(
transaction, "select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"where file.stream_hash in {} "
"order by c.rowid desc", [
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
claim = StoredStreamClaim(stream_hash, *claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
signed_claims[claim.channel_claim_id] = []
signed_claims[claim.channel_claim_id].append(claim)
files.append(
{
"rowid": rowid,
"stream_hash": stream_hash,
"file_name": file_name, # hex
"download_directory": download_dir, # hex
"blob_data_rate": data_rate,
"status": status,
"sd_hash": sd_hash,
"key": stream_key,
"stream_name": stream_name, # hex
"suggested_file_name": suggested_file_name, # hex
"claim": claim
}
)
for claim_name, claim_id in _batched_select(
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
list(signed_claims.keys())):
for claim in signed_claims[claim_id]:
claim.channel_name = claim_name
return files
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
blob_hashes.append((descriptor.sd_hash, ))
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
data_payment_rate: float, status: str) -> int:
transaction.execute(
"insert or replace into file values (?, ?, ?, ?, ?)",
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
)
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
class SQLiteStorage(SQLiteMixin):
CREATE_TABLES_QUERY = """
pragma foreign_keys=on;
@@ -184,11 +277,12 @@ class SQLiteStorage(SQLiteMixin):
);
"""
def __init__(self, conf: Config, path, loop=None):
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
super().__init__(path)
self.conf = conf
self.content_claim_callbacks = {}
self.loop = loop or asyncio.get_event_loop()
self.time_getter = time_getter or time.time
async def run_and_return_one_or_none(self, query, *args):
for row in await self.db.execute_fetchall(query, args):
@@ -202,15 +296,16 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # blob functions # # # # # # # # #
def add_completed_blob(self, blob_hash: str):
log.debug("Adding a completed blob. blob_hash=%s", blob_hash)
return self.db.execute("update blob set status='finished' where blob.blob_hash=?", (blob_hash, ))
def set_should_announce(self, blob_hash: str, next_announce_time: int, should_announce: int):
return self.db.execute(
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
)
def add_completed_blob(self, blob_hash: str, length: int):
def _add_blob(transaction: sqlite3.Connection):
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
(blob_hash, length, 0, 0, "pending", 0, 0)
)
transaction.execute(
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
)
return self.db.run(_add_blob)
def get_blob_status(self, blob_hash: str):
return self.run_and_return_one_or_none(
@@ -247,8 +342,9 @@ class SQLiteStorage(SQLiteMixin):
"select count(*) from blob where status='finished'"
)
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
def _update_last_announced_blobs(transaction: sqlite3.Connection):
last_announced = self.time_getter()
return transaction.executemany(
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
"where blob_hash=?",
@@ -259,7 +355,7 @@ class SQLiteStorage(SQLiteMixin):
def should_single_announce_blobs(self, blob_hashes, immediate=False):
def set_single_announce(transaction):
now = self.loop.time()
now = int(self.time_getter())
for blob_hash in blob_hashes:
if immediate:
transaction.execute(
@@ -274,21 +370,23 @@ class SQLiteStorage(SQLiteMixin):
def get_blobs_to_announce(self):
def get_and_update(transaction):
timestamp = self.loop.time()
if self.conf.announce_head_blobs_only:
timestamp = int(self.time_getter())
if self.conf.announce_head_and_sd_only:
r = transaction.execute(
"select blob_hash from blob "
"where blob_hash is not null and "
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished'",
(timestamp,)
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
else:
r = transaction.execute(
"select blob_hash from blob where blob_hash is not null "
"and next_announce_time<? and status='finished'", (timestamp,)
"and next_announce_time<? and status='finished' "
"order by next_announce_time asc limit ?",
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
)
blobs = [b[0] for b in r.fetchall()]
return blobs
return [b[0] for b in r.fetchall()]
return self.db.run(get_and_update)
def delete_blobs_from_db(self, blob_hashes):
@@ -296,11 +394,31 @@ class SQLiteStorage(SQLiteMixin):
transaction.executemany(
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
)
return self.db.run(delete_blobs)
return self.db.run_with_foreign_keys_disabled(delete_blobs)
def get_all_blob_hashes(self):
return self.run_and_return_list("select blob_hash from blob")
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
to_update = [
(blob_hash, )
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
if blob_hash not in blob_files
]
transaction.executemany(
"update blob set status='pending' where blob_hash=?",
to_update
)
return {
blob_hash
for blob_hash, in _batched_select(
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
list(blob_files)
)
}
return self.db.run(_sync_blobs)
# # # # # # # # # stream functions # # # # # # # # #
async def stream_exists(self, sd_hash: str) -> bool:
@@ -313,42 +431,20 @@ class SQLiteStorage(SQLiteMixin):
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
return streams is not None
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
return self.run_and_return_one_or_none(
"select rowid from file where stream_hash=?", stream_hash
)
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
def _store_stream(transaction: sqlite3.Connection):
# add the head blob and set it to be announced
transaction.execute(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
(
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
)
)
# add the rest of the blobs with announcement off
if len(descriptor.blobs) > 2:
transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
for blob in descriptor.blobs[1:-1]]
)
# associate the blobs to the stream
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
binascii.hexlify(descriptor.stream_name.encode()).decode(),
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
# add the stream
transaction.executemany(
"insert or ignore into stream_blob values (?, ?, ?, ?)",
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
for blob in descriptor.blobs]
)
return self.db.run(store_stream, sd_blob, descriptor)
return self.db.run(_store_stream)
def get_blobs_for_stream(self, stream_hash, only_completed=False):
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
def _get_blobs_for_stream(transaction):
crypt_blob_infos = []
stream_blobs = transaction.execute(
"select blob_hash, position, iv from stream_blob where stream_hash=?", (stream_hash, )
"select blob_hash, position, iv from stream_blob where stream_hash=? "
"order by position asc", (stream_hash, )
).fetchall()
if only_completed:
lengths = transaction.execute(
@@ -370,7 +466,8 @@ class SQLiteStorage(SQLiteMixin):
for blob_hash, position, iv in stream_blobs:
blob_length = blob_length_dict.get(blob_hash, 0)
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
if not blob_hash:
break
return crypt_blob_infos
return self.db.run(_get_blobs_for_stream)
@@ -385,68 +482,58 @@ class SQLiteStorage(SQLiteMixin):
)
def delete_stream(self, descriptor: 'StreamDescriptor'):
def _delete_stream(transaction: sqlite3.Connection):
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
transaction.executemany("delete from blob where blob_hash=?",
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
return self.db.run(_delete_stream)
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
# # # # # # # # # file stuff # # # # # # # # #
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
def save_downloaded_file(self, stream_hash, file_name, download_directory,
data_payment_rate) -> typing.Awaitable[int]:
return self.save_published_file(
stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
status="running"
stream_hash, file_name, download_directory, data_payment_rate, status="running"
)
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
status="finished"):
return self.db.execute(
"insert into file values (?, ?, ?, ?, ?)",
(stream_hash, file_name, download_directory, data_payment_rate, status)
)
status="finished") -> typing.Awaitable[int]:
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
stream_name, suggested_file_name) -> typing.Dict:
return {
"row_id": rowid,
"stream_hash": stream_hash,
"file_name": file_name,
"download_directory": download_dir,
"blob_data_rate": data_rate,
"status": status,
"sd_hash": sd_hash,
"key": stream_key,
"stream_name": stream_name,
"suggested_file_name": suggested_file_name
}
def _get_all_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
file_infos = list(map(lambda a: _lbry_file_dict(*a), transaction.execute(
"select file.rowid, file.*, stream.* "
"from file inner join stream on file.stream_hash=stream.stream_hash"
).fetchall()))
stream_hashes = [file_info['stream_hash'] for file_info in file_infos]
claim_infos = get_claims_from_stream_hashes(transaction, stream_hashes)
for index in range(len(file_infos)): # pylint: disable=consider-using-enumerate
file_infos[index]['claim'] = claim_infos.get(file_infos[index]['stream_hash'])
return file_infos
results = await self.db.run(_get_all_files)
if results:
return results
return []
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)
def change_file_status(self, stream_hash: str, new_status: str):
log.info("update file status %s -> %s", stream_hash, new_status)
log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
stream_hash
))
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
download_directory: str):
def _recover(transaction: sqlite3.Connection):
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
for descriptor, sd_blob in descriptors_and_sds:
content_claim = transaction.execute(
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
).fetchone()
delete_stream(transaction, descriptor) # this will also delete the content claim
store_stream(transaction, sd_blob, descriptor)
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
download_directory, 0.0, 'stopped')
if content_claim:
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
transaction.executemany(
"update file set status='stopped' where stream_hash=?",
[(stream_hash, ) for stream_hash in stream_hashes]
)
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
transaction.executemany(
f"update file set download_directory=? where stream_hash=?",
[(download_dir, stream_hash) for stream_hash in stream_hashes]
)
await self.db.run_with_foreign_keys_disabled(_recover)
def get_all_stream_hashes(self):
return self.run_and_return_list("select stream_hash from stream")
@@ -502,19 +589,12 @@ class SQLiteStorage(SQLiteMixin):
height = claim_info['height']
address = claim_info['address']
sequence = claim_info['claim_sequence']
certificate_id = claim_info['value'].signing_channel_id
try:
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
except AttributeError:
certificate_id = None
try:
if claim_info['value'].get('stream', {}).get('source', {}).get('sourceType') == "lbry_sd_hash":
source_hash = claim_info['value'].get('stream', {}).get('source', {}).get('source')
else:
source_hash = None
except AttributeError:
source_hash = claim_info['value'].stream.sd_hash
except (AttributeError, ValueError):
source_hash = None
serialized = claim_info.get('hex') or binascii.hexlify(
smart_decode(claim_info['value']).serialized).decode()
serialized = binascii.hexlify(claim_info['value'].to_bytes())
transaction.execute(
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
@@ -534,12 +614,12 @@ class SQLiteStorage(SQLiteMixin):
stream_hash = stream_hash[0]
known_outpoint = transaction.execute(
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
)
).fetchone()
known_claim_id = transaction.execute(
"select claim_id from claim "
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
"where c3.stream_hash=?", (stream_hash,)
)
).fetchone()
if not known_claim_id:
content_claims_to_update.append((stream_hash, outpoint))
elif known_outpoint != outpoint:
@@ -578,7 +658,7 @@ class SQLiteStorage(SQLiteMixin):
).fetchone()
if not claim_info:
raise Exception("claim not found")
new_claim_id, claim = claim_info[0], ClaimDict.deserialize(binascii.unhexlify(claim_info[1]))
new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1]))
# certificate claims should not be in the content_claim table
if not claim.is_stream:
@@ -591,7 +671,7 @@ class SQLiteStorage(SQLiteMixin):
if not known_sd_hash:
raise Exception("stream not found")
# check the claim contains the same sd hash
if known_sd_hash[0].encode() != claim.source_hash:
if known_sd_hash[0] != claim.stream.sd_hash:
raise Exception("stream mismatch")
# if there is a current claim associated to the file, check that the new claim is an update to it
@@ -682,7 +762,7 @@ class SQLiteStorage(SQLiteMixin):
if success:
return self.db.execute(
"insert or replace into reflected_stream values (?, ?, ?)",
(sd_hash, reflector_address, self.loop.time())
(sd_hash, reflector_address, self.time_getter())
)
return self.db.execute(
"delete from reflected_stream where sd_hash=? and reflector_address=?",
@@ -694,5 +774,5 @@ class SQLiteStorage(SQLiteMixin):
"select s.sd_hash from stream s "
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
"where r.timestamp is null or r.timestamp < ?",
self.loop.time() - self.conf.auto_re_reflect_interval
int(self.time_getter()) - 86400
)
-2
View File
@@ -2,7 +2,6 @@ import platform
import os
import logging.handlers
from lbrynet.schema import __version__ as schema_version
from lbrynet import build_type, __version__ as lbrynet_version
log = logging.getLogger(__name__)
@@ -16,7 +15,6 @@ def get_platform() -> dict:
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": lbrynet_version,
"lbryschema_version": schema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
-11
View File
@@ -1,11 +0,0 @@
__node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
)
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
from lbrynet.extras.wallet.manager import LbryWalletManager
from lbrynet.extras.wallet.network import Network
-47
View File
@@ -1,47 +0,0 @@
from collections import namedtuple
import msgpack
from torba.server.util import cachedproperty
# Classes representing data and their serializers, if any.
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
'''Claim information as its stored on database'''
@classmethod
def from_serialized(cls, serialized):
return cls(*msgpack.loads(serialized))
@property
def serialized(self):
return msgpack.dumps(self)
class NameClaim(namedtuple("NameClaim", "name value")):
pass
class ClaimUpdate(namedtuple("ClaimUpdate", "name claim_id value")):
pass
class ClaimSupport(namedtuple("ClaimSupport", "name claim_id")):
pass
class LBRYTx(namedtuple("Tx", "version inputs outputs locktime")):
'''Transaction that can contain claim, update or support in its outputs.'''
@cachedproperty
def is_coinbase(self):
return self.inputs[0].is_coinbase
@cachedproperty
def has_claims(self):
for output in self.outputs:
if output.claim:
return True
return False
class TxClaimOutput(namedtuple("TxClaimOutput", "value pk_script claim")):
pass
-126
View File
@@ -1,126 +0,0 @@
import struct
from torba.server.enum import Enumeration
from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
# TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)
opcodes = Enumeration("Opcodes", [
("OP_0", 0), ("OP_PUSHDATA1", 76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED",
"OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7",
"OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
"OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF",
"OP_VERIFY",
"OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER",
"OP_2ROT", "OP_2SWAP",
"OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL",
"OP_ROT",
"OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT",
"OP_AND",
"OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD",
"OP_1SUB", "OP_2MUL",
"OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL",
"OP_DIV",
"OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR",
"OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN",
"OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
"OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
"OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
"OP_CHECKMULTISIGVERIFY", "OP_NOP1", "OP_NOP2", "OP_NOP3", "OP_NOP4", "OP_NOP5",
"OP_CLAIM_NAME",
"OP_SUPPORT_CLAIM", "OP_UPDATE_CLAIM",
("OP_SINGLEBYTE_END", 0xF0),
("OP_DOUBLEBYTE_BEGIN", 0xF000),
"OP_PUBKEY", "OP_PUBKEYHASH",
("OP_INVALIDOPCODE", 0xFFFF),
])
def script_GetOp(bytes):
i = 0
while i < len(bytes):
vch = None
opcode = bytes[i]
i += 1
if opcode <= opcodes.OP_PUSHDATA4:
nSize = opcode
if opcode == opcodes.OP_PUSHDATA1:
nSize = bytes[i]
i += 1
elif opcode == opcodes.OP_PUSHDATA2:
(nSize,) = struct.unpack_from('<H', bytes, i)
i += 2
elif opcode == opcodes.OP_PUSHDATA4:
(nSize,) = struct.unpack_from('<I', bytes, i)
i += 4
if i + nSize > len(bytes):
vch = "_INVALID_" + bytes[i:]
i = len(bytes)
else:
vch = bytes[i:i + nSize]
i += nSize
yield (opcode, vch, i)
def decode_claim_script(bytes_script):
try:
decoded_script = [x for x in script_GetOp(bytes_script)]
except Exception as e:
print(e)
return None
if len(decoded_script) <= 6:
return False
op = 0
claim_type = decoded_script[op][0]
if claim_type == opcodes.OP_UPDATE_CLAIM:
if len(decoded_script) <= 7:
return False
if claim_type not in [
opcodes.OP_CLAIM_NAME,
opcodes.OP_SUPPORT_CLAIM,
opcodes.OP_UPDATE_CLAIM
]:
return False
op += 1
value = None
claim_id = None
claim = None
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
return False
name = decoded_script[op][1]
op += 1
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
return False
if decoded_script[0][0] in [
opcodes.OP_SUPPORT_CLAIM,
opcodes.OP_UPDATE_CLAIM
]:
claim_id = decoded_script[op][1]
if len(claim_id) != 20:
return False
else:
value = decoded_script[op][1]
op += 1
if decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
value = decoded_script[op][1]
op += 1
if decoded_script[op][0] != opcodes.OP_2DROP:
return False
op += 1
if decoded_script[op][0] != opcodes.OP_DROP and decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
return False
elif decoded_script[op][0] != opcodes.OP_2DROP and decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
return False
op += 1
if decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
if name is None or value is None:
return False
claim = NameClaim(name, value)
elif decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
if name is None or value is None or claim_id is None:
return False
claim = ClaimUpdate(name, claim_id, value)
elif decoded_script[0][0] == opcodes.OP_SUPPORT_CLAIM:
if name is None or claim_id is None:
return False
claim = ClaimSupport(name, claim_id)
return claim, decoded_script[op:]
-21
View File
@@ -1,21 +0,0 @@
from torba.server.tx import Deserializer
from lbrynet.extras.wallet.server.opcodes import decode_claim_script
from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
class LBRYDeserializer(Deserializer):
def _read_output(self):
value = self._read_le_int64()
script = self._read_varbytes() # pk_script
claim = decode_claim_script(script)
claim = claim[0] if claim else None
return TxClaimOutput(value, script, claim)
def read_tx(self):
return LBRYTx(
self._read_le_int32(), # version
self._read_inputs(), # inputs
self._read_outputs(), # outputs
self._read_le_uint32() # locktime
)
+4 -5
View File
@@ -1,6 +1,5 @@
build:
rm proto/* -rf
cd proto && protoc --python_out=. -I ../../../../types/proto/ ../../../../types/proto/*.proto
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i proto/*.py
touch proto/__init__.py
rm types/v2/* -rf
touch types/v2/__init__.py
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py
+1 -13
View File
@@ -1,13 +1 @@
import os
__version__ = "0.0.16"
BLOCKCHAIN_NAME_ENVVAR = "LBRYSCHEMA_BLOCKCHAIN_NAME"
if BLOCKCHAIN_NAME_ENVVAR in os.environ:
if os.environ[BLOCKCHAIN_NAME_ENVVAR] in ['lbrycrd_main', 'lbrycrd_regtest',
'lbrycrd_testnet']:
BLOCKCHAIN_NAME = os.environ[BLOCKCHAIN_NAME_ENVVAR]
else:
raise OSError("invalid blockchain name: %s" % os.environ[BLOCKCHAIN_NAME_ENVVAR])
else:
BLOCKCHAIN_NAME = "lbrycrd_main"
from .claim import Claim
-64
View File
@@ -1,64 +0,0 @@
import lbrynet.schema
from lbrynet.schema.base import b58encode, b58decode, validate_b58_checksum
from lbrynet.schema.hashing import double_sha256, hash160
from lbrynet.schema.error import InvalidAddress
from lbrynet.schema.schema import ADDRESS_LENGTH, ADDRESS_PREFIXES, PUBKEY_ADDRESS, SCRIPT_ADDRESS
def validate_address_length(addr_bytes):
if len(addr_bytes) != ADDRESS_LENGTH:
raise InvalidAddress("Invalid address length: %i" % len(addr_bytes))
def validate_address_prefix(addr_bytes):
prefix = addr_bytes[0]
if prefix not in ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME].values():
raise InvalidAddress("Invalid address prefix: %.2X" % prefix)
def validate_lbrycrd_address_bytes(addr_bytes):
validate_address_length(addr_bytes)
validate_address_prefix(addr_bytes)
validate_b58_checksum(addr_bytes)
return addr_bytes
def decode_address(v):
"""decode and validate a b58 address"""
return validate_lbrycrd_address_bytes(b58decode(v))
def encode_address(addr_bytes):
"""validate and encode an address as b58"""
v = validate_lbrycrd_address_bytes(addr_bytes)
return b58encode(v)
def hash_160_bytes_to_address(h160, addrtype=PUBKEY_ADDRESS):
if addrtype == PUBKEY_ADDRESS:
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS])
elif addrtype == SCRIPT_ADDRESS:
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][SCRIPT_ADDRESS])
else:
raise Exception("Invalid address prefix")
return b58encode(prefix + h160 + double_sha256(prefix + h160)[0:4])
def public_key_to_address(public_key):
return hash_160_bytes_to_address(hash160(public_key))
def address_to_hash_160(addr):
bytes = decode_address(addr)
prefix, pubkey_bytes, addr_checksum = bytes[0], bytes[1:21], bytes[21:]
if prefix == chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS]):
return PUBKEY_ADDRESS, pubkey_bytes
return SCRIPT_ADDRESS, pubkey_bytes
def is_address(addr):
try:
addr_bytes = decode_address(addr)
return True
except InvalidAddress:
return False
+59 -75
View File
@@ -1,90 +1,74 @@
from lbrynet.schema.schema import ADDRESS_CHECKSUM_LENGTH
from lbrynet.schema.hashing import double_sha256
from lbrynet.schema.error import InvalidAddress
from binascii import hexlify, unhexlify
from google.protobuf.message import DecodeError
from google.protobuf.json_format import MessageToDict
alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
class Signable:
__slots__ = (
'message', 'version', 'signature',
'signature_type', 'unsigned_payload', 'signing_channel_hash'
)
def scrub_input(v):
if isinstance(v, str) and not isinstance(v, bytes):
v = v.encode('ascii')
return v
message_class = None
def __init__(self, message=None):
self.message = message or self.message_class()
self.version = 2
self.signature = None
self.signature_type = 'SECP256k1'
self.unsigned_payload = None
self.signing_channel_hash = None
def b58encode_int(i, default_one=True):
'''Encode an integer using Base58'''
if not i and default_one:
return alphabet[0:1]
string = b""
while i:
i, idx = divmod(i, 58)
string = alphabet[idx:idx+1] + string
return string
@property
def is_undetermined(self):
return self.message.WhichOneof('type') is None
@property
def signing_channel_id(self):
return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
def b58encode(v):
'''Encode a string using Base58'''
@signing_channel_id.setter
def signing_channel_id(self, channel_id: str):
self.signing_channel_hash = unhexlify(channel_id)[::-1]
v = scrub_input(v)
@property
def is_signed(self):
return self.signature is not None
nPad = len(v)
v = v.lstrip(b'\0')
nPad -= len(v)
def to_dict(self):
return MessageToDict(self.message)
p, acc = 1, 0
for c in reversed(v):
acc += p * c
p = p << 8
def to_message_bytes(self) -> bytes:
return self.message.SerializeToString()
result = b58encode_int(acc, default_one=False)
def to_bytes(self) -> bytes:
pieces = bytearray()
if self.is_signed:
pieces.append(1)
pieces.extend(self.signing_channel_hash)
pieces.extend(self.signature)
else:
pieces.append(0)
pieces.extend(self.to_message_bytes())
return bytes(pieces)
return alphabet[0:1] * nPad + result
@classmethod
def from_bytes(cls, data: bytes):
signable = cls()
if data[0] == 0:
signable.message.ParseFromString(data[1:])
elif data[0] == 1:
signable.signing_channel_hash = data[1:21]
signable.signature = data[21:85]
signable.message.ParseFromString(data[85:])
else:
raise DecodeError('Could not determine message format version.')
return signable
def __len__(self):
return len(self.to_bytes())
def b58decode_int(v):
'''Decode a Base58 encoded string as an integer'''
v = scrub_input(v)
decimal = 0
for char in v:
decimal = decimal * 58 + alphabet.index(char)
return decimal
def b58decode(v):
'''Decode a Base58 encoded string'''
v = scrub_input(v)
origlen = len(v)
v = v.lstrip(alphabet[0:1])
newlen = len(v)
acc = b58decode_int(v)
result = []
while acc > 0:
acc, mod = divmod(acc, 256)
result.append(mod)
return b'\0' * (origlen - newlen) + bytes(reversed(result))
def validate_b58_checksum(addr_bytes):
addr_without_checksum = addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
addr_checksum = addr_bytes[-ADDRESS_CHECKSUM_LENGTH:]
if double_sha256(addr_without_checksum)[:ADDRESS_CHECKSUM_LENGTH] != addr_checksum:
raise InvalidAddress("Invalid address checksum")
def b58decode_strip_checksum(v):
addr_bytes = b58decode(v)
validate_b58_checksum(addr_bytes)
return addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
def b58encode_with_checksum(addr_bytes):
addr_checksum = double_sha256(addr_bytes)[:ADDRESS_CHECKSUM_LENGTH]
return b58encode(addr_bytes + addr_checksum)
def __bytes__(self):
return self.to_bytes()
+714 -162
View File
@@ -1,198 +1,750 @@
import os.path
import json
import binascii
from google.protobuf import json_format # pylint: disable=no-name-in-module
from google.protobuf.message import DecodeError as DecodeError_pb # pylint: disable=no-name-in-module,import-error
from string import ascii_letters
from typing import List, Tuple, Iterator, TypeVar, Generic
from decimal import Decimal
from binascii import hexlify, unhexlify
from collections import OrderedDict
from google.protobuf.json_format import MessageToDict
from google.protobuf.message import DecodeError
from hachoir.parser import createParser as binary_file_parser
from hachoir.metadata import extractMetadata as binary_file_metadata
from hachoir.core.log import log as hachoir_log
from lbrynet.schema.schema.claim import Claim
from lbrynet.schema.proto import claim_pb2
from lbrynet.schema.signature import Signature
from lbrynet.schema.validator import get_validator
from lbrynet.schema.signer import get_signer
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
from lbrynet.schema.error import DecodeError
from lbrynet.schema.fee import Fee
from torba.client.hash import Base58
from torba.client.constants import COIN
from lbrynet.schema import compat
from lbrynet.schema.base import Signable
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.schema.types.v2.claim_pb2 import (
Claim as ClaimMessage,
Fee as FeeMessage,
Location as LocationMessage,
Language as LanguageMessage
)
class ClaimDict(OrderedDict):
def __init__(self, claim_dict=None, detached_signature: Signature=None):
if isinstance(claim_dict, claim_pb2.Claim):
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
(self.__class__.__name__, self.__class__.__name__))
self.detached_signature = detached_signature
OrderedDict.__init__(self, claim_dict or [])
hachoir_log.use_print = False
@property
def protobuf_dict(self):
"""Claim dictionary using base64 to represent bytes"""
return json.loads(json_format.MessageToJson(self.protobuf, True))
class Claim(Signable):
@property
def protobuf(self):
"""Claim message object"""
__slots__ = 'version',
message_class = ClaimMessage
return Claim.load(self)
@property
def serialized(self):
"""Serialized Claim protobuf"""
if self.detached_signature:
return self.detached_signature.serialized
return self.protobuf.SerializeToString()
@property
def serialized_no_signature(self):
"""Serialized Claim protobuf without publisherSignature field"""
claim = self.protobuf
claim.ClearField("publisherSignature")
return ClaimDict.load_protobuf(claim).serialized
@property
def has_signature(self):
return self.protobuf.HasField("publisherSignature") or (
self.detached_signature and self.detached_signature.raw_signature
)
@property
def is_certificate(self):
claim = self.protobuf
return CLAIM_TYPE_NAMES[claim.claimType] == "certificate"
def __init__(self, claim_message=None):
super().__init__(claim_message)
self.version = 2
@property
def is_stream(self):
claim = self.protobuf
return CLAIM_TYPE_NAMES[claim.claimType] == "stream"
return self.message.WhichOneof('type') == 'stream'
@property
def source_hash(self):
claim = self.protobuf
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
return None
return binascii.hexlify(claim.stream.source.source)
def is_channel(self):
return self.message.WhichOneof('type') == 'channel'
@property
def has_fee(self):
claim = self.protobuf
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
return None
if claim.stream.metadata.HasField("fee"):
return True
return False
def stream_message(self):
if self.is_undetermined:
self.message.stream.SetInParent()
if not self.is_stream:
raise ValueError('Claim is not a stream.')
return self.message.stream
@property
def source_fee(self):
claim = self.protobuf
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
return None
if claim.stream.metadata.HasField("fee"):
return Fee.load_protobuf(claim.stream.metadata.fee)
return None
def stream(self) -> 'Stream':
return Stream(self)
@property
def certificate_id(self) -> str:
if self.protobuf.HasField("publisherSignature"):
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
if self.detached_signature and self.detached_signature.certificate_id:
return binascii.hexlify(self.detached_signature.certificate_id).decode()
def channel_message(self):
if self.is_undetermined:
self.message.channel.SetInParent()
if not self.is_channel:
raise ValueError('Claim is not a channel.')
return self.message.channel
@property
def signature(self):
if not self.has_signature:
return None
return binascii.hexlify(self.protobuf.publisherSignature.signature)
def channel(self) -> 'Channel':
return Channel(self)
@property
def protobuf_len(self):
"""Length of serialized string"""
return self.protobuf.ByteSize()
@property
def json_len(self):
"""Length of json encoded string"""
return len(json.dumps(self.claim_dict))
@property
def claim_dict(self):
"""Claim dictionary with bytes represented as hex and base58"""
return dict(encode_fields(self, self.detached_signature))
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
@classmethod
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
"""
Load a ClaimDict from a dictionary with base64 encoded bytes
(as returned by the protobuf json formatter)
"""
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
@classmethod
def load_protobuf(cls, protobuf_claim, detached_signature=None):
"""Load ClaimDict from a protobuf Claim message"""
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
@classmethod
def load_dict(cls, claim_dict):
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
def from_bytes(cls, data: bytes) -> 'Claim':
try:
claim_dict, detached_signature = decode_fields(claim_dict)
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
except json_format.ParseError as err:
raise DecodeError(str(err))
return super().from_bytes(data)
except DecodeError:
claim = cls()
if data[0] == ord('{'):
claim.version = 0
compat.from_old_json_schema(claim, data)
elif data[0] not in (0, 1):
claim.version = 1
compat.from_types_v1(claim, data)
else:
raise
return claim
@classmethod
def deserialize(cls, serialized):
"""Load a ClaimDict from a serialized protobuf string"""
detached_signature = Signature.flagged_parse(serialized)
temp_claim = claim_pb2.Claim()
I = TypeVar('I')
class BaseMessageList(Generic[I]):
__slots__ = 'message',
item_class = None
def __init__(self, message):
self.message = message
def add(self) -> I:
return self.item_class(self.message.add())
def extend(self, values: List[str]):
for value in values:
self.append(value)
def append(self, value: str):
raise NotImplemented
def __len__(self):
return len(self.message)
def __iter__(self) -> Iterator[I]:
for lang in self.message:
yield self.item_class(lang)
def __getitem__(self, item) -> I:
return self.item_class(self.message[item])
class Dimmensional:
__slots__ = ()
@property
def width(self) -> int:
return self.message.width
@width.setter
def width(self, width: int):
self.message.width = width
@property
def height(self) -> int:
return self.message.height
@height.setter
def height(self, height: int):
self.message.height = height
@property
def dimensions(self) -> Tuple[int, int]:
return self.width, self.height
@dimensions.setter
def dimensions(self, dimensions: Tuple[int, int]):
self.message.width, self.message.height = dimensions
class Playable:
__slots__ = ()
@property
def duration(self) -> int:
return self.message.duration
@duration.setter
def duration(self, duration: int):
self.message.duration = duration
def set_duration_from_path(self, file_path):
try:
temp_claim.ParseFromString(detached_signature.payload)
except DecodeError_pb:
raise DecodeError(DecodeError_pb)
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
file_metadata = binary_file_metadata(binary_file_parser(file_path))
self.duration = file_metadata.getValues('duration')[0].seconds
except:
pass
@classmethod
def generate_certificate(cls, private_key, curve=SECP256k1):
signer = get_signer(curve).load_pem(private_key)
return cls.load_protobuf(signer.certificate)
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
signer = get_signer(curve).load_pem(private_key)
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
return ClaimDict.load_protobuf(signed, signature)
class Image(Dimmensional):
def validate_signature(self, claim_address, certificate, name=None):
if isinstance(certificate, ClaimDict):
certificate = certificate.protobuf
curve = CURVE_NAMES[certificate.certificate.keyType]
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
return validator.validate_claim_signature(self, claim_address, name)
__slots__ = 'message',
def validate_private_key(self, private_key, certificate_id):
certificate = self.protobuf
if CLAIM_TYPE_NAMES[certificate.claimType] != "certificate":
return
curve = CURVE_NAMES[certificate.certificate.keyType]
validator = get_validator(curve).load_from_certificate(certificate, certificate_id)
signing_key = validator.signing_key_from_pem(private_key)
return validator.validate_private_key(signing_key)
def __init__(self, image_message):
self.message = image_message
def get_validator(self, certificate_id):
"""
Get a lbrynet.schema.validator.Validator object for a certificate claim
:param certificate_id: claim id of this certificate claim
:return: None or lbrynet.schema.validator.Validator object
"""
class Video(Dimmensional, Playable):
claim = self.protobuf
if CLAIM_TYPE_NAMES[claim.claimType] != "certificate":
return
curve = CURVE_NAMES[claim.certificate.keyType]
return get_validator(curve).load_from_certificate(claim, certificate_id)
__slots__ = 'message',
def __init__(self, video_message):
self.message = video_message
class Audio(Playable):
__slots__ = 'message',
def __init__(self, audio_message):
self.message = audio_message
class File:
__slots__ = '_file',
def __init__(self, file_message):
self._file = file_message
@property
def name(self) -> str:
return self._file.name
@name.setter
def name(self, name: str):
self._file.name = name
@property
def size(self) -> int:
return self._file.size
@size.setter
def size(self, size: int):
self._file.size = size
class Fee:
__slots__ = '_fee',
def __init__(self, fee_message):
self._fee = fee_message
@property
def currency(self) -> str:
return FeeMessage.Currency.Name(self._fee.currency)
@property
def address(self) -> str:
return Base58.encode(self._fee.address)
@address.setter
def address(self, address: str):
self._fee.address = Base58.decode(address)
@property
def address_bytes(self) -> bytes:
return self._fee.address
@address_bytes.setter
def address_bytes(self, address: bytes):
self._fee.address = address
@property
def amount(self) -> Decimal:
if self.currency == 'LBC':
return self.lbc
if self.currency == 'BTC':
return self.btc
if self.currency == 'USD':
return self.usd
DEWIES = Decimal(COIN)
@property
def lbc(self) -> Decimal:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('LBC can only be returned for LBC fees.')
return Decimal(self._fee.amount / self.DEWIES)
@lbc.setter
def lbc(self, amount: Decimal):
self.dewies = int(amount * self.DEWIES)
@property
def dewies(self) -> int:
if self._fee.currency != FeeMessage.LBC:
raise ValueError('Dewies can only be returned for LBC fees.')
return self._fee.amount
@dewies.setter
def dewies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.LBC
SATOSHIES = Decimal(COIN)
@property
def btc(self) -> Decimal:
if self._fee.currency != FeeMessage.BTC:
raise ValueError('BTC can only be returned for BTC fees.')
return Decimal(self._fee.amount / self.SATOSHIES)
@btc.setter
def btc(self, amount: Decimal):
self.satoshis = int(amount * self.SATOSHIES)
@property
def satoshis(self) -> int:
if self._fee.currency != FeeMessage.BTC:
raise ValueError('Satoshies can only be returned for BTC fees.')
return self._fee.amount
@satoshis.setter
def satoshis(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.BTC
PENNIES = Decimal(100.0)
@property
def usd(self) -> Decimal:
if self._fee.currency != FeeMessage.USD:
raise ValueError('USD can only be returned for USD fees.')
return Decimal(self._fee.amount / self.PENNIES)
@usd.setter
def usd(self, amount: Decimal):
self.pennies = int(amount * self.PENNIES)
@property
def pennies(self) -> int:
if self._fee.currency != FeeMessage.USD:
raise ValueError('Pennies can only be returned for USD fees.')
return self._fee.amount
@pennies.setter
def pennies(self, amount: int):
self._fee.amount = amount
self._fee.currency = FeeMessage.USD
class Language:
__slots__ = 'message',
def __init__(self, message):
self.message = message
@property
def langtag(self) -> str:
langtag = []
if self.language:
langtag.append(self.language)
if self.script:
langtag.append(self.script)
if self.region:
langtag.append(self.region)
return '-'.join(langtag)
@langtag.setter
def langtag(self, langtag: str):
parts = langtag.split('-')
self.language = parts.pop(0)
if parts and len(parts[0]) == 4:
self.script = parts.pop(0)
if parts and len(parts[0]) == 2:
self.region = parts.pop(0)
assert not parts, f"Failed to parse language tag: {langtag}"
@property
def language(self) -> str:
if self.message.language:
return LanguageMessage.Language.Name(self.message.language)
@language.setter
def language(self, language: str):
self.message.language = LanguageMessage.Language.Value(language)
@property
def script(self) -> str:
if self.message.script:
return LanguageMessage.Script.Name(self.message.script)
@script.setter
def script(self, script: str):
self.message.script = LanguageMessage.Script.Value(script)
@property
def region(self) -> str:
if self.message.region:
return LocationMessage.Country.Name(self.message.region)
@region.setter
def region(self, region: str):
self.message.region = LocationMessage.Country.Value(region)
class LanguageList(BaseMessageList[Language]):
__slots__ = ()
item_class = Language
def append(self, value: str):
self.add().langtag = value
class Location:
__slots__ = 'message',
def __init__(self, message):
self.message = message
def from_value(self, value):
if isinstance(value, str) and value.startswith('{'):
value = json.loads(value)
if isinstance(value, dict):
for key, val in value.items():
setattr(self, key, val)
elif isinstance(value, str):
parts = value.split(':')
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
country = parts and parts.pop(0)
if country:
self.country = country
state = parts and parts.pop(0)
if state:
self.state = state
city = parts and parts.pop(0)
if city:
self.city = city
code = parts and parts.pop(0)
if code:
self.code = code
latitude = parts and parts.pop(0)
if latitude:
self.latitude = latitude
longitude = parts and parts.pop(0)
if longitude:
self.longitude = longitude
else:
raise ValueError(f'Could not parse country value: {value}')
@property
def country(self) -> str:
if self.message.country:
return LocationMessage.Country.Name(self.message.country)
@country.setter
def country(self, country: str):
self.message.country = LocationMessage.Country.Value(country)
@property
def state(self) -> str:
return self.message.state
@state.setter
def state(self, state: str):
self.message.state = state
@property
def city(self) -> str:
return self.message.city
@city.setter
def city(self, city: str):
self.message.city = city
@property
def code(self) -> str:
return self.message.code
@code.setter
def code(self, code: str):
self.message.code = code
GPS_PRECISION = Decimal('10000000')
@property
def latitude(self) -> str:
if self.message.latitude:
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
@latitude.setter
def latitude(self, latitude: str):
latitude = Decimal(latitude)
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
self.message.latitude = int(latitude * self.GPS_PRECISION)
@property
def longitude(self) -> str:
if self.message.longitude:
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
@longitude.setter
def longitude(self, longitude: str):
longitude = Decimal(longitude)
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
self.message.longitude = int(longitude * self.GPS_PRECISION)
class LocationList(BaseMessageList[Location]):
__slots__ = ()
item_class = Location
def append(self, value):
self.add().from_value(value)
class BaseClaimSubType:
__slots__ = 'claim', 'message'
def __init__(self, claim: Claim):
self.claim = claim or Claim()
@property
def title(self) -> str:
return self.message.title
@title.setter
def title(self, title: str):
self.message.title = title
@property
def description(self) -> str:
return self.message.description
@description.setter
def description(self, description: str):
self.message.description = description
@property
def thumbnail_url(self) -> str:
return self.message.thumbnail_url
@thumbnail_url.setter
def thumbnail_url(self, thumbnail_url: str):
self.message.thumbnail_url = thumbnail_url
@property
def tags(self) -> List:
return self.message.tags
@property
def languages(self) -> LanguageList:
return LanguageList(self.message.languages)
@property
def langtags(self) -> List[str]:
return [l.langtag for l in self.languages]
@property
def locations(self) -> LocationList:
return LocationList(self.message.locations)
def to_dict(self):
return MessageToDict(self.message, preserving_proto_field_name=True)
def update(self, **kwargs):
for l in ('tags', 'languages', 'locations'):
if kwargs.pop(f'clear_{l}', False):
self.message.ClearField('tags')
items = kwargs.pop(l, None)
if items is not None:
if isinstance(items, str):
getattr(self, l).append(items)
elif isinstance(items, list):
getattr(self, l).extend(items)
else:
raise ValueError(f"Unknown {l} value: {items}")
for key, value in kwargs.items():
setattr(self, key, value)
class Channel(BaseClaimSubType):
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.channel_message
@property
def public_key(self) -> str:
return hexlify(self.message.public_key).decode()
@public_key.setter
def public_key(self, sd_public_key: str):
self.message.public_key = unhexlify(sd_public_key.encode())
@property
def public_key_bytes(self) -> bytes:
return self.message.public_key
@public_key_bytes.setter
def public_key_bytes(self, public_key: bytes):
self.message.public_key = public_key
@property
def contact_email(self) -> str:
return self.message.contact_email
@contact_email.setter
def contact_email(self, contact_email: str):
self.message.contact_email = contact_email
@property
def homepage_url(self) -> str:
return self.message.homepage_url
@homepage_url.setter
def homepage_url(self, homepage_url: str):
self.message.homepage_url = homepage_url
@property
def cover_url(self) -> str:
return self.message.cover_url
@cover_url.setter
def cover_url(self, cover_url: str):
self.message.cover_url = cover_url
class Stream(BaseClaimSubType):
__slots__ = ()
def __init__(self, claim: Claim = None):
super().__init__(claim)
self.message = self.claim.stream_message
def update(
self, file_path=None, stream_type=None,
fee_currency=None, fee_amount=None, fee_address=None,
**kwargs):
duration_was_not_set = True
sub_types = ('image', 'video', 'audio')
for key in list(kwargs.keys()):
for sub_type in sub_types:
if key.startswith(f'{sub_type}_'):
stream_type = sub_type
sub_obj = getattr(self, sub_type)
sub_obj_attr = key[len(f'{sub_type}_'):]
setattr(sub_obj, sub_obj_attr, kwargs.pop(key))
if sub_obj_attr == 'duration':
duration_was_not_set = False
break
if stream_type is not None:
if stream_type not in sub_types:
raise Exception(
f"stream_type of '{stream_type}' is not valid, must be one of: {sub_types}"
)
sub_obj = getattr(self, stream_type)
if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
sub_obj.set_duration_from_path(file_path)
super().update(**kwargs)
if file_path is not None:
self.media_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
self.file.size = os.path.getsize(file_path)
if self.file.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
if fee_amount and fee_currency:
if fee_address:
self.fee.address = fee_address
if fee_currency.lower() == 'lbc':
self.fee.lbc = Decimal(fee_amount)
elif fee_currency.lower() == 'btc':
self.fee.btc = Decimal(fee_amount)
elif fee_currency.lower() == 'usd':
self.fee.usd = Decimal(fee_amount)
else:
raise Exception(f'Unknown currency type: {fee_currency}')
@property
def sd_hash(self) -> str:
return hexlify(self.message.sd_hash).decode()
@sd_hash.setter
def sd_hash(self, sd_hash: str):
self.message.sd_hash = unhexlify(sd_hash.encode())
@property
def sd_hash_bytes(self) -> bytes:
return self.message.sd_hash
@sd_hash_bytes.setter
def sd_hash_bytes(self, sd_hash: bytes):
self.message.sd_hash = sd_hash
@property
def author(self) -> str:
return self.message.author
@author.setter
def author(self, author: str):
self.message.author = author
@property
def license(self) -> str:
return self.message.license
@license.setter
def license(self, license: str):
self.message.license = license
@property
def license_url(self) -> str:
return self.message.license_url
@license_url.setter
def license_url(self, license_url: str):
self.message.license_url = license_url
@property
def release_time(self) -> int:
return self.message.release_time
@release_time.setter
def release_time(self, release_time: int):
self.message.release_time = release_time
@property
def media_type(self) -> str:
return self.message.media_type
@media_type.setter
def media_type(self, media_type: str):
self.message.media_type = media_type
@property
def fee(self) -> Fee:
return Fee(self.message.fee)
@property
def has_fee(self) -> bool:
return self.message.HasField('fee')
@property
def file(self) -> File:
return File(self.message.file)
@property
def image(self) -> Image:
return Image(self.message.image)
@property
def video(self) -> Video:
return Video(self.message.video)
@property
def audio(self) -> Audio:
return Audio(self.message.audio)
+86
View File
@@ -0,0 +1,86 @@
import json
from decimal import Decimal
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
def from_old_json_schema(claim, payload: bytes):
value = json.loads(payload)
stream = claim.stream
stream.media_type = value.get('content_type', value.get('content-type', 'application/octet-stream'))
stream.title = value.get('title', '')
stream.description = value.get('description', '')
stream.thumbnail_url = value.get('thumbnail', '')
stream.author = value.get('author', '')
stream.license = value.get('license', '')
stream.license_url = value.get('license_url', '')
language = value.get('language', '')
if language:
if language.lower() == 'english':
language = 'en'
try:
stream.languages.append(language)
except:
pass
stream.sd_hash = value['sources']['lbry_sd_hash']
if value.get('nsfw', False):
stream.tags.append('mature')
if "fee" in value:
fee = value["fee"]
currency = list(fee.keys())[0]
if currency == 'LBC':
stream.fee.lbc = Decimal(fee[currency]['amount'])
elif currency == 'USD':
stream.fee.usd = Decimal(fee[currency]['amount'])
elif currency == 'BTC':
stream.fee.btc = Decimal(fee[currency]['amount'])
else:
raise ValueError(f'Unknown currency: {currency}')
stream.fee.address = fee[currency]['address']
return claim
def from_types_v1(claim, payload: bytes):
old = OldClaimMessage()
old.ParseFromString(payload)
if old.claimType == 1:
stream = claim.stream
stream.title = old.stream.metadata.title
stream.description = old.stream.metadata.description
stream.author = old.stream.metadata.author
stream.license = old.stream.metadata.license
stream.license_url = old.stream.metadata.licenseUrl
stream.thumbnail_url = old.stream.metadata.thumbnail
if old.stream.metadata.HasField('language'):
stream.languages.add().message.language = old.stream.metadata.language
stream.media_type = old.stream.source.contentType
stream.sd_hash_bytes = old.stream.source.source
if old.stream.metadata.nsfw:
stream.tags.append('mature')
if old.stream.metadata.HasField('fee'):
fee = old.stream.metadata.fee
stream.fee.address_bytes = fee.address
currency = FeeMessage.Currency.Name(fee.currency)
if currency == 'LBC':
stream.fee.lbc = Decimal(fee.amount)
elif currency == 'USD':
stream.fee.usd = Decimal(fee.amount)
elif currency == 'BTC':
stream.fee.btc = Decimal(fee.amount)
else:
raise ValueError(f'Unsupported currency: {currency}')
if old.HasField('publisherSignature'):
sig = old.publisherSignature
claim.signature = sig.signature
claim.signature_type = KeyType.Name(sig.signatureType)
claim.signing_channel_hash = sig.certificateId
old.ClearField("publisherSignature")
claim.unsigned_payload = old.SerializeToString()
elif old.claimType == 2:
channel = claim.channel
channel.public_key_bytes = old.certificate.publicKey
else:
raise ValueError('claimType must be 1 for Streams and 2 for Channel')
return claim
-61
View File
@@ -1,61 +0,0 @@
import json
import binascii
from google.protobuf import json_format # pylint: disable=no-name-in-module
from lbrynet.schema.error import DecodeError, InvalidAddress
from lbrynet.schema.legacy.migrate import migrate as schema_migrator
from lbrynet.schema.claim import ClaimDict
def migrate_json_claim_value(decoded_json):
try:
if 'fee' in decoded_json:
old_fee = decoded_json['fee']
if not old_fee[list(old_fee.keys())[0]]['amount']:
del decoded_json['fee']
return migrate_json_claim_value(decoded_json)
except (TypeError, AttributeError, InvalidAddress):
raise DecodeError("Failed to decode claim")
try:
pb_migrated = schema_migrator(decoded_json)
return pb_migrated
except json_format.ParseError as parse_error:
raise DecodeError("Failed to parse protobuf: %s" % parse_error)
except Exception as err:
raise DecodeError("Failed to migrate claim: %s" % err)
def smart_decode(claim_value):
"""
Decode a claim value
Try decoding claim protobuf, if this fails try decoding json and migrating it.
If unable to decode or migrate, raise DecodeError
"""
# if already decoded, return
if isinstance(claim_value, ClaimDict):
return claim_value
elif isinstance(claim_value, dict):
return ClaimDict.load_dict(claim_value)
try:
claim_value = binascii.unhexlify(claim_value)
except (TypeError, ValueError):
pass
if claim_value[0] in ['{', ord('{')]:
try:
if isinstance(claim_value, bytes):
claim_value = claim_value.decode()
decoded_json = json.loads(claim_value)
return migrate_json_claim_value(decoded_json)
except (ValueError, TypeError):
pass
try:
if isinstance(claim_value, str):
claim_value = claim_value.encode()
return ClaimDict.deserialize(claim_value)
except (DecodeError, InvalidAddress, KeyError, TypeError):
raise DecodeError(claim_value)
-90
View File
@@ -1,90 +0,0 @@
import base64, binascii
from copy import deepcopy
from lbrynet.schema.address import decode_address, encode_address
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
from lbrynet.schema.schema import SIGNATURE
from lbrynet.schema.error import DecodeError, InvalidAddress
from lbrynet.schema.signature import Signature
def encode_fields(claim_dictionary, detached_signature: Signature):
"""Encode bytes to hex and b58 for return by ClaimDict"""
claim_dictionary = deepcopy(claim_dictionary)
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
claim_value = claim_dictionary[claim_type]
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
if 'fee' in claim_value['metadata']:
try:
address = encode_address(claim_value['metadata']['fee']['address'])
except InvalidAddress as err:
raise DecodeError("Invalid fee address: %s" % err)
claim_value['metadata']['fee']['address'] = address
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
public_key = claim_value["publicKey"]
claim_value["publicKey"] = binascii.hexlify(public_key).decode()
if SIGNATURE in claim_dictionary:
encoded_sig = binascii.hexlify(claim_dictionary[SIGNATURE]['signature']).decode()
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
elif detached_signature and detached_signature.raw_signature:
claim_dictionary[SIGNATURE] = {
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
}
claim_dictionary[claim_type] = claim_value
return claim_dictionary
def decode_fields(claim_dictionary):
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
detached_signature = None
claim_dictionary = deepcopy(claim_dictionary)
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
claim_value = claim_dictionary[claim_type]
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
claim_value['source']['source'] = binascii.unhexlify(claim_value['source']['source'])
if 'fee' in claim_value['metadata']:
try:
address = decode_address(claim_value['metadata']['fee']['address'])
except InvalidAddress as err:
raise DecodeError("Invalid fee address: %s" % err)
claim_value['metadata']['fee']['address'] = address
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
public_key = binascii.unhexlify(claim_value["publicKey"])
claim_value["publicKey"] = public_key
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
del claim_dictionary[SIGNATURE]
claim_dictionary[claim_type] = claim_value
return claim_dictionary, detached_signature
def decode_b64_fields(claim_dictionary):
"""Decode b64 encoded bytes in protobuf generated dictionary to be given to ClaimDict"""
claim_dictionary = deepcopy(claim_dictionary)
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
claim_value = claim_dictionary[claim_type]
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
claim_value['source']['source'] = base64.b64decode(claim_value['source']['source'])
if 'fee' in claim_value['metadata']:
address = base64.b64decode(claim_value['metadata']['fee']['address'])
claim_value['metadata']['fee']['address'] = address
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
public_key = base64.b64decode(claim_value["publicKey"])
claim_value["publicKey"] = public_key
if SIGNATURE in claim_dictionary:
encoded_sig = base64.b64decode(claim_dictionary[SIGNATURE]['signature'])
encoded_cert_id = base64.b64decode(claim_dictionary[SIGNATURE]['certificateId'])
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
claim_dictionary[claim_type] = claim_value
return claim_dictionary
-22
View File
@@ -1,22 +0,0 @@
class UnknownSourceType(Exception):
pass
class InvalidSourceHashLength(Exception):
pass
class DecodeError(Exception):
pass
class URIParseError(Exception):
pass
class CertificateError(Exception):
pass
class InvalidAddress(Exception):
pass
-79
View File
@@ -1,79 +0,0 @@
from collections import OrderedDict
from lbrynet.schema.address import encode_address, decode_address
from lbrynet.schema.schema import CURRENCY_NAMES, CURRENCY_MAP
from lbrynet.schema.schema.fee import Fee as FeeHelper
from lbrynet.schema.proto import fee_pb2
def migrate(fee):
if len(list(fee.keys())) == 3 and 'currency' in fee and 'amount' in fee and 'address' in fee:
return FeeHelper.load({
"version": "_0_0_1",
"currency": fee['currency'],
"amount": fee['amount'],
"address": decode_address(fee['address'])
})
if len(list(fee.keys())) > 1:
raise Exception("Invalid fee")
currency = list(fee.keys())[0]
amount = fee[currency]['amount']
address = fee[currency]['address']
return FeeHelper.load({
"version": "_0_0_1",
"currency": currency,
"amount": amount,
"address": decode_address(address)
})
class Fee(OrderedDict):
def __init__(self, fee):
if (len(fee) == 4 and "version" in fee and "currency" in fee
and "amount" in fee and "address" in fee):
OrderedDict.__init__(self, fee)
else:
OrderedDict.__init__(self, Fee.load_protobuf(migrate(fee)))
@property
def currency(self):
return self['currency']
@property
def address(self):
return self['address']
@property
def amount(self):
return self['amount']
@property
def version(self):
return self['version']
@property
def protobuf(self):
pb = {
"version": self.version,
"currency": CURRENCY_MAP[self.currency],
"address": decode_address(self.address),
"amount": self.amount
}
return FeeHelper.load(pb)
@classmethod
def load_protobuf(cls, pb):
return cls({
"version": pb.version,
"currency": CURRENCY_NAMES[pb.currency],
"address": encode_address(pb.address),
"amount": pb.amount
})
@classmethod
def deserialize(cls, serialized):
pb = fee_pb2.Fee()
pb.ParseFromString(serialized)
return cls.load_protobuf(pb)
-23
View File
@@ -1,23 +0,0 @@
import hashlib
def sha256(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.sha256(x).digest()
def double_sha256(x):
return sha256(sha256(x))
def ripemd160(x):
if isinstance(x, str):
x = x.encode('utf-8')
md = hashlib.new('ripemd160')
md.update(x)
return md.digest()
def hash160(x):
return ripemd160(sha256(x))
-59
View File
@@ -1,59 +0,0 @@
import jsonschema
class StructuredDict(dict):
"""
A dictionary that enforces a structure specified by a schema, and supports
migration between different versions of the schema.
"""
# To be specified in sub-classes, an array in the format
# [(version, schema, migration), ...]
_versions = []
# Used internally to allow schema lookups by version number
_schemas = {}
version = None
def __init__(self, value, starting_version, migrate=True, target_version=None):
dict.__init__(self, value)
self.version = starting_version
self._schemas = dict([(version, schema) for (version, schema, _) in self._versions])
self.validate(starting_version)
if migrate:
self.migrate(target_version)
def _upgrade_version_range(self, start_version, end_version):
after_starting_version = False
for version, schema, migration in self._versions:
if not after_starting_version:
if version == self.version:
after_starting_version = True
continue
yield version, schema, migration
if end_version and version == end_version:
break
def validate(self, version):
jsonschema.validate(self, self._schemas[version])
def migrate(self, target_version=None):
if target_version:
assert self._versions.index(target_version) > self._versions.index(self.version), \
"Current version is above target version"
for version, schema, migration in self._upgrade_version_range(self.version, target_version):
migration(self)
try:
self.validate(version)
except jsonschema.ValidationError as e:
raise jsonschema.ValidationError(
"Could not migrate to version %s due to validation error: %s" %
(version, e.message))
self.version = version
-276
View File
@@ -1,276 +0,0 @@
VER_001 = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'LBRY metadata schema 0.0.1',
'definitions': {
'fee_info': {
'type': 'object',
'properties': {
'amount': {
'type': 'number',
'minimum': 0,
'exclusiveMinimum': True,
},
'address': {
'type': 'string'
}
},
}
},
'type': 'object',
'properties': {
'ver': {
'type': 'string',
'default': '0.0.1'
},
'title': {
'type': 'string'
},
'description': {
'type': 'string'
},
'author': {
'type': 'string'
},
'language': {
'type': 'string'
},
'license': {
'type': 'string'
},
'content-type': {
'type': 'string'
},
'sources': {
'type': 'object',
'properties': {
'lbry_sd_hash': {
'type': 'string'
},
'btih': {
'type': 'string'
},
'url': {
'type': 'string'
}
},
'additionalProperties': False
},
'thumbnail': {
'type': 'string'
},
'preview': {
'type': 'string'
},
'fee': {
'properties': {
'LBC': {'$ref': '#/definitions/fee_info'},
'BTC': {'$ref': '#/definitions/fee_info'},
'USD': {'$ref': '#/definitions/fee_info'}
}
},
'contact': {
'type': 'number'
},
'pubkey': {
'type': 'string'
},
},
'required': [
'title', 'description', 'author', 'language', 'license', 'content-type', 'sources'],
'additionalProperties': False
}
VER_002 = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'LBRY metadata schema 0.0.2',
'definitions': {
'fee_info': {
'type': 'object',
'properties': {
'amount': {
'type': 'number',
'minimum': 0,
'exclusiveMinimum': True,
},
'address': {
'type': 'string'
}
},
}
},
'type': 'object',
'properties': {
'ver': {
'type': 'string',
'enum': ['0.0.2'],
},
'title': {
'type': 'string'
},
'description': {
'type': 'string'
},
'author': {
'type': 'string'
},
'language': {
'type': 'string'
},
'license': {
'type': 'string'
},
'content-type': {
'type': 'string'
},
'sources': {
'type': 'object',
'properties': {
'lbry_sd_hash': {
'type': 'string'
},
'btih': {
'type': 'string'
},
'url': {
'type': 'string'
}
},
'additionalProperties': False
},
'thumbnail': {
'type': 'string'
},
'preview': {
'type': 'string'
},
'fee': {
'properties': {
'LBC': {'$ref': '#/definitions/fee_info'},
'BTC': {'$ref': '#/definitions/fee_info'},
'USD': {'$ref': '#/definitions/fee_info'}
}
},
'contact': {
'type': 'number'
},
'pubkey': {
'type': 'string'
},
'license_url': {
'type': 'string'
},
'nsfw': {
'type': 'boolean',
'default': False
},
},
'required': [
'ver', 'title', 'description', 'author', 'language', 'license',
'content-type', 'sources', 'nsfw'
],
'additionalProperties': False
}
VER_003 = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'LBRY metadata schema 0.0.3',
'definitions': {
'fee_info': {
'type': 'object',
'properties': {
'amount': {
'type': 'number',
'minimum': 0,
'exclusiveMinimum': True,
},
'address': {
'type': 'string'
}
},
}
},
'type': 'object',
'properties': {
'ver': {
'type': 'string',
'enum': ['0.0.3'],
},
'title': {
'type': 'string'
},
'description': {
'type': 'string'
},
'author': {
'type': 'string'
},
'language': {
'type': 'string'
},
'license': {
'type': 'string'
},
'content_type': {
'type': 'string'
},
'sources': {
'type': 'object',
'properties': {
'lbry_sd_hash': {
'type': 'string'
},
'btih': {
'type': 'string'
},
'url': {
'type': 'string'
}
},
'additionalProperties': False
},
'thumbnail': {
'type': 'string'
},
'preview': {
'type': 'string'
},
'fee': {
'properties': {
'LBC': {'$ref': '#/definitions/fee_info'},
'BTC': {'$ref': '#/definitions/fee_info'},
'USD': {'$ref': '#/definitions/fee_info'}
}
},
'contact': {
'type': 'number'
},
'pubkey': {
'type': 'string'
},
'license_url': {
'type': 'string'
},
'nsfw': {
'type': 'boolean',
'default': False
},
'sig': {
'type': 'string'
}
},
'required': [
'ver', 'title', 'description', 'author', 'language', 'license',
'content_type', 'sources', 'nsfw'
],
'additionalProperties': False,
'dependencies': {
'pubkey': ['sig'],
'sig': ['pubkey']
}
}
-79
View File
@@ -1,79 +0,0 @@
"""
migrate claim json schema (0.0.1-3) to protobuf (0.1.0)
"""
from lbrynet.schema.legacy import metadata_schemas
from lbrynet.schema.claim import ClaimDict
from .StructuredDict import StructuredDict
def migrate_001_to_002(metadata):
metadata['ver'] = '0.0.2'
metadata['nsfw'] = False
def migrate_002_to_003(metadata):
metadata['ver'] = '0.0.3'
if 'content-type' in metadata:
metadata['content_type'] = metadata['content-type']
del metadata['content-type']
class LegacyMetadata(StructuredDict):
current_version = '0.0.3'
_versions = [
('0.0.1', metadata_schemas.VER_001, None),
('0.0.2', metadata_schemas.VER_002, migrate_001_to_002),
('0.0.3', metadata_schemas.VER_003, migrate_002_to_003)
]
def __init__(self, metadata, migrate=True, target_version=None):
if not isinstance(metadata, dict):
raise TypeError("{} is not a dictionary".format(metadata))
starting_version = metadata.get('ver', '0.0.1')
StructuredDict.__init__(self, metadata, starting_version, migrate, target_version)
def migrate_003_to_010(value):
migrated_to_003 = LegacyMetadata(value)
metadata = {
"version": "_0_1_0"
}
for k in ["author", "description", "language", "license", "nsfw", "thumbnail", "title",
"preview"]:
if k in migrated_to_003:
metadata.update({k: migrated_to_003[k]})
if 'license_url' in migrated_to_003:
metadata['licenseUrl'] = migrated_to_003['license_url']
if "fee" in migrated_to_003:
fee = migrated_to_003["fee"]
currency = list(fee.keys())[0]
amount = fee[currency]['amount']
address = fee[currency]['address']
metadata.update(dict(fee={"currency": currency, "version": "_0_0_1",
"amount": amount, "address": address}))
source = {
"source": migrated_to_003['sources']['lbry_sd_hash'],
"contentType": migrated_to_003['content_type'],
"sourceType": "lbry_sd_hash",
"version": "_0_0_1"
}
migrated = {
"version": "_0_0_1",
"claimType": "streamType",
"stream": {
"version": "_0_0_1",
"metadata": metadata,
"source": source
}
}
return ClaimDict.load_dict(migrated)
def migrate(value):
return migrate_003_to_010(value)
@@ -154,5 +154,7 @@ types_map = {
def guess_media_type(path):
_, ext = os.path.splitext(path)
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
return types_map.get(ext, default)
return types_map.get(
ext.strip().lower(),
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
)
-18
View File
@@ -1,18 +0,0 @@
syntax = "proto2";
enum KeyType {
UNKNOWN_PUBLIC_KEY_TYPE = 0;
NIST256p = 1;
NIST384p = 2;
SECP256k1 = 3;
}
message Certificate {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required KeyType keyType = 2;
required bytes publicKey = 4;
}
-22
View File
@@ -1,22 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/stream.proto";
import "lbrynet/schema/proto/certificate.proto";
import "lbrynet/schema/proto/signature.proto";
message Claim {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
enum ClaimType {
UNKNOWN_CLAIM_TYPE = 0;
streamType = 1;
certificateType = 2;
}
required ClaimType claimType = 2;
optional Stream stream = 3;
optional Certificate certificate = 4;
optional Signature publisherSignature = 5;
}
-18
View File
@@ -1,18 +0,0 @@
syntax = "proto2";
message Fee {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
enum Currency {
UNKNOWN_CURRENCY = 0;
LBC = 1;
BTC = 2;
USD = 3;
}
required Version version = 1;
required Currency currency = 2;
required bytes address = 3;
required float amount = 4;
}
-212
View File
@@ -1,212 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/fee.proto";
message Metadata {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
_0_0_2 = 2;
_0_0_3 = 3;
_0_1_0 = 4;
}
enum Language {
UNKNOWN_LANGUAGE = 0;
en = 1;
aa = 2;
ab = 3;
ae = 4;
af = 5;
ak = 6;
am = 7;
an = 8;
ar = 9;
as = 10;
av = 11;
ay = 12;
az = 13;
ba = 14;
be = 15;
bg = 16;
bh = 17;
bi = 18;
bm = 19;
bn = 20;
bo = 21;
br = 22;
bs = 23;
ca = 24;
ce = 25;
ch = 26;
co = 27;
cr = 28;
cs = 29;
cu = 30;
cv = 31;
cy = 32;
da = 33;
de = 34;
dv = 35;
dz = 36;
ee = 37;
el = 38;
eo = 39;
es = 40;
et = 41;
eu = 42;
fa = 43;
ff = 44;
fi = 45;
fj = 46;
fo = 47;
fr = 48;
fy = 49;
ga = 50;
gd = 51;
gl = 52;
gn = 53;
gu = 54;
gv = 55;
ha = 56;
he = 57;
hi = 58;
ho = 59;
hr = 60;
ht = 61;
hu = 62;
hy = 63;
hz = 64;
ia = 65;
id = 66;
ie = 67;
ig = 68;
ii = 69;
ik = 70;
io = 71;
is = 72;
it = 73;
iu = 74;
ja = 75;
jv = 76;
ka = 77;
kg = 78;
ki = 79;
kj = 80;
kk = 81;
kl = 82;
km = 83;
kn = 84;
ko = 85;
kr = 86;
ks = 87;
ku = 88;
kv = 89;
kw = 90;
ky = 91;
la = 92;
lb = 93;
lg = 94;
li = 95;
ln = 96;
lo = 97;
lt = 98;
lu = 99;
lv = 100;
mg = 101;
mh = 102;
mi = 103;
mk = 104;
ml = 105;
mn = 106;
mr = 107;
ms = 108;
mt = 109;
my = 110;
na = 111;
nb = 112;
nd = 113;
ne = 114;
ng = 115;
nl = 116;
nn = 117;
no = 118;
nr = 119;
nv = 120;
ny = 121;
oc = 122;
oj = 123;
om = 124;
or = 125;
os = 126;
pa = 127;
pi = 128;
pl = 129;
ps = 130;
pt = 131;
qu = 132;
rm = 133;
rn = 134;
ro = 135;
ru = 136;
rw = 137;
sa = 138;
sc = 139;
sd = 140;
se = 141;
sg = 142;
si = 143;
sk = 144;
sl = 145;
sm = 146;
sn = 147;
so = 148;
sq = 149;
sr = 150;
ss = 151;
st = 152;
su = 153;
sv = 154;
sw = 155;
ta = 156;
te = 157;
tg = 158;
th = 159;
ti = 160;
tk = 161;
tl = 162;
tn = 163;
to = 164;
tr = 165;
ts = 166;
tt = 167;
tw = 168;
ty = 169;
ug = 170;
uk = 171;
ur = 172;
uz = 173;
ve = 174;
vi = 175;
vo = 176;
wa = 177;
wo = 178;
xh = 179;
yi = 180;
yo = 181;
za = 182;
zh = 183;
zu = 184;
}
required Version version = 1;
required Language language = 2;
required string title = 3;
required string description = 4;
required string author = 5;
required string license = 6;
required bool nsfw = 7;
optional Fee fee = 8;
optional string thumbnail = 9;
optional string preview = 10;
optional string licenseUrl = 11;
}
-14
View File
@@ -1,14 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/certificate.proto";
message Signature {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required KeyType signatureType = 2;
required bytes signature = 3;
required bytes certificateId = 4;
}
-16
View File
@@ -1,16 +0,0 @@
syntax = "proto2";
message Source {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
enum SourceTypes {
UNKNOWN_SOURCE_TYPE = 0;
lbry_sd_hash = 1;
}
required SourceTypes sourceType = 2;
required bytes source = 3;
required string contentType = 4;
}
-15
View File
@@ -1,15 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/metadata.proto";
import "lbrynet/schema/proto/source.proto";
message Stream {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required Metadata metadata = 2;
required Source source = 3;
}
-102
View File
@@ -1,102 +0,0 @@
V_0_0_1 = "_0_0_1"
V_0_0_2 = "_0_0_2"
V_0_0_3 = "_0_0_3"
V_0_1_0 = "_0_1_0"
VERSION_MAP = {
V_0_0_1: 1,
V_0_0_2: 2,
V_0_0_3: 3,
V_0_1_0: 4,
}
VERSION_NAMES = {
1: V_0_0_1,
2: V_0_0_2,
3: V_0_0_3,
4: V_0_1_0
}
LBC = "LBC"
BTC = "BTC"
USD = "USD"
CURRENCY_MAP = {
LBC: 1,
BTC: 2,
USD: 3
}
CURRENCY_NAMES = {
1: LBC,
2: BTC,
3: USD
}
ADDRESS_LENGTH = 25
ADDRESS_CHECKSUM_LENGTH = 4
VERSION = "version"
STREAM_TYPE = "streamType"
CERTIFICATE_TYPE = "certificateType"
CLAIM_TYPE = "claimType"
SIGNATURE = "publisherSignature"
CLAIM_TYPES = {
STREAM_TYPE: "stream",
CERTIFICATE_TYPE: "certificate"
}
CLAIM_TYPE_NAMES = {
1: "stream",
2: "certificate"
}
LBRY_SD_HASH = "lbry_sd_hash"
LBRY_SD_HASH_LENGTH = 48
SOURCE_TYPES = {
LBRY_SD_HASH: 1
}
NIST256p = "NIST256p"
NIST384p = "NIST384p"
SECP256k1 = "SECP256k1"
ECDSA_CURVES = {
NIST256p: 1,
NIST384p: 2,
SECP256k1: 3
}
CURVE_NAMES = {
1: NIST256p,
2: NIST384p,
3: SECP256k1
}
SHA256 = "sha256"
SHA384 = "sha384"
B58_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
assert len(B58_CHARS) == 58
PUBKEY_ADDRESS = 0
SCRIPT_ADDRESS = 5
ADDRESS_PREFIXES = {
"lbrycrd_main": {
PUBKEY_ADDRESS: 85,
SCRIPT_ADDRESS: 122
},
"lbrycrd_regtest": {
PUBKEY_ADDRESS: 111,
SCRIPT_ADDRESS: 196
},
"lbrycrd_testnet": {
PUBKEY_ADDRESS: 111,
SCRIPT_ADDRESS: 196
},
}
-54
View File
@@ -1,54 +0,0 @@
from copy import deepcopy
from lbrynet.schema.proto import certificate_pb2 as cert_pb
from lbrynet.schema.schema.schema import Schema
from lbrynet.schema.schema import VERSION_MAP, V_0_0_1, ECDSA_CURVES, CURVE_NAMES
from lbrynet.schema.validator import get_key_type_from_dem
class _ECDSAKeyHelper(object):
def __init__(self, key):
self._key = key
@property
def der(self):
return self._key.to_der()
@property
def curve_name(self):
return self._key.curve.name
class Certificate(Schema):
@classmethod
def load(cls, message):
_key = deepcopy(message)
_message_pb = cert_pb.Certificate()
if isinstance(_key, dict):
_message_pb.publicKey = _key.pop("publicKey")
_message_pb.version = VERSION_MAP[_key.pop("version")]
_message_pb.keyType = ECDSA_CURVES[_key.pop("keyType")]
else:
_message_pb.version = _key.version
_message_pb.keyType = _key.keyType
_message_pb.publicKey = _key.publicKey
if _message_pb.keyType not in CURVE_NAMES:
raise Exception("Unknown curve")
if get_key_type_from_dem(_message_pb.publicKey) != _message_pb.keyType:
raise Exception("Curve mismatch")
return cls._load(_key, _message_pb)
@classmethod
def load_from_key_obj(cls, key, key_type):
if key_type in ECDSA_CURVES:
_key = _ECDSAKeyHelper(key)
else:
raise Exception("Unknown key type: %s" % str(type(key)))
if key_type != _key.curve_name:
raise Exception("Curve mismatch")
msg = {
"version": V_0_0_1,
"keyType": key_type,
"publicKey": _key.der,
}
return cls.load(msg)
-51
View File
@@ -1,51 +0,0 @@
from copy import deepcopy
from lbrynet.schema.proto import claim_pb2 as claim_pb
from lbrynet.schema.schema import VERSION_MAP
from lbrynet.schema.schema.signature import Signature
from lbrynet.schema.schema.certificate import Certificate
from lbrynet.schema.schema.schema import Schema
from lbrynet.schema.schema.stream import Stream
class Claim(Schema):
CLAIM_TYPE_STREAM = 1
CLAIM_TYPE_CERT = 2
@classmethod
def load(cls, message):
_claim = deepcopy(message)
_message_pb = claim_pb.Claim()
_message_pb.version = VERSION_MAP[_claim.pop("version")]
if "certificate" in _claim:
_cert = _claim.pop("certificate")
if isinstance(_cert, dict):
cert = Certificate.load(_cert)
else:
cert = _cert
claim_type = Claim.CLAIM_TYPE_CERT
_message_pb.certificate.MergeFrom(cert)
elif "stream" in _claim:
_stream = _claim.pop("stream")
if isinstance(_stream, dict):
stream = Stream.load(_stream)
else:
stream = _stream
claim_type = Claim.CLAIM_TYPE_STREAM
_message_pb.stream.MergeFrom(stream)
else:
raise AttributeError
_message_pb.claimType = claim_type
if "publisherSignature" in _claim:
_publisherSignature = _claim.pop("publisherSignature")
if isinstance(_publisherSignature, dict):
publisherSignature = Signature.load(_publisherSignature)
else:
publisherSignature = _publisherSignature
_message_pb.publisherSignature.MergeFrom(publisherSignature)
return cls._load(_claim, _message_pb)
-17
View File
@@ -1,17 +0,0 @@
from copy import deepcopy
from lbrynet.schema.schema.schema import Schema
from lbrynet.schema.proto import fee_pb2 as fee_pb
from lbrynet.schema.schema import VERSION_MAP, CURRENCY_MAP
class Fee(Schema):
@classmethod
def load(cls, message):
_fee = deepcopy(message)
currency = CURRENCY_MAP[_fee.pop('currency')]
_message_pb = fee_pb.Fee()
_message_pb.version = VERSION_MAP[_fee.pop("version")]
_message_pb.currency = currency
_message_pb.address = _fee.pop('address')
return cls._load(_fee, _message_pb)
-17
View File
@@ -1,17 +0,0 @@
from copy import deepcopy
from lbrynet.schema.proto import metadata_pb2 as metadata_pb
from lbrynet.schema.schema.fee import Fee
from lbrynet.schema.schema.schema import Schema
from lbrynet.schema.schema import VERSION_MAP
class Metadata(Schema):
@classmethod
def load(cls, message):
_metadata = deepcopy(message)
_message_pb = metadata_pb.Metadata()
_message_pb.version = VERSION_MAP[_metadata.pop("version")]
if 'fee' in _metadata:
fee_pb = Fee.load(_metadata.pop('fee'))
_message_pb.fee.CopyFrom(fee_pb)
return cls._load(_metadata, _message_pb)
-15
View File
@@ -1,15 +0,0 @@
import json
import google.protobuf.json_format as json_pb # pylint: disable=no-name-in-module
from google.protobuf.message import Message # pylint: disable=no-name-in-module,import-error
class Schema(Message):
@classmethod
def load(cls, message):
raise NotImplementedError
@classmethod
def _load(cls, data, message):
if isinstance(data, dict):
data = json.dumps(data)
return json_pb.Parse(data, message)
-17
View File
@@ -1,17 +0,0 @@
from copy import deepcopy
from lbrynet.schema.proto import signature_pb2 as signature_pb
from lbrynet.schema.schema import VERSION_MAP, ECDSA_CURVES
from lbrynet.schema.schema.schema import Schema
class Signature(Schema):
@classmethod
def load(cls, message):
_signature = deepcopy(message)
_message_pb = signature_pb.Signature()
_message_pb.version = VERSION_MAP[_signature.pop("version")]
_message_pb.signatureType = ECDSA_CURVES[_signature.pop("signatureType")]
_message_pb.certificateId = _signature.pop("certificateId")
_message_pb.signature = _signature.pop("signature")
return cls._load(_signature, _message_pb)
-19
View File
@@ -1,19 +0,0 @@
from copy import deepcopy
from lbrynet.schema.proto import source_pb2 as source_pb
from lbrynet.schema.schema import SOURCE_TYPES, LBRY_SD_HASH_LENGTH, VERSION_MAP
from lbrynet.schema.schema.schema import Schema
from lbrynet.schema.error import InvalidSourceHashLength
class Source(Schema):
@classmethod
def load(cls, message):
_source = deepcopy(message)
sd_hash = _source.pop('source')
assert len(sd_hash) == LBRY_SD_HASH_LENGTH, InvalidSourceHashLength(len(sd_hash))
_message_pb = source_pb.Source()
_message_pb.version = VERSION_MAP[_source.pop("version")]
_message_pb.sourceType = SOURCE_TYPES[_source.pop('sourceType')]
_message_pb.source = sd_hash
_message_pb.contentType = _source.pop('contentType')
return cls._load(_source, _message_pb)
-20
View File
@@ -1,20 +0,0 @@
from copy import deepcopy
from lbrynet.schema.schema.source import Source
from lbrynet.schema.proto import stream_pb2 as stream_pb
from lbrynet.schema.schema import VERSION_MAP
from lbrynet.schema.schema.metadata import Metadata
from lbrynet.schema.schema.schema import Schema
class Stream(Schema):
@classmethod
def load(cls, message):
_claim = deepcopy(message)
source = Source.load(_claim.pop('source'))
metadata = Metadata.load(_claim.pop('metadata'))
_message_pb = stream_pb.Stream()
_message_pb.version = VERSION_MAP[_claim.pop("version")]
_message_pb.source.CopyFrom(source)
_message_pb.metadata.CopyFrom(metadata)
return cls._load(_claim, _message_pb)
-44
View File
@@ -1,44 +0,0 @@
from collections import namedtuple
from typing import Union
LEGACY = namedtuple('Legacy', 'payload')
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
FLAGS = {
LEGACY: 0x80,
NAMED_SECP256K1: 0x01
}
class Signature:
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
self.data = data
@property
def payload(self):
return self.data.payload
@property
def certificate_id(self):
if type(self.data) == NAMED_SECP256K1:
return self.data.certificate_id
@property
def raw_signature(self):
if type(self.data) == NAMED_SECP256K1:
return self.data.raw_signature
@classmethod
def flagged_parse(cls, binary: bytes):
flag = binary[0]
if flag == FLAGS[NAMED_SECP256K1]:
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
else:
return cls(LEGACY(binary))
@property
def serialized(self):
if isinstance(self.data, NAMED_SECP256K1):
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
elif isinstance(self.data, LEGACY):
return self.payload
-121
View File
@@ -1,121 +0,0 @@
import ecdsa
import hashlib
import binascii
from lbrynet.schema.address import decode_address
from lbrynet.schema.encoding import decode_b64_fields
from lbrynet.schema.schema.certificate import Certificate
from lbrynet.schema.schema.claim import Claim
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
from lbrynet.schema.validator import validate_claim_id
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
class NIST_ECDSASigner(object):
CURVE = None
CURVE_NAME = None
HASHFUNC = hashlib.sha256
HASHFUNC_NAME = SHA256
def __init__(self, private_key):
self._private_key = private_key
@property
def private_key(self):
return self._private_key
@property
def public_key(self):
return self.private_key.get_verifying_key()
@property
def certificate(self):
certificate_claim = {
VERSION: V_0_0_1,
CLAIM_TYPE: CERTIFICATE_TYPE,
CLAIM_TYPES[CERTIFICATE_TYPE]: Certificate.load_from_key_obj(self.public_key,
self.CURVE_NAME)
}
return Claim.load(certificate_claim)
@classmethod
def load_pem(cls, pem_string):
return cls(ecdsa.SigningKey.from_pem(pem_string, hashfunc=cls.HASHFUNC_NAME))
@classmethod
def generate(cls):
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
def sign(self, *fields):
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, detached=False):
validate_claim_id(cert_claim_id)
raw_cert_id = binascii.unhexlify(cert_claim_id)
decoded_addr = decode_address(claim_address)
if detached:
assert name, "Name is required for detached signatures"
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
signature = self.sign(
name.lower().encode(),
decoded_addr,
claim.serialized_no_signature,
raw_cert_id,
)
else:
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
if detached:
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
signature,
raw_cert_id,
claim.serialized_no_signature
))
# -- Legacy signer (signature inside protobuf) --
if not isinstance(self.private_key, ecdsa.SigningKey):
raise Exception("Not given a signing key")
sig_dict = {
"version": V_0_0_1,
"signatureType": self.CURVE_NAME,
"signature": signature,
"certificateId": raw_cert_id
}
msg = {
"version": V_0_0_1,
"stream": decode_b64_fields(claim.protobuf_dict)['stream'],
"publisherSignature": sig_dict
}
proto = Claim.load(msg)
return proto, Signature.flagged_parse(proto.SerializeToString())
class NIST256pSigner(NIST_ECDSASigner):
CURVE = ecdsa.NIST256p
CURVE_NAME = NIST256p
class NIST384pSigner(NIST_ECDSASigner):
CURVE = ecdsa.NIST384p
CURVE_NAME = NIST384p
HASHFUNC = hashlib.sha384
HASHFUNC_NAME = SHA384
class SECP256k1Signer(NIST_ECDSASigner):
CURVE = ecdsa.SECP256k1
CURVE_NAME = SECP256k1
def get_signer(curve):
if curve == NIST256p:
return NIST256pSigner
elif curve == NIST384p:
return NIST384pSigner
elif curve == SECP256k1:
return SECP256k1Signer
else:
raise Exception("Unknown curve: %s" % str(curve))
+6
View File
@@ -0,0 +1,6 @@
from lbrynet.schema.base import Signable
class Support(Signable):
__slots__ = ()
message_class = None # TODO: add support protobufs
File renamed without changes.
File renamed without changes.
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/certificate.proto
# source: certificate.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/certificate.proto',
package='',
name='certificate.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
)
_KEYTYPE = _descriptor.EnumDescriptor(
name='KeyType',
full_name='KeyType',
full_name='legacy_pb.KeyType',
filename=None,
file=DESCRIPTOR,
values=[
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=187,
serialized_end=268,
serialized_start=197,
serialized_end=278,
)
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
@@ -62,7 +62,7 @@ SECP256k1 = 3
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Certificate.Version',
full_name='legacy_pb.Certificate.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=143,
serialized_end=185,
serialized_start=153,
serialized_end=195,
)
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
_CERTIFICATE = _descriptor.Descriptor(
name='Certificate',
full_name='Certificate',
full_name='legacy_pb.Certificate',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Certificate.version', index=0,
name='version', full_name='legacy_pb.Certificate.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='keyType', full_name='Certificate.keyType', index=1,
name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='publicKey', full_name='Certificate.publicKey', index=2,
name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
number=4, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=43,
serialized_end=185,
serialized_start=33,
serialized_end=195,
)
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
DESCRIPTOR = _CERTIFICATE,
__module__ = 'lbrynet.schema.proto.certificate_pb2'
# @@protoc_insertion_point(class_scope:Certificate)
__module__ = 'certificate_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
))
_sym_db.RegisterMessage(Certificate)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/fee.proto
# source: fee.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/fee.proto',
package='',
name='fee.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
)
_FEE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Fee.Version',
full_name='legacy_pb.Fee.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=139,
serialized_end=181,
serialized_start=149,
serialized_end=191,
)
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
_FEE_CURRENCY = _descriptor.EnumDescriptor(
name='Currency',
full_name='Fee.Currency',
full_name='legacy_pb.Fee.Currency',
filename=None,
file=DESCRIPTOR,
values=[
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=183,
serialized_end=242,
serialized_start=193,
serialized_end=252,
)
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
_FEE = _descriptor.Descriptor(
name='Fee',
full_name='Fee',
full_name='legacy_pb.Fee',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Fee.version', index=0,
name='version', full_name='legacy_pb.Fee.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='currency', full_name='Fee.currency', index=1,
name='currency', full_name='legacy_pb.Fee.currency', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='Fee.address', index=2,
name='address', full_name='legacy_pb.Fee.address', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='amount', full_name='Fee.amount', index=3,
name='amount', full_name='legacy_pb.Fee.amount', index=3,
number=4, type=2, cpp_type=6, label=2,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=35,
serialized_end=242,
serialized_start=25,
serialized_end=252,
)
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
DESCRIPTOR = _FEE,
__module__ = 'lbrynet.schema.proto.fee_pb2'
# @@protoc_insertion_point(class_scope:Fee)
__module__ = 'fee_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Fee)
))
_sym_db.RegisterMessage(Fee)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/claim.proto
# source: legacy_claim.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
from . import stream_pb2 as stream__pb2
from . import certificate_pb2 as certificate__pb2
from . import signature_pb2 as signature__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/claim.proto',
package='',
name='legacy_claim.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
_CLAIM_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Claim.Version',
full_name='legacy_pb.Claim.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=329,
serialized_end=371,
serialized_start=313,
serialized_end=355,
)
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
name='ClaimType',
full_name='Claim.ClaimType',
full_name='legacy_pb.Claim.ClaimType',
filename=None,
file=DESCRIPTOR,
values=[
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=373,
serialized_end=445,
serialized_start=357,
serialized_end=429,
)
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
_CLAIM = _descriptor.Descriptor(
name='Claim',
full_name='Claim',
full_name='legacy_pb.Claim',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Claim.version', index=0,
name='version', full_name='legacy_pb.Claim.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claimType', full_name='Claim.claimType', index=1,
name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='stream', full_name='Claim.stream', index=2,
name='stream', full_name='legacy_pb.Claim.stream', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='certificate', full_name='Claim.certificate', index=3,
name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=150,
serialized_end=445,
serialized_start=84,
serialized_end=429,
)
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
_CLAIM_VERSION.containing_type = _CLAIM
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
DESCRIPTOR = _CLAIM,
__module__ = 'lbrynet.schema.proto.claim_pb2'
# @@protoc_insertion_point(class_scope:Claim)
__module__ = 'legacy_claim_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Claim)
))
_sym_db.RegisterMessage(Claim)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/metadata.proto
# source: metadata.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
from . import fee_pb2 as fee__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/metadata.proto',
package='',
name='metadata.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02tyLine truncated
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
dependencies=[fee__pb2.DESCRIPTOR,])
_METADATA_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Metadata.Version',
full_name='legacy_pb.Metadata.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=316,
serialized_end=394,
serialized_start=315,
serialized_end=393,
)
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
name='Language',
full_name='Metadata.Language',
full_name='legacy_pb.Metadata.Language',
filename=None,
file=DESCRIPTOR,
values=[
@@ -809,91 +809,91 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=397,
serialized_end=1958,
serialized_start=396,
serialized_end=1957,
)
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
_METADATA = _descriptor.Descriptor(
name='Metadata',
full_name='Metadata',
full_name='legacy_pb.Metadata',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Metadata.version', index=0,
name='version', full_name='legacy_pb.Metadata.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='language', full_name='Metadata.language', index=1,
name='language', full_name='legacy_pb.Metadata.language', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='Metadata.title', index=2,
name='title', full_name='legacy_pb.Metadata.title', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='Metadata.description', index=3,
name='description', full_name='legacy_pb.Metadata.description', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='author', full_name='Metadata.author', index=4,
name='author', full_name='legacy_pb.Metadata.author', index=4,
number=5, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license', full_name='Metadata.license', index=5,
name='license', full_name='legacy_pb.Metadata.license', index=5,
number=6, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='nsfw', full_name='Metadata.nsfw', index=6,
name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6,
number=7, type=8, cpp_type=7, label=2,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='fee', full_name='Metadata.fee', index=7,
name='fee', full_name='legacy_pb.Metadata.fee', index=7,
number=8, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail', full_name='Metadata.thumbnail', index=8,
name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='preview', full_name='Metadata.preview', index=9,
name='preview', full_name='legacy_pb.Metadata.preview', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10,
number=11, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -913,13 +913,13 @@ _METADATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=72,
serialized_end=1958,
serialized_start=41,
serialized_end=1957,
)
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
_METADATA_VERSION.containing_type = _METADATA
_METADATA_LANGUAGE.containing_type = _METADATA
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
@@ -927,8 +927,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
DESCRIPTOR = _METADATA,
__module__ = 'lbrynet.schema.proto.metadata_pb2'
# @@protoc_insertion_point(class_scope:Metadata)
__module__ = 'metadata_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Metadata)
))
_sym_db.RegisterMessage(Metadata)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/signature.proto
# source: signature.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
from . import certificate_pb2 as certificate__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/signature.proto',
package='',
name='signature.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
dependencies=[certificate__pb2.DESCRIPTOR,])
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Signature.Version',
full_name='legacy_pb.Signature.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=206,
serialized_end=248,
serialized_start=195,
serialized_end=237,
)
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
_SIGNATURE = _descriptor.Descriptor(
name='Signature',
full_name='Signature',
full_name='legacy_pb.Signature',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Signature.version', index=0,
name='version', full_name='legacy_pb.Signature.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='signatureType', full_name='Signature.signatureType', index=1,
name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='signature', full_name='Signature.signature', index=2,
name='signature', full_name='legacy_pb.Signature.signature', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='certificateId', full_name='Signature.certificateId', index=3,
name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
number=4, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=81,
serialized_end=248,
serialized_start=50,
serialized_end=237,
)
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
_SIGNATURE_VERSION.containing_type = _SIGNATURE
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
DESCRIPTOR = _SIGNATURE,
__module__ = 'lbrynet.schema.proto.signature_pb2'
# @@protoc_insertion_point(class_scope:Signature)
__module__ = 'signature_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Signature)
))
_sym_db.RegisterMessage(Signature)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/source.proto
# source: source.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/source.proto',
package='',
name='source.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
)
_SOURCE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Source.Version',
full_name='legacy_pb.Source.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=160,
serialized_end=202,
serialized_start=170,
serialized_end=212,
)
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
name='SourceTypes',
full_name='Source.SourceTypes',
full_name='legacy_pb.Source.SourceTypes',
filename=None,
file=DESCRIPTOR,
values=[
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=204,
serialized_end=260,
serialized_start=214,
serialized_end=270,
)
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
_SOURCE = _descriptor.Descriptor(
name='Source',
full_name='Source',
full_name='legacy_pb.Source',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Source.version', index=0,
name='version', full_name='legacy_pb.Source.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sourceType', full_name='Source.sourceType', index=1,
name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source', full_name='Source.source', index=2,
name='source', full_name='legacy_pb.Source.source', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='contentType', full_name='Source.contentType', index=3,
name='contentType', full_name='legacy_pb.Source.contentType', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=38,
serialized_end=260,
serialized_start=28,
serialized_end=270,
)
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
DESCRIPTOR = _SOURCE,
__module__ = 'lbrynet.schema.proto.source_pb2'
# @@protoc_insertion_point(class_scope:Source)
__module__ = 'source_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Source)
))
_sym_db.RegisterMessage(Source)
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/stream.proto
# source: stream.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
from . import metadata_pb2 as metadata__pb2
from . import source_pb2 as source__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/stream.proto',
package='',
name='stream.proto',
package='legacy_pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
_STREAM_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Stream.Version',
full_name='legacy_pb.Stream.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=208,
serialized_end=250,
serialized_start=186,
serialized_end=228,
)
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
_STREAM = _descriptor.Descriptor(
name='Stream',
full_name='Stream',
full_name='legacy_pb.Stream',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Stream.version', index=0,
name='version', full_name='legacy_pb.Stream.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='metadata', full_name='Stream.metadata', index=1,
name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
number=2, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source', full_name='Stream.source', index=2,
name='source', full_name='legacy_pb.Stream.source', index=2,
number=3, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=110,
serialized_end=250,
serialized_start=58,
serialized_end=228,
)
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
_STREAM_VERSION.containing_type = _STREAM
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
DESCRIPTOR = _STREAM,
__module__ = 'lbrynet.schema.proto.stream_pb2'
# @@protoc_insertion_point(class_scope:Stream)
__module__ = 'stream_pb2'
# @@protoc_insertion_point(class_scope:legacy_pb.Stream)
))
_sym_db.RegisterMessage(Stream)
File renamed without changes.
File diff suppressed because it is too large. Load diff
+5 -2
View File
@@ -1,5 +1,4 @@
import re
from lbrynet.schema.error import URIParseError
PROTOCOL = 'lbry://'
CHANNEL_CHAR = '@'
@@ -13,6 +12,10 @@ CLAIM_ID_MAX_LENGTH = 40
CHANNEL_NAME_MIN_LENGTH = 1
class URIParseError(Exception):
pass
class URI(object):
__slots__ = ['name', 'claim_sequence', 'bid_position', 'claim_id', 'path']
@@ -139,7 +142,7 @@ def get_schema_regex():
protocol = _named("protocol", re.escape(PROTOCOL))
# Define basic building blocks
valid_name_char = "[a-zA-Z0-9\-]" # these characters are the only valid name characters
valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/
name_content = valid_name_char + '+'
name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
-142
View File
@@ -1,142 +0,0 @@
from string import hexdigits
import ecdsa
import hashlib
import binascii
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import load_der_public_key
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
from cryptography.exceptions import InvalidSignature
from ecdsa.util import sigencode_der
from lbrynet.schema.address import decode_address
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, ECDSA_CURVES, CURVE_NAMES
def validate_claim_id(claim_id):
if not len(claim_id) == 40:
raise Exception("Incorrect claimid length: %i" % len(claim_id))
if isinstance(claim_id, bytes):
claim_id = claim_id.decode('utf-8')
if set(claim_id).difference(hexdigits):
raise Exception("Claim id is not hex encoded")
class Validator:
CURVE_NAME = None
HASHFUNC = hashlib.sha256
def __init__(self, public_key, certificate_claim_id):
validate_claim_id(certificate_claim_id)
if CURVE_NAMES.get(get_key_type_from_dem(public_key)) != self.CURVE_NAME:
raise Exception("Curve mismatch")
self._public_key = public_key
self._certificate_claim_id = certificate_claim_id
@property
def public_key(self):
return self._public_key
@property
def certificate_claim_id(self):
return self._certificate_claim_id
@classmethod
def signing_key_from_pem(cls, pem):
return ecdsa.SigningKey.from_pem(pem, hashfunc=cls.HASHFUNC)
@classmethod
def signing_key_from_der(cls, der):
return ecdsa.SigningKey.from_der(der, hashfunc=cls.HASHFUNC)
@classmethod
def load_from_certificate(cls, certificate_claim, certificate_claim_id):
certificate = certificate_claim.certificate
return cls(certificate.publicKey, certificate_claim_id)
def validate_signature(self, digest, signature):
public_key = load_der_public_key(self.public_key, default_backend())
if len(signature) == 64:
hash = hashes.SHA256()
elif len(signature) == 96:
hash = hashes.SHA384()
signature = binascii.hexlify(signature)
r = int(signature[:int(len(signature)/2)], 16)
s = int(signature[int(len(signature)/2):], 16)
encoded_sig = sigencode_der(r, s, len(signature)*4)
try:
public_key.verify(encoded_sig, digest, ec.ECDSA(Prehashed(hash)))
return True
except InvalidSignature:
# TODO Fixme. This is what is expected today on the outer calls. This should be implementation independent
# but requires changing everything calling that
from ecdsa import BadSignatureError
raise BadSignatureError
def validate_claim_signature(self, claim, claim_address, name):
to_sign = bytearray()
if claim.detached_signature and claim.detached_signature.raw_signature:
assert name is not None, "Name is required for verifying detached signatures."
to_sign.extend(name.lower().encode())
signature = claim.detached_signature.raw_signature
payload = claim.detached_signature.payload
else:
# extract and serialize the stream from the claim, then check the signature
signature = binascii.unhexlify(claim.signature)
payload = claim.serialized_no_signature
decoded_address = decode_address(claim_address)
if signature is None:
raise Exception("No signature to validate")
to_sign.extend(decoded_address)
to_sign.extend(payload)
to_sign.extend(binascii.unhexlify(self.certificate_claim_id))
return self.validate_signature(self.HASHFUNC(to_sign).digest(), signature)
def validate_private_key(self, private_key):
if not isinstance(private_key, ecdsa.SigningKey):
raise TypeError("Not given a signing key, given a %s" % str(type(private_key)))
return private_key.get_verifying_key().to_der() == self.public_key
class NIST256pValidator(Validator):
CURVE_NAME = NIST256p
HASHFUNC = hashlib.sha256
class NIST384pValidator(Validator):
CURVE_NAME = NIST384p
HASHFUNC = hashlib.sha384
class SECP256k1Validator(Validator):
CURVE_NAME = SECP256k1
HASHFUNC = hashlib.sha256
def get_validator(curve):
if curve == NIST256p:
return NIST256pValidator
elif curve == NIST384p:
return NIST384pValidator
elif curve == SECP256k1:
return SECP256k1Validator
else:
raise Exception("Unknown curve: %s" % str(curve))
def get_key_type_from_dem(pubkey_dem):
name = serialization.load_der_public_key(pubkey_dem, default_backend()).curve.name
if name == 'secp256k1':
return ECDSA_CURVES[SECP256k1]
elif name == 'secp256r1':
return ECDSA_CURVES[NIST256p]
elif name == 'secp384r1':
return ECDSA_CURVES[NIST384p]
raise Exception("unexpected curve: %s" % name)
+68 -33
View File
@@ -15,13 +15,13 @@ log = logging.getLogger(__name__)
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
base_name, ext = os.path.splitext(file_name)
base_name, ext = os.path.splitext(os.path.basename(file_name))
i = 0
while os.path.isfile(os.path.join(download_directory, file_name)):
i += 1
file_name = "%s_%i%s" % (base_name, i, ext)
return os.path.join(download_directory, file_name)
return file_name
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
@@ -29,7 +29,9 @@ async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_dir
class StreamAssembler:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
output_file_name: typing.Optional[str] = None):
self.output_file_name = output_file_name
self.loop = loop
self.blob_manager = blob_manager
self.sd_hash = sd_hash
@@ -43,11 +45,11 @@ class StreamAssembler:
self.written_bytes: int = 0
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
if not blob or not self.stream_handle or self.stream_handle.closed:
return False
def _decrypt_and_write():
if self.stream_handle.closed:
return False
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
self.stream_handle.seek(offset)
_decrypted = blob.decrypt(
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
@@ -55,47 +57,80 @@ class StreamAssembler:
self.stream_handle.write(_decrypted)
self.stream_handle.flush()
self.written_bytes += len(_decrypted)
return True
log.debug("decrypted %s", blob.blob_hash[:8])
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
if decrypted:
log.info("decrypted %s", blob.blob_hash[:8])
return
await self.loop.run_in_executor(None, _decrypt_and_write)
return True
async def setup(self):
pass
async def after_got_descriptor(self):
pass
async def after_finished(self):
pass
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
if not os.path.isdir(output_dir):
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
await self.setup()
self.sd_blob = await self.get_blob(self.sd_hash)
await self.blob_manager.blob_completed(self.sd_blob)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
self.sd_blob)
await self.after_got_descriptor()
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
self.output_path = os.path.join(output_dir, self.output_file_name)
if not self.got_descriptor.is_set():
self.got_descriptor.set()
self.output_path = await get_next_available_file_name(self.loop, output_dir,
output_file_name or self.descriptor.suggested_file_name)
self.stream_handle = open(self.output_path, 'wb')
await self.blob_manager.storage.store_stream(
self.sd_blob, self.descriptor
)
await self.blob_manager.blob_completed(self.sd_blob)
written_blobs = None
save_tasks = []
try:
for blob_info in self.descriptor.blobs[:-1]:
while True:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
break
except ValueError as err:
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
self.descriptor.sd_hash, str(err))
continue
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
self.stream_finished_event.set()
with open(self.output_path, 'wb') as stream_handle:
self.stream_handle = stream_handle
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
if blob_info.blob_num != i:
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
return
while self.stream_handle and not self.stream_handle.closed:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
if blob and blob.length != blob_info.length:
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
await self.blob_manager.delete_blobs([blob_info.blob_hash])
continue
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
written_blobs = i
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
break
except FileNotFoundError:
log.debug("stream assembler stopped")
return
except (ValueError, IOError, OSError):
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
self.descriptor.sd_hash)
continue
finally:
self.stream_handle.close()
if written_blobs == len(self.descriptor.blobs) - 2:
log.debug("finished decrypting and assembling stream")
if save_tasks:
await asyncio.wait(save_tasks)
await self.after_finished()
self.stream_finished_event.set()
else:
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
len(self.descriptor.blobs) - 2)
if self.output_path and os.path.isfile(self.output_path):
log.debug("erasing incomplete file assembly: %s", self.output_path)
os.unlink(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
f = asyncio.Future(loop=self.loop)
f.set_result(self.blob_manager.get_blob(blob_hash, length))
return await f
return self.blob_manager.get_blob(blob_hash, length)
+67 -9
View File
@@ -4,6 +4,7 @@ import binascii
import logging
import typing
import asyncio
from collections import OrderedDict
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from lbrynet.blob import MAX_BLOB_SIZE
from lbrynet.blob.blob_info import BlobInfo
@@ -58,6 +59,10 @@ class StreamDescriptor:
self.stream_hash = stream_hash or self.get_stream_hash()
self.sd_hash = sd_hash
@property
def length(self):
return len(self.as_json())
def get_stream_hash(self) -> str:
return self.calculate_stream_hash(
binascii.hexlify(self.stream_name.encode()), self.key.encode(),
@@ -78,15 +83,46 @@ class StreamDescriptor:
[blob_info.as_dict() for blob_info in self.blobs]), sort_keys=True
).encode()
async def make_sd_blob(self):
sd_hash = self.calculate_sd_hash()
sd_data = self.as_json()
sd_blob = BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
def old_sort_json(self) -> bytes:
blobs = []
for b in self.blobs:
blobs.append(OrderedDict(
[('length', b.length), ('blob_num', b.blob_num), ('iv', b.iv)] if not b.blob_hash else
[('length', b.length), ('blob_num', b.blob_num), ('blob_hash', b.blob_hash), ('iv', b.iv)]
))
if not b.blob_hash:
break
return json.dumps(
OrderedDict([
('stream_name', binascii.hexlify(self.stream_name.encode()).decode()),
('blobs', blobs),
('stream_type', 'lbryfile'),
('key', self.key),
('suggested_file_name', binascii.hexlify(self.suggested_file_name.encode()).decode()),
('stream_hash', self.stream_hash),
])
).encode()
def calculate_old_sort_sd_hash(self):
h = get_lbry_hash_obj()
h.update(self.old_sort_json())
return h.hexdigest()
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
old_sort: typing.Optional[bool] = False):
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
if not old_sort:
sd_data = self.as_json()
else:
sd_data = self.old_sort_json()
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
if blob_file_obj:
blob_file_obj.set_length(len(sd_data))
if not sd_blob.get_is_verified():
writer = sd_blob.open_for_writing()
writer.write(sd_data)
await sd_blob.verified.wait()
await sd_blob.close()
sd_blob.close()
return sd_blob
@classmethod
@@ -95,13 +131,19 @@ class StreamDescriptor:
assert os.path.isfile(blob.file_path)
with open(blob.file_path, 'rb') as f:
json_bytes = f.read()
decoded = json.loads(json_bytes.decode())
try:
decoded = json.loads(json_bytes.decode())
except json.JSONDecodeError:
blob.delete()
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
if decoded['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in decoded['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
descriptor = cls(
loop, blob_dir,
binascii.unhexlify(decoded['stream_name']).decode(),
@@ -154,8 +196,8 @@ class StreamDescriptor:
@classmethod
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None
) -> 'StreamDescriptor':
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
old_sort: bool = False) -> 'StreamDescriptor':
blobs: typing.List[BlobInfo] = []
@@ -174,7 +216,7 @@ class StreamDescriptor:
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
blobs
)
sd_blob = await descriptor.make_sd_blob()
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
descriptor.sd_hash = sd_blob.blob_hash
return descriptor
@@ -184,3 +226,19 @@ class StreamDescriptor:
def upper_bound_decrypted_length(self) -> int:
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
@classmethod
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
suggested_file_name: str, key: str,
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
blobs, stream_hash, sd_blob.blob_hash)
if descriptor.calculate_sd_hash() == sd_blob.blob_hash: # first check for a normal valid sd
old_sort = False
elif descriptor.calculate_old_sort_sd_hash() == sd_blob.blob_hash: # check if old field sorting works
old_sort = True
else:
return
await descriptor.make_sd_blob(sd_blob, old_sort)
return descriptor
+78 -205
View File
@@ -2,12 +2,14 @@ import os
import asyncio
import typing
import logging
from lbrynet.utils import drain_tasks, cancel_task
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
@@ -19,217 +21,88 @@ def drain_into(a: list, b: list):
b.append(a.pop())
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
output_file_name: typing.Optional[str] = None,
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
max_connections_per_stream: typing.Optional[int] = 8):
super().__init__(loop, blob_manager, sd_hash)
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.current_blob: 'BlobFile' = None
self.download_task: asyncio.Task = None
self.accumulate_connections_task: asyncio.Task = None
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.output_dir = output_dir or os.getcwd()
class StreamDownloader(StreamAssembler):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
super().__init__(loop, blob_manager, sd_hash, output_file_name)
self.config = config
self.output_dir = output_dir or self.config.download_dir
self.output_file_name = output_file_name
self._lock = asyncio.Lock(loop=self.loop)
self.max_connections_per_stream = max_connections_per_stream
self.fixed_peers = fixed_peers or []
self.blob_downloader: typing.Optional[BlobDownloader] = None
self.search_queue = asyncio.Queue(loop=loop)
self.peer_queue = asyncio.Queue(loop=loop)
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.descriptor: typing.Optional[StreamDescriptor]
self.node: typing.Optional['Node'] = None
self.assemble_task: typing.Optional[asyncio.Task] = None
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
self.fixed_peers_delay: typing.Optional[float] = None
self.added_fixed_peers = False
async def _update_current_blob(self, blob: 'BlobFile'):
async with self._lock:
drain_tasks(self.running_download_requests)
self.current_blob = blob
if not blob.get_is_verified():
self._update_requests()
async def setup(self): # start the peer accumulator and initialize the downloader
if self.blob_downloader:
raise Exception("downloader is already set up")
if self.node:
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.search_queue.put_nowait(self.sd_hash)
async def _request_blob(self, peer: 'KademliaPeer'):
if self.current_blob.get_is_verified():
log.info("already verified")
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
peer.address, peer.tcp_port, self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self._lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
async def after_got_descriptor(self):
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to search")
def _update_requests(self):
self.new_peer_event.clear()
if self.current_blob.blob_hash not in self.requested_from:
self.requested_from[self.current_blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.current_blob.blob_hash][peer] = task
self.running_download_requests.append(task)
async def after_finished(self):
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
self.blob_downloader.close()
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
async with self._lock:
if len(self.running_download_requests) < self.max_connections_per_stream:
# update the running download requests
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
async with self._lock:
if self.current_blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.current_blob
else:
for task in download_tasks:
if task and not task.done():
self.running_download_requests.append(task)
return
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.assemble_task:
self.assemble_task.cancel()
self.assemble_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader = None
if self.stream_handle:
if not self.stream_handle.closed:
self.stream_handle.close()
self.stream_handle = None
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
self.output_path and os.path.isfile(self.output_path):
os.remove(self.output_path)
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
blob = self.blob_manager.get_blob(blob_hash, length)
await self._update_current_blob(blob)
if blob.get_is_verified():
return blob
return await self.blob_downloader.download_blob(blob_hash, length)
# the blob must be downloaded
try:
while not self.current_blob.get_is_verified():
if not self.active_connections: # wait for a new connection
await self.new_peer_event.wait()
continue
blob = await self.wait_for_download_or_new_peer()
if blob:
drain_tasks(self.running_download_requests)
return blob
return blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
def add_fixed_peers(self):
async def _add_fixed_peers():
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
for address, port in addresses
])
async def _accumulate_connections(self, node: 'Node'):
blob_queue = asyncio.Queue(loop=self.loop)
blob_queue.put_nowait(self.sd_hash)
task = asyncio.create_task(self.got_descriptor.wait())
added_peers = asyncio.Event(loop=self.loop)
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
if self.fixed_peers:
def check_added_peers():
if not added_peers.is_set():
self._add_peer_protocols(self.fixed_peers)
log.info("no dht peers for download yet, adding fixed peer")
added_peers.set()
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
def got_descriptor(f):
try:
f.result()
except asyncio.CancelledError:
return
log.info("add head blob hash to peer search")
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
task.add_done_callback(got_descriptor)
try:
async with node.stream_peer_search_junction(blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
if not added_peers.is_set():
added_peers.set()
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
if not self.config.reflector_servers:
return
except asyncio.CancelledError:
pass
finally:
if task and not task.done():
task.cancel()
log.info("cancelled head blob task")
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
add_fixed_peers_timer.cancel()
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()):
self.fixed_peers_delay = 0.0
else:
self.fixed_peers_delay = self.config.fixed_peer_delay
self.loop.create_task(_add_fixed_peers())
async def stop(self):
cancel_task(self.accumulate_connections_task)
self.accumulate_connections_task = None
drain_tasks(self.running_download_requests)
while self.requested_from:
_, peer_task_dict = self.requested_from.popitem()
while peer_task_dict:
peer, task = peer_task_dict.popitem()
try:
cancel_task(task)
except asyncio.CancelledError:
pass
while self.active_connections:
_, client = self.active_connections.popitem()
if client:
await client.close()
log.info("stopped downloader")
async def _download(self):
try:
log.info("download and decrypt stream")
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
log.info(
"downloaded stream %s -> %s", self.sd_hash, self.output_path
)
await self.blob_manager.storage.change_file_status(
self.descriptor.stream_hash, 'finished'
)
except asyncio.CancelledError:
pass
finally:
await self.stop()
def download(self, node: 'Node'):
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
self.download_task = self.loop.create_task(self._download())
def download(self, node: typing.Optional['Node'] = None):
self.node = node
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
self.add_fixed_peers()
+75 -40
View File
@@ -2,13 +2,17 @@ import os
import asyncio
import typing
import logging
from lbrynet.extras.daemon.mime_types import guess_media_type
import binascii
from lbrynet.utils import generate_id
from lbrynet.schema.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
from lbrynet.extras.daemon.storage import StoredStreamClaim
if typing.TYPE_CHECKING:
from lbrynet.extras.daemon.storage import StoredStreamClaim
from lbrynet.schema.claim import Claim
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.node import Node
log = logging.getLogger(__name__)
@@ -18,20 +22,29 @@ class ManagedStream:
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
download_id: typing.Optional[str] = None):
self.loop = loop
self.blob_manager = blob_manager
self.rowid = rowid
self.download_directory = download_directory
self.file_name = file_name
self._file_name = file_name
self.descriptor = descriptor
self.downloader = downloader
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self._store_after_finished: asyncio.Task = None
self.fully_reflected = asyncio.Event(loop=self.loop)
self.tx = None
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
@property
def file_name(self) -> typing.Optional[str]:
return self.downloader.output_file_name if self.downloader else self._file_name
@property
def status(self) -> str:
@@ -83,7 +96,7 @@ class ManagedStream:
@property
def metadata(self) ->typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.claim_dict['stream']['metadata']
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def blobs_completed(self) -> int:
@@ -98,13 +111,23 @@ class ManagedStream:
def sd_hash(self):
return self.descriptor.sd_hash
def as_dict(self) -> typing.Dict:
full_path = os.path.join(self.download_directory, self.file_name)
if not os.path.exists(full_path):
full_path = None
mime_type = guess_media_type(os.path.basename(self.file_name))
@property
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
if self.downloader:
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
def as_dict(self) -> typing.Dict:
full_path = self.full_path if self.output_file_exists else None
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
if self.downloader and self.downloader.written_bytes:
written_bytes = self.downloader.written_bytes
elif full_path:
written_bytes = os.stat(full_path).st_size
@@ -115,6 +138,7 @@ class ManagedStream:
'file_name': self.file_name,
'download_directory': self.download_directory,
'points_paid': 0.0,
'tx': self.tx,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
@@ -128,6 +152,7 @@ class ManagedStream:
'written_bytes': written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
'status': self.status,
'claim_id': self.claim_id,
'txid': self.txid,
@@ -147,50 +172,60 @@ class ManagedStream:
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
)
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
await blob_manager.blob_completed(sd_blob)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
await blob_manager.blob_completed(sd_blob)
for blob in descriptor.blobs[:-1]:
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
status=cls.STATUS_FINISHED)
async def stop_download(self):
def start_download(self, node: typing.Optional['Node']):
self.downloader.download(node)
self.update_status(self.STATUS_RUNNING)
def stop_download(self):
if self.downloader:
await self.downloader.stop()
if not self.finished:
self.update_status(self.STATUS_STOPPED)
self.downloader.stop()
self.downloader = None
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
await self.loop.create_connection(lambda: protocol, host, port)
except ConnectionRefusedError:
return sent
try:
await protocol.send_handshake()
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
try:
sent_sd, needed = await protocol.send_descriptor()
if sent_sd:
sent.append(self.sd_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
for blob_hash in needed:
try:
if not sent_sd and not needed:
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return []
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
for blob_hash in we_have:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
if protocol.transport:
protocol.transport.close()
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
return sent
finally:
if protocol.transport:
protocol.transport.close()
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return sent
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
self.stream_claim_info = StoredStreamClaim(
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
+8 -2
View File
@@ -15,6 +15,7 @@ log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
self.descriptor = descriptor
@@ -26,12 +27,14 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
log.info("Connected to reflector")
log.debug("Connected to reflector")
self.connected.set()
def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None
self.connected.clear()
if self.reflected_blobs:
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
def data_received(self, data):
try:
@@ -45,7 +48,7 @@ class StreamReflectorClient(asyncio.Protocol):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
self.pending_request = self.loop.create_task(self.response_queue.get())
return await self.pending_request
finally:
self.pending_request = None
@@ -80,8 +83,11 @@ class StreamReflectorClient(asyncio.Protocol):
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
needed.append(blob.blob_hash)
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
self.reflected_blobs.append(sd_blob.blob_hash)
else:
log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8])
if needed:
log.info("Reflector needs %i blobs for %s", len(needed), sd_blob.blob_hash[:8])
return sent_sd, needed
async def send_blob(self, blob_hash: str):
+12 -9
View File
@@ -37,7 +37,8 @@ class ReflectorServerProtocol(asyncio.Protocol):
try:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob: %s", err)
log.error("error receiving blob: %s", err)
self.transport.close()
return
try:
request = json.loads(data.decode())
@@ -67,28 +68,30 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_sd_blob": True})
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.writer.close_handle()
self.writer = None
except (asyncio.TimeoutError, asyncio.CancelledError):
self.send_response({"received_sd_blob": True})
except asyncio.TimeoutError:
self.send_response({"received_sd_blob": False})
self.transport.close()
finally:
self.incoming.clear()
self.writer.close_handle()
self.writer = None
self.transport.close()
return
else:
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
if self.writer:
self.writer.close_handle()
self.writer = None
self.send_response({"send_sd_blob": False, 'needed': [
blob.blob_hash for blob in self.descriptor.blobs[:-1]
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
]})
return
return
elif self.descriptor:
if 'blob_hash' not in request:
@@ -105,7 +108,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
try:
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_blob": True})
except (asyncio.TimeoutError, asyncio.CancelledError):
except asyncio.TimeoutError:
self.send_response({"received_blob": False})
self.incoming.clear()
self.writer.close_handle()
+402 -171
View File
@@ -4,21 +4,29 @@ import typing
import binascii
import logging
import random
from decimal import Decimal
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
DownloadDataTimeout, DownloadSDTimeout
from lbrynet.utils import generate_id
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import ClaimDict
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
from lbrynet.schema.claim import Claim
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.extras.daemon.storage import lbc_to_dewies
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.analytics import AnalyticsManager
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
from lbrynet.wallet import LbryWalletManager
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
filter_fields = [
'rowid',
'status',
'file_name',
'sd_hash',
@@ -31,7 +39,9 @@ filter_fields = [
'nout',
'channel_claim_id',
'channel_name',
'full_status'
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream'
]
comparison_operators = {
@@ -45,205 +55,231 @@ comparison_operators = {
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
analytics_manager: typing.Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.wallet = wallet
self.storage = storage
self.node = node
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.analytics_manager = analytics_manager
self.streams: typing.Set[ManagedStream] = set()
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
self.resume_downloading_task: asyncio.Task = None
self.re_reflect_task: asyncio.Task = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.fixed_peers = fixed_peers
self.reflector_servers = reflector_servers
self.running_reflector_uploads: typing.List[asyncio.Task] = []
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, claim_info['value'])
async def start_stream(self, stream: ManagedStream) -> bool:
"""
Resume or rebuild a partial or completed stream
"""
if not stream.running and not stream.output_file_exists:
if stream.downloader:
stream.downloader.stop()
stream.downloader = None
# the directory is gone, can happen when the folder that contains a published file is deleted
# reset the download directory to the default and update the file name
if not os.path.isdir(stream.download_directory):
stream.download_directory = self.config.download_dir
stream.downloader = self.make_downloader(
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
)
if stream.status != ManagedStream.STATUS_FINISHED:
await self.storage.change_file_status(stream.stream_hash, 'running')
stream.update_status('running')
stream.start_download(self.node)
try:
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
self.config.download_timeout)
except asyncio.TimeoutError:
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
return False
file_name = os.path.basename(stream.downloader.output_path)
output_dir = os.path.dirname(stream.downloader.output_path)
await self.storage.change_file_download_dir_and_file_name(
stream.stream_hash, output_dir, file_name
)
stream._file_name = file_name
stream.download_directory = output_dir
self.wait_for_stream_finished(stream)
return True
return True
async def stop_stream(self, stream: ManagedStream):
stream.stop_download()
if not stream.finished and stream.output_file_exists:
try:
os.remove(stream.full_path)
except OSError as err:
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
str(err))
if stream.running:
stream.update_status(ManagedStream.STATUS_STOPPED)
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
return StreamDownloader(
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
)
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = []
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
sd_blob = self.blob_manager.get_blob(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash)
descriptor = await StreamDescriptor.recover(
self.blob_manager.blob_dir, sd_blob, stream_hash, stream_name, suggested_file_name, key, blobs
)
if not descriptor:
return
to_restore.append((descriptor, sd_blob))
await asyncio.gather(*[
recover_stream(
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
) for file_info in file_infos
])
if to_restore:
await self.storage.recover_streams(to_restore, self.config.download_dir)
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
claim: typing.Optional['StoredStreamClaim']):
sd_blob = self.blob_manager.get_blob(sd_hash)
if not sd_blob.get_is_verified():
return
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
except InvalidStreamDescriptorError as err:
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
return
if status == ManagedStream.STATUS_RUNNING:
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
else:
downloader = None
stream = ManagedStream(
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
)
self.streams.add(stream)
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
async def load_streams_from_database(self):
infos = await self.storage.get_all_lbry_files()
for file_info in infos:
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
if sd_blob.get_is_verified():
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
downloader = StreamDownloader(
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
)
stream = ManagedStream(
self.loop, self.blob_manager, descriptor,
binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(),
downloader, file_info['status'], file_info['claim']
)
self.streams.add(stream)
to_recover = []
for file_info in await self.storage.get_all_lbry_files():
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_recover.append(file_info)
if to_recover:
log.info("Attempting to recover %i streams", len(to_recover))
await self.recover_streams(to_recover)
to_start = []
for file_info in await self.storage.get_all_lbry_files():
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
to_start.append(file_info)
log.info("Initializing %i files", len(to_start))
await asyncio.gather(*[
self.add_stream(
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
file_info['claim']
) for file_info in to_start
])
log.info("Started stream manager with %i files", len(self.streams))
async def resume(self):
if not self.node:
log.warning("no DHT node given, cannot resume downloads")
return
await self.node.joined.wait()
resumed = 0
for stream in self.streams:
if stream.status == ManagedStream.STATUS_RUNNING:
resumed += 1
stream.downloader.download(self.node)
self.wait_for_stream_finished(stream)
if resumed:
log.info("resuming %i downloads", resumed)
if self.node:
await self.node.joined.wait()
else:
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
t = [
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
]
if t:
log.info("resuming %i downloads", len(t))
async def reflect_streams(self):
streams = list(self.streams)
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= 10:
await asyncio.gather(*batch)
while True:
if self.config.reflect_streams and self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
batch = []
if batch:
await asyncio.gather(*batch)
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch)
batch = []
if batch:
await asyncio.gather(*batch)
await asyncio.sleep(300, loop=self.loop)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
async def stop(self):
def stop(self):
if self.resume_downloading_task and not self.resume_downloading_task.done():
self.resume_downloading_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
stream = self.streams.pop()
await stream.stop_download()
stream.stop_download()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads:
self.running_reflector_uploads.pop().cancel()
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
if self.reflector_servers:
host, port = random.choice(self.reflector_servers)
self.loop.create_task(stream.upload_to_reflector(host, port))
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers:
host, port = random.choice(self.config.reflector_servers)
task = self.loop.create_task(stream.upload_to_reflector(host, port))
self.running_reflector_uploads.append(task)
task.add_done_callback(
lambda _: None
if task not in self.running_reflector_uploads else self.running_reflector_uploads.remove(task)
)
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
await stream.stop_download()
self.streams.remove(stream)
await self.stop_stream(stream)
if stream in self.streams:
self.streams.remove(stream)
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor)
if delete_file and stream.output_file_exists:
os.remove(stream.full_path)
blob_hashes = [stream.sd_hash]
for blob_info in stream.descriptor.blobs[:-1]:
blob_hashes.append(blob_info.blob_hash)
for blob_hash in blob_hashes:
blob = self.blob_manager.get_blob(blob_hash)
if blob.get_is_verified():
await blob.delete()
if delete_file:
path = os.path.join(stream.download_directory, stream.file_name)
if os.path.isfile(path):
os.remove(path)
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
try:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
except asyncio.CancelledError:
pass
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
sd_blob_timeout: typing.Optional[float] = 60
) -> typing.Optional[ManagedStream]:
claim = ClaimDict.load_dict(claim_info['value'])
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
try:
downloader.download(node)
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
except (asyncio.TimeoutError, asyncio.CancelledError):
log.info("stream timeout")
await downloader.stop()
log.info("stopped stream")
return
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
await self.blob_manager.storage.save_downloaded_file(
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
data_rate
)
await self.blob_manager.storage.save_content_claim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
)
stored_claim = StoredStreamClaim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
claim_info.get('channel_name')
)
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
stored_claim)
self.streams.add(stream)
try:
await stream.downloader.wrote_bytes_event.wait()
self.wait_for_stream_finished(stream)
return stream
except asyncio.CancelledError:
await downloader.stop()
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None,
sd_blob_timeout: typing.Optional[float] = 60,
fee_amount: typing.Optional[float] = 0.0,
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
claim = ClaimDict.load_dict(claim_info['value'])
if fee_address and fee_amount:
if fee_amount > await self.wallet.default_account.get_balance():
raise Exception("not enough funds")
sd_hash = claim.source_hash.decode()
if sd_hash in self.starting_streams:
return await self.starting_streams[sd_hash]
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
if already_started:
return already_started[0]
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
stream_task = self.loop.create_task(
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
)
try:
await asyncio.wait_for(stream_task, sd_blob_timeout)
stream = await stream_task
self.starting_streams[sd_hash].set_result(stream)
if fee_address and fee_amount:
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
return stream
except (asyncio.TimeoutError, asyncio.CancelledError):
return
finally:
if sd_hash in self.starting_streams:
del self.starting_streams[sd_hash]
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
if streams:
return streams[0]
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
comparison: typing.Optional[str] = None,
@@ -260,6 +296,8 @@ class StreamManager:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in comparison_operators:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by.keys():
if search not in filter_fields:
raise ValueError(f"'{search}' is not a valid search operation")
@@ -268,8 +306,6 @@ class StreamManager:
streams = []
for stream in self.streams:
for search, val in search_by.items():
if search == 'full_status':
continue
if comparison_operators[comparison](getattr(stream, search), val):
streams.append(stream)
break
@@ -280,3 +316,198 @@ class StreamManager:
if reverse:
streams.reverse()
return streams
def wait_for_stream_finished(self, stream: ManagedStream):
async def _wait_for_stream_finished():
if stream.downloader and stream.running:
await stream.downloader.stream_finished_event.wait()
stream.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
stream.download_id, stream.claim_name, stream.sd_hash
))
task = self.loop.create_task(_wait_for_stream_finished())
self.update_stream_finished_futs.append(task)
task.add_done_callback(
lambda _: None if task not in self.update_stream_finished_futs else
self.update_stream_finished_futs.remove(task)
)
async def _store_stream(self, downloader: StreamDownloader) -> int:
file_name = os.path.basename(downloader.output_path)
download_directory = os.path.dirname(downloader.output_path)
if not await self.storage.stream_exists(downloader.sd_hash):
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.storage.file_exists(downloader.sd_hash):
return await self.storage.save_downloaded_file(
downloader.descriptor.stream_hash, file_name, download_directory,
0.0
)
else:
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim) -> typing.Tuple[
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
await self.start_stream(existing[0])
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
f"download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
await self.start_stream(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
return None, existing_for_claim_id[0]
return None, None
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
download_id: str, outpoint: str, claim: Claim, resolved: typing.Dict,
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
downloader.download(self.node)
await downloader.got_descriptor.wait()
got_descriptor_time.set_result(self.loop.time() - start_time)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, outpoint
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
stream.set_claim(resolved, claim)
await stream.downloader.wrote_bytes_event.wait()
self.streams.add(stream)
return stream
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['hex']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
file_name)
), timeout
)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
claim.stream.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
time_to_first_bytes, None if not error else error.__class__.__name__
)
)
if error:
raise error
return stream
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> ManagedStream:
timeout = timeout or self.config.download_timeout
if uri in self.starting_streams:
return await self.starting_streams[uri]
fut = asyncio.Future(loop=self.loop)
self.starting_streams[uri] = fut
try:
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
fut.set_result(stream)
except Exception as err:
fut.set_exception(err)
try:
return await fut
finally:
del self.starting_streams[uri]
+57 -10
View File
@@ -7,9 +7,14 @@ import string
import json
import typing
import asyncio
import ssl
import logging
import ipaddress
import pkg_resources
from lbrynet.schema.claim import ClaimDict
import contextlib
import certifi
import aiohttp
from lbrynet.schema.claim import Claim
from lbrynet.cryptoutils import get_lbry_hash_obj
@@ -68,13 +73,12 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5):
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
server = socket.gethostbyname(server)
conn = socket.create_connection((server, port), timeout)
conn.close()
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
@@ -83,19 +87,23 @@ def check_connection(server="lbry.io", port=80, timeout=5):
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout)
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except Exception as ex:
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
except Exception as ex:
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
server, port)
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
@@ -107,8 +115,8 @@ def short_hash(hash_str):
def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, ClaimDict):
return stream_info.source_hash
if isinstance(stream_info, Claim):
return stream_info.stream.sd_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\
@@ -136,3 +144,42 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
try:
if ipaddress.ip_address(url):
return url
except ValueError:
pass
loop = asyncio.get_running_loop()
return (await loop.getaddrinfo(
url, port,
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
))[0][4][0]
def get_ssl_context() -> ssl.SSLContext:
return ssl.create_default_context(
purpose=ssl.Purpose.CLIENT_AUTH, capath=certifi.where()
)
@contextlib.asynccontextmanager
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
async with aiohttp.ClientSession() as session:
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
yield response
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
try:
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
return
+11
View File
@@ -0,0 +1,11 @@
__node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.0/lbrycrd-linux.zip'
)
__spvserver__ = 'lbrynet.wallet.server.coin.LBCRegTest'
from lbrynet.wallet.manager import LbryWalletManager
from lbrynet.wallet.network import Network
from lbrynet.wallet.ledger import MainNetLedger, RegTestLedger, TestNetLedger
@@ -1,38 +1,51 @@
import json
import logging
import binascii
from hashlib import sha256
from string import hexdigits
from lbrynet.schema.validator import validate_claim_id
from torba.client.baseaccount import BaseAccount
from torba.client.basetransaction import TXORef
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.signer import SECP256k1, get_signer
log = logging.getLogger(__name__)
def generate_certificate():
secp256k1_private_key = get_signer(SECP256k1).generate().private_key.to_pem()
return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key
def validate_claim_id(claim_id):
if not len(claim_id) == 40:
raise Exception("Incorrect claimid length: %i" % len(claim_id))
if isinstance(claim_id, bytes):
claim_id = claim_id.decode('utf-8')
if set(claim_id).difference(hexdigits):
raise Exception("Claim id is not hex encoded")
class Account(BaseAccount):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.certificates = {}
self.channel_keys = {}
def add_certificate_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
self.certificates[ref.id] = private_key
@property
def hash(self) -> bytes:
h = sha256(json.dumps(self.to_dict(False)).encode())
for cert in sorted(self.channel_keys.keys()):
h.update(cert.encode())
return h.digest()
def get_certificate_private_key(self, ref: TXORef):
return self.certificates.get(ref.id)
def apply(self, d: dict):
super().apply(d)
self.channel_keys.update(d.get('certificates', {}))
def add_channel_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.channel_keys, 'Trying to add a duplicate channel private key.'
self.channel_keys[ref.id] = private_key
def get_channel_private_key(self, ref: TXORef):
return self.channel_keys.get(ref.id)
async def maybe_migrate_certificates(self):
if not self.certificates:
if not self.channel_keys:
return
addresses = {}
@@ -46,7 +59,7 @@ class Account(BaseAccount):
}
double_hex_encoded_to_pop = []
for maybe_claim_id in list(self.certificates):
for maybe_claim_id in list(self.channel_keys):
if ':' not in maybe_claim_id:
try:
validate_claim_id(maybe_claim_id)
@@ -58,7 +71,7 @@ class Account(BaseAccount):
maybe_claim_id_bytes = maybe_claim_id_bytes.encode()
decoded_double_hex = binascii.unhexlify(maybe_claim_id_bytes).decode()
validate_claim_id(decoded_double_hex)
if decoded_double_hex in self.certificates:
if decoded_double_hex in self.channel_keys:
log.warning("don't know how to migrate certificate %s", decoded_double_hex)
else:
log.info("claim id was double hex encoded, fixing it")
@@ -67,9 +80,9 @@ class Account(BaseAccount):
continue
for double_encoded_claim_id, correct_claim_id in double_hex_encoded_to_pop:
self.certificates[correct_claim_id] = self.certificates.pop(double_encoded_claim_id)
self.channel_keys[correct_claim_id] = self.channel_keys.pop(double_encoded_claim_id)
for maybe_claim_id in list(self.certificates):
for maybe_claim_id in list(self.channel_keys):
results['total'] += 1
if ':' not in maybe_claim_id:
try:
@@ -104,8 +117,8 @@ class Account(BaseAccount):
.format(maybe_claim_id)
)
tx_nout = '{txid}:{nout}'.format(**claim)
self.certificates[tx_nout] = self.certificates[maybe_claim_id]
del self.certificates[maybe_claim_id]
self.channel_keys[tx_nout] = self.channel_keys[maybe_claim_id]
del self.channel_keys[maybe_claim_id]
log.info(
"Migrated certificate with claim_id '%s' ('%s') to a new look up key %s.",
maybe_claim_id, txo.script.values['claim_name'], tx_nout
@@ -165,7 +178,7 @@ class Account(BaseAccount):
return super().get_balance(confirmations, **constraints)
@classmethod
def get_private_key_from_seed(cls, ledger: 'baseledger.BaseLedger', seed: str, password: str):
def get_private_key_from_seed(cls, ledger, seed: str, password: str):
return super().get_private_key_from_seed(
ledger, seed, password or 'lbryum'
)
@@ -173,35 +186,30 @@ class Account(BaseAccount):
@classmethod
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
account = super().from_dict(ledger, wallet, d)
account.certificates = d.get('certificates', {})
account.channel_keys = d.get('certificates', {})
return account
def to_dict(self):
def to_dict(self, include_channel_keys=True):
d = super().to_dict()
d['certificates'] = self.certificates
if include_channel_keys:
d['certificates'] = self.channel_keys
return d
async def get_details(self, **kwargs):
details = await super().get_details(**kwargs)
details['certificates'] = len(self.certificates)
details['certificates'] = len(self.channel_keys)
return details
def get_claim(self, claim_id=None, txid=None, nout=None):
if claim_id is not None:
return self.ledger.db.get_claims(account=self, claim_id=claim_id)
elif txid is not None and nout is not None:
return self.ledger.db.get_claims(**{'account': self, 'txo.txid': txid, 'txo.position': nout})
@staticmethod
def constraint_utxos_sans_claims(constraints):
def constraint_spending_utxos(constraints):
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
def get_utxos(self, **constraints):
self.constraint_utxos_sans_claims(constraints)
self.constraint_spending_utxos(constraints)
return super().get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_utxos_sans_claims(constraints)
self.constraint_spending_utxos(constraints)
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
@@ -210,32 +218,23 @@ class Account(BaseAccount):
def get_claim_count(self, **constraints):
return self.ledger.db.get_claim_count(account=self, **constraints)
def get_streams(self, **constraints):
return self.ledger.db.get_streams(account=self, **constraints)
def get_stream_count(self, **constraints):
return self.ledger.db.get_stream_count(account=self, **constraints)
def get_channels(self, **constraints):
return self.ledger.db.get_channels(account=self, **constraints)
def get_channel_count(self, **constraints):
return self.ledger.db.get_channel_count(account=self, **constraints)
async def send_to_addresses(self, amount, addresses, broadcast=False):
tx_class = self.ledger.transaction_class
tx = await tx_class.create(
inputs=[],
outputs=[
tx_class.output_class.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))
for address in addresses
],
funding_accounts=[self],
change_account=self
)
def get_supports(self, **constraints):
return self.ledger.db.get_supports(account=self, **constraints)
if broadcast:
await self.ledger.broadcast(tx)
else:
await self.ledger.release_outputs(
[txi.txo_ref.txo for txi in tx.inputs]
)
return tx
def get_support_count(self, **constraints):
return self.ledger.db.get_support_count(account=self, **constraints)
async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self)
@@ -74,9 +74,10 @@ def verify_proof(proof, root_hash, name):
if 'txhash' in proof and 'nOut' in proof:
if not verified_value:
raise InvalidProofError("mismatch between proof claim and outcome")
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
if 'txhash' in proof and 'nOut' in proof:
if name != reverse_computed_name[::-1]:
if name != target:
raise InvalidProofError("name did not match proof")
if not name.startswith(reverse_computed_name[::-1]):
if not name.startswith(target):
raise InvalidProofError("name fragment does not match proof")
return True
Loaded 100 of 173 files, more files were not shown because too many files have changed in this diff. Show more