Compare commits

...
Author SHA1 Message Date
Lex Berezhny 5671224fe0 insert_transactions and other client speed ups
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (postgres, blockchain) (push) Waiting to run
ci / tests / integration (sqlite, blockchain) (push) Waiting to run
2021-01-19 12:32:05 -05:00
Lex Berezhny f3c1dcef81 fix unit test 2021-01-18 10:35:57 -05:00
Lex Berezhny a2c4608bc8 match many addresses at once and use 100k filters 2021-01-18 09:47:25 -05:00
Lex Berezhny 5562e84722 100k filters 2021-01-13 00:44:23 -05:00
Lex Berezhny 970d0ba6e1 debugging during client sync 2021-01-11 14:04:45 -05:00
Lex Berezhny e4f37a7891 save async 2021-01-11 10:59:56 -05:00
Lex Berezhny 5e3b8cb7ea save syncrono 2021-01-11 10:59:00 -05:00
Lex Berezhny 369dcaafc3 fix unit test 2021-01-11 10:46:19 -05:00
Lex Berezhny 17577d8e85 better output and bulk insert 2021-01-11 10:36:46 -05:00
Lex Berezhny f89c7540b7 bulk insert 2021-01-11 10:36:21 -05:00
Lex Berezhny 4b45fbc3e1 bulk insert 2021-01-11 10:35:32 -05:00
Lex Berezhny fa75895131 lint 2021-01-11 09:37:01 -05:00
Lex Berezhny 3f3e370d36 regenerate metadata.py 2021-01-11 01:18:22 -05:00
Lex Berezhny 0620cf78b2 client sync fixes 2021-01-11 00:56:15 -05:00
Lex Berezhny d274145328 debugging prints 2021-01-10 13:54:53 -05:00
Lex Berezhny f532a1fbcd fix blockchain sync query performance issue 2021-01-10 12:41:21 -05:00
Victor Shyba 953b8dab76 check if constraint is NOT there before creating 2021-01-10 10:05:49 -05:00
Victor Shyba a411bda620 release time is 64 bits on protobuf, update postgres schema 2021-01-10 10:05:49 -05:00
Lex Berezhny c4e4e81ac3 memory address -> bytes 2021-01-10 10:00:57 -05:00
Lex Berezhny 665921b3b1 being able to start light client from clil 2021-01-08 01:18:57 -05:00
Lex Berezhny a26535252d fix pyinstaller install 2021-01-08 00:53:31 -05:00
Lex Berezhny bdb82a3661 fix which tests run 2021-01-08 00:53:31 -05:00
Lex Berezhny 1e7100e6b4 fix test in postgres 2021-01-08 00:53:31 -05:00
Lex Berezhny 51467c546b lint 2021-01-08 00:53:31 -05:00
Lex Berezhny c42b08b090 signature validation with grace period 2021-01-08 00:53:31 -05:00
Lex Berezhny 5eed7d87d3 working client tx sync 2021-01-08 00:53:31 -05:00
Lex Berezhny 522dc72dc1 client sync algorithms 2021-01-08 00:53:31 -05:00
Lex Berezhny d74fa05a8b client wallet sync 2021-01-08 00:53:31 -05:00
Lex Berezhny becef285f6 sync_apply progress 2021-01-08 00:53:31 -05:00
Lex Berezhny 5d738534cb multi daemon testing 2021-01-08 00:53:31 -05:00
Lex Berezhny f69426bbad minor refactor and fix for a test 2020-12-21 13:04:16 -05:00
Victor Shyba 546205ba69 check for information_schema.table_constraint instead of filters 2020-12-21 13:04:16 -05:00
Victor Shyba e65e49768a fix windows complaints on permissions for wallet file test 2020-12-21 13:04:16 -05:00
Victor Shyba 83a1a4ea3e account for the number of variables on each iteration 2020-12-21 13:04:16 -05:00
Victor Shyba 96b506f9fa fixes for old keys 2020-12-21 13:04:16 -05:00
Victor Shyba b31368a3a7 ClaimMeta attr typo 2020-12-21 13:04:16 -05:00
Victor Shyba 426a0449fd fix initial sync for sync without claims 2020-12-21 13:04:16 -05:00
Victor Shyba 2a911e0135 fix bug on index application 2020-12-08 18:03:49 -05:00
Lex Berezhny 4592e21424 minor refactoring 2020-12-07 07:40:16 -05:00
Victor Shyba 6df67f225f decrease usage of expensive call on tests 2020-12-07 07:40:16 -05:00
Victor Shyba c484a8abf5 fixes from review 2020-12-07 07:40:16 -05:00
Victor Shyba f4a8be6c19 use coalesce to high value instead of nulls last to be ANSI compatible 2020-12-07 07:40:16 -05:00
Victor Shyba 8968893216 reduce diff 2020-12-07 07:40:16 -05:00
Victor Shyba 09aa3039e9 fix compat issues between postgres/sqlite 2020-12-07 07:40:16 -05:00
Victor Shyba c1c9c2883a fix some of the tests 2020-12-07 07:40:16 -05:00
Victor Shyba 9c269c90bb accept FILTERING_CHANNEL_IDS and BLOCKING_CHANNEL_IDS env vars as the old SDK 2020-12-07 07:40:16 -05:00
Victor Shyba e4449d2ec7 refactor level to get_xyz_... 2020-12-07 07:40:16 -05:00
Victor Shyba 4e0728572d pylint 2020-12-07 07:40:16 -05:00
Victor Shyba d3ec200e46 support blocking content 2020-12-07 07:40:16 -05:00
Victor Shyba 90e7821283 filter whole channels 2020-12-07 07:40:16 -05:00
Victor Shyba a5eccb9a92 filter reposts 2020-12-07 07:40:16 -05:00
Victor Shyba f83289f876 add owner column, apply censor 2020-12-07 07:40:16 -05:00
Victor Shyba 6bb8a69efc load hashes into claim filter table 2020-12-07 07:40:16 -05:00
Victor Shyba 76b3bfe975 add simple trending 2020-12-07 07:40:16 -05:00
Victor Shyba f3710fa153 check all kwargs are consumed on claim search 2020-12-07 07:40:16 -05:00
Victor Shyba 2b62330980 fix channel argument 2020-12-07 07:40:16 -05:00
Victor Shyba 036e16d1fc decrease default to 999 until it updates 2020-12-01 16:20:05 -05:00
Victor Shyba ea691e78b8 batch under sqlite variable limit 2020-12-01 16:20:05 -05:00
Jack Robison 435a80d4c3 Update README.md 2020-12-01 12:34:04 -05:00
Lex Berezhny ec89baa831 lint fixes and revert sum_support overwrite during merging 2020-11-17 15:20:25 -05:00
Lex Berezhny 85cf19bb2d test_order_by fixed 2020-11-17 14:53:48 -05:00
Lex Berezhny 1971539369 test fixes 2020-11-17 14:53:48 -05:00
Lex Berezhny 5d4e61d089 reposts 2020-11-17 14:53:48 -05:00
Lex Berezhny 2ee65752b2 lint 2020-11-17 14:53:48 -05:00
Lex Berezhny 1d31a96c9b test_claim_commands.py 2020-11-17 14:53:48 -05:00
Lex Berezhny e6a9417988 transaction list 2020-11-17 14:53:48 -05:00
Lex Berezhny 1e4613fd8a more commands 2020-11-17 14:53:48 -05:00
Lex Berezhny d459d6a26b wallet commands progress 2020-11-17 14:53:48 -05:00
Lex Berezhny d8f5b11158 coalesce null balance 2020-11-17 14:53:48 -05:00
Lex Berezhny 6764d52634 postgres optimized balance query 2020-11-17 14:53:48 -05:00
Lex Berezhny 4d8cc494e1 add txo_unspent_by_address index 2020-11-17 14:53:48 -05:00
Lex Berezhny 22a04fab24 balance 2020-11-17 14:53:48 -05:00
Lex Berezhny 9f9fdd2d1a + include_total 2020-11-17 14:53:48 -05:00
Lex Berezhny 2ec91dbca7 send dict instead of Account and PubKey objects when adding keys 2020-11-17 14:53:48 -05:00
Lex Berezhny 1a4a2db1b1 command fixes 2020-11-17 14:53:48 -05:00
Lex Berezhny cc2837c021 account_add fix 2020-11-17 14:53:48 -05:00
Lex Berezhny 5cdfbcc88e full node wallet support 2020-11-17 14:53:48 -05:00
Lex Berezhny 6ed2fa20ec working db based wallet and wallet sync progress 2020-11-17 14:53:48 -05:00
Lex Berezhny 4356d23cc1 minor test fix 2020-11-17 14:53:48 -05:00
Lex Berezhny 2f4b5d2ffe lint 2020-11-17 14:53:48 -05:00
Lex Berezhny 0a06810f36 working mempool 2020-11-17 14:53:48 -05:00
Lex Berezhny 9ce8910b42 updated unit test imports 2020-11-17 14:53:48 -05:00
Lex Berezhny fa85558d71 wip 2020-11-17 14:53:48 -05:00
Alex Grintsvayg 999e4209fa add missing flag in usage 2020-11-11 16:59:53 -05:00
Alex Grintsvayg 7d82a4573f add indexes to optimize sum_supports query 2020-11-11 16:56:32 -05:00
Victor Shyba c1aa9b8150 ignore blocks marked as BLOCK_FAILED_VALID and BLOCK_FAILED_CHILD 2020-10-26 16:33:26 -04:00
Alex Grintsvayg daed032bb7 return total instead of percent so frontend can display percentage precisely 2020-10-21 10:34:18 -04:00
Alex Grintsvayg abebc0d878 add more tests, flag to exclude own supports 2020-10-21 10:34:18 -04:00
Alex Grintsvayg 4bb0344e05 add sum_supports api 2020-10-16 10:44:47 -04:00
Lex Berezhny 0215ce6b79 renamed expander functions to have different name from actual arguments by prepending extract_ 2020-10-05 17:24:02 -04:00
Lex Berezhny b9cf130344 fix for takeover height calculation 2020-09-25 15:02:31 -04:00
Lex Berezhny 85b8877933 Merge pull request #3039 from lbryio/fwss_fix_resolve
Include enough information in resolve protobuf so its possible to verify signatures
2020-09-24 20:57:56 -04:00
Lex Berezhny bba9aec4f2 added respoted_count calculation 2020-09-24 15:19:30 -04:00
Lex Berezhny 2853bb437d make TXs pickleable 2020-09-24 12:27:06 -04:00
Victor Shyba 2d7929f113 fixme->TODO 2020-09-23 14:39:24 -03:00
Victor Shyba 24386c74f4 fix test_nulls 2020-09-23 08:38:33 -04:00
Victor Shyba 0a2c161ace make short_url and canonical_url match old SDK 2020-09-23 08:38:33 -04:00
Victor Shyba 18b25f5146 add and test takeover info on resolve 2020-09-23 08:38:33 -04:00
Victor Shyba 1a5292aaf4 add expiration_height 2020-09-23 08:38:33 -04:00
Victor Shyba 39ba2a8b7f log errors for on_rpc 2020-09-23 08:38:33 -04:00
Victor Shyba 25b63c988f further fixes to effective amount 2020-09-23 08:38:33 -04:00
Victor Shyba fdac6416a1 tests: find_claim_txo for reliability in some places 2020-09-23 08:38:33 -04:00
Victor Shyba 4783e6da1f fix effective_amount 2020-09-23 08:38:33 -04:00
Victor Shyba 9c1d593e54 fix set_reference 2020-09-23 08:38:33 -04:00
Victor Shyba fa34ff88bc refactor db, make resolve censor right 2020-09-23 08:38:33 -04:00
Victor Shyba 6690e63ea4 remove commented code and unused import 2020-09-23 08:38:33 -04:00
Victor Shyba c7eb60619f separate test for not found message 2020-09-23 08:38:33 -04:00
Victor Shyba cb98fbc619 fix not found raising, add a test 2020-09-23 08:38:33 -04:00
Victor Shyba 4fcfa0b193 add channel information on resolve protobuf and test for its presence 2020-09-23 08:38:33 -04:00
Lex Berezhny 8cde120928 start --full-node -> start node 2020-09-23 08:37:45 -04:00
Lex Berezhny 052ef73226 pylint 2020-09-21 22:17:45 -04:00
Lex Berezhny 082a91dc15 use LEFT JOIN instead of IN for faster get_block_range_without_filters() 2020-09-21 22:04:50 -04:00
Lex Berezhny d10a88c79b script hash addresses 2020-09-21 21:46:41 -04:00
Lex Berezhny 62a3022a3d support for old public key addresses 2020-09-21 15:27:47 -04:00
Lex Berezhny 4e6bdb64a8 Merge pull request #3043 from lbryio/bip-158
Sync block filters
2020-09-21 12:02:50 -04:00
Lex Berezhny 0b34c4ea86 got tests to pass on sqlite and postgres and added filter groups at various block height scales 2020-09-21 10:26:19 -04:00
Jack Robison 5b9e40c0c0 sync block filters 2020-09-17 11:25:50 -04:00
Jack Robison 30dff02674 logging formatter 2020-09-17 11:25:28 -04:00
Lex Berezhny 20c7fe1e82 Merge pull request #3048 from lbryio/asyncio_loop_fix
asyncio loop fix
2020-09-16 20:16:13 -04:00
Lex Berezhny 23f186e6a1 pylint and other fixes 2020-09-16 20:02:34 -04:00
Lex Berezhny 14436b3955 instantiate objects inside async loop 2020-09-16 19:50:51 -04:00
Lex Berezhny a96ceba6f5 Daemon.from_config 2020-09-16 16:40:05 -04:00
Lex Berezhny f26aadbd44 move daemon construction from cli into daemon function 2020-09-16 16:34:55 -04:00
Lex Berezhny 144eb248e3 Merge pull request #3040 from lbryio/fwss_wait_lbrycrd
Synchronizer: wait for lbrycrd to start if its not ready, instead of raising
2020-09-14 13:21:10 -04:00
Victor Shyba a554c8838c use previous exception insted of declaring a new one 2020-09-14 14:08:59 -03:00
Victor Shyba 2a0089a4dd test starting and waiting normally as before too 2020-09-14 14:08:59 -03:00
Victor Shyba d6bcbd631f improve session close error message 2020-09-14 14:08:59 -03:00
Victor Shyba 39a4c4e590 exit if zmq misconfigured. tell whats wrong 2020-09-14 14:08:59 -03:00
Victor Shyba cb60cd99f4 use create_task instead 2020-09-14 14:08:59 -03:00
Victor Shyba a5c117b542 wait for lbrycrd on start 2020-09-14 14:08:59 -03:00
Victor Shyba 47a8c005d9 change zmq port for tests so it doesnt conflict with running lbrycrd 2020-09-14 14:08:59 -03:00
Lex Berezhny 6e50e83e28 Merge pull request #3044 from lbryio/fwss_pin_tqdm
pin tqdm and fix ovrride
2020-09-14 12:59:25 -04:00
Victor Shyba 50c8fc269b fix tqdm override and pin version 2020-09-14 13:46:31 -03:00
Alex Grintsvayg 336a0f6ae1 minor 2020-09-11 14:08:06 -04:00
Lex Berezhny e1c33dccab Merge pull request #3036 from lbryio/fwss_fix_parser
fix doc generator for docopt edge cases
2020-09-04 15:48:57 -04:00
Lex Berezhny 2f56f7f0e0 fix test 2020-09-04 15:40:59 -04:00
Lex Berezhny 28413742cc reduce diff 2 2020-09-04 15:38:31 -04:00
Lex Berezhny 6484894b36 reduce diff 2020-09-04 15:37:45 -04:00
Lex Berezhny c5fd9643f1 tx_kwargs expander should have wallet_id 2020-09-04 15:36:18 -04:00
Victor Shyba aa6d78b515 fix definitions, bump metadata 2020-09-04 15:09:28 -03:00
Lex Berezhny 33e266a0f4 unit test to make sure all generated help can be parsed by docopt 2020-09-04 10:57:11 -04:00
Lex Berezhny b4ee07162d made doc parsing error more helpful 2020-09-04 10:17:45 -04:00
Victor Shyba 0f63103db5 fix wallet_id parameter 2020-09-04 01:36:49 -03:00
Victor Shyba c3884352db improve error messages 2020-09-04 01:29:25 -03:00
Victor Shyba 8c525b6dfc fix tests 2020-09-04 01:25:56 -03:00
Victor Shyba d89b074615 bump metadata.py 2020-09-04 01:20:39 -03:00
Victor Shyba f775b0ed55 fix all comments with line breaks starting with -- 2020-09-04 01:20:27 -03:00
Victor Shyba 60c333c6e2 stream filter already have signed filter 2020-09-04 00:58:08 -03:00
Victor Shyba 42b7f8ff71 raise when -- ends up in a continuation line instead of auto fixing 2020-09-04 00:45:02 -03:00
Victor Shyba c981c767b9 raise instead of implicit deduplication of arguments 2020-09-04 00:44:55 -03:00
Victor Shyba 10ad4ed8d1 update metadata.py 2020-09-04 00:22:14 -03:00
Victor Shyba e436ae7edd test for wrong linebreaks and improve fix 2020-09-04 00:22:14 -03:00
Victor Shyba c498619cca test using a new expander explicitly 2020-09-04 00:22:14 -03:00
Victor Shyba 382facf264 test for duplicated kwargs 2020-09-04 00:22:14 -03:00
Victor Shyba 7304d24dfd dont break lines on -- to avoid docopt parsing issues 2020-09-04 00:22:14 -03:00
Victor Shyba b330c35282 remove duplicated kwargs from parser.py 2020-09-04 00:22:14 -03:00
Lex Berezhny 6dfa78afa3 Merge pull request #3034 from lbryio/wip_fwss
misc fixes
2020-09-03 23:21:15 -04:00
Victor Shyba 1c79daaafc linter: ignore raise-missing-from 2020-09-04 00:15:54 -03:00
Victor Shyba e1b55f017b remove traceback usage, add test 2020-09-03 15:21:53 -03:00
Victor Shyba 6bbfb45de7 return errors when websocket req fails 2020-09-02 20:38:30 -03:00
Victor Shyba 9b15799c72 makefile default to match the number of CPUs and not arbitrary 28 2020-09-02 18:50:22 -03:00
Victor Shyba bc678f2146 session close is async, fix annoying error message 2020-09-02 18:49:41 -03:00
Victor Shyba 7c5211d420 fix lbrycrdd url 2020-09-02 14:34:00 -03:00
Victor Shyba 70e50780c3 dropdb ignores already dropped db 2020-09-02 14:33:36 -03:00
Lex Berezhny 7ebb9d06df discover and run all unittests 2020-08-20 14:27:28 -04:00
Lex Berezhny b99de6b872 skip tests needing to be updated to use new APIs 2020-08-20 14:24:08 -04:00
Lex Berezhny 889464e51d drop components 2020-08-20 14:22:46 -04:00
Lex Berezhny 35c3ff1e30 test cleaup and fixing 2020-08-20 13:31:58 -04:00
Lex Berezhny aa75b9bb25 moved wallet specific unit tests into wallet dir 2020-08-20 13:30:33 -04:00
Lex Berezhny f995ceae8b pylint 2020-08-20 10:46:28 -04:00
Lex Berezhny c1803434aa support clean exit 2020-08-20 10:44:29 -04:00
Lex Berezhny 4dfbdcc2d7 Update docker-compose.yml 2020-08-14 00:29:51 -04:00
Lex Berezhny 272940b6d6 removed docker/hooks 2020-08-14 03:47:53 +00:00
Lex Berezhny baf384d6e0 cleanup 2020-08-14 03:42:04 +00:00
Lex Berezhny a012c04974 docker-compose.yml 2020-08-14 03:38:33 +00:00
Lex Berezhny 1fa117a104 upgrade distro 2020-08-14 00:33:44 +00:00
Lex Berezhny 33a157959d index added 2020-08-13 12:18:39 -04:00
Lex Berezhny 747eace4ab lint 2020-08-13 12:08:35 -04:00
Lex Berezhny 8fa2d746e7 re-enable all github workflows 2020-08-13 11:57:10 -04:00
Lex Berezhny a77b1f9997 move to docker dir 2020-08-13 11:50:20 -04:00
Lex Berezhny c914f24089 chmod lbrynet 2020-08-13 11:35:24 -04:00
Lex Berezhny e41a71a64e no dockerignore 2020-08-13 11:25:21 -04:00
Lex Berezhny e59e238fd5 dockerfile bin error 2020-08-13 11:17:45 -04:00
Lex Berezhny 4c331d00e7 moved Dockerfile 2020-08-13 00:40:12 -04:00
Lex Berezhny f56229bcce Dockerfile 2020-08-13 00:29:44 -04:00
Lex Berezhny abf1247f61 runs-on github action 2020-08-13 00:20:45 -04:00
Lex Berezhny 4b230a97f9 docker 2020-08-13 00:17:43 -04:00
Lex Berezhny bf7ac1562f docker 2020-08-12 03:34:37 -04:00
Lex Berezhny 2000d75c7a start --full-node 2020-08-12 00:08:06 -04:00
Lex Berezhny 1259002b51 systemd script installation 2020-08-11 22:50:24 -04:00
Lex Berezhny 70d9f4cf79 convert claim_id to channel_hash 2020-08-10 18:53:00 -04:00
Lex Berezhny 23723f8041 added support_search to api 2020-08-10 18:37:21 -04:00
Lex Berezhny 51a0f7ddc8 add index for release_time 2020-08-10 17:48:26 -04:00
Lex Berezhny 1228700487 typo 2020-08-10 14:10:15 -04:00
Lex Berezhny 67c5c192f3 fix order_by and include_totals 2020-08-10 14:03:12 -04:00
Lex Berezhny 6b9cf5b48c wip simulate_sync_console 2020-08-07 20:45:52 -04:00
Lex Berezhny 0a91bd35c5 db.resolve takes list 2020-08-06 11:45:23 -04:00
Lex Berezhny 84639cfb2e protobuf claim_search 2020-08-04 13:49:59 -04:00
Lex Berezhny 0b245aab31 use base64 for protobuf response 2020-08-04 11:47:03 -04:00
Lex Berezhny ee3db31541 protobuf resolve response 2020-08-04 10:41:49 -04:00
Lex Berezhny 8a3b960a85 delete tag when rewinding 2020-08-03 22:36:14 -04:00
Lex Berezhny 2952609972 fixed bug when maintenance sync spanned multiple files 2020-08-03 16:53:40 -04:00
Lex Berezhny 9ab8a7dd81 magic numbers 2020-08-03 12:24:13 -04:00
Lex Berezhny a7555932a9 Merge pull request #3012 from lbryio/unclosed_warnings_fix
fix unclosed transport warnings on tests
2020-08-03 10:33:51 -04:00
Victor Shyba 9411b26fd3 fix unclosed transport warnings on tests 2020-08-03 03:30:29 -03:00
Lex Berezhny 85db7d3ce7 lbrycrd print text response 2020-07-31 14:38:45 -04:00
Lex Berezhny 87f1d5b0ae added lbrycrd-rpc-user/pass 2020-07-30 14:51:04 -04:00
Lex Berezhny 64b8caeb5c cleanup lbrycrd 2020-07-29 11:46:44 -04:00
Lex Berezhny 3315175d1c raise unauthorized error when cannot connect to lbrycrd 2020-07-29 11:32:43 -04:00
Lex Berezhny a802d1f686 clear tasks 2020-07-29 00:41:40 -04:00
Lex Berezhny 684e389283 basicConfig 2020-07-29 00:35:39 -04:00
Lex Berezhny 895719a13d basic logging 2020-07-29 00:14:40 -04:00
Lex Berezhny 8f2cce7f61 removed extraneous @staticmethod annotation 2020-07-27 12:33:40 -04:00
Lex Berezhny 9114a9794d fix channel integration test 2020-07-27 12:19:12 -04:00
Lex Berezhny a1f3254261 lint 2020-07-27 11:52:24 -04:00
Lex Berezhny dbc0da2817 event generator and better ZMQ handling 2020-07-27 10:58:57 -04:00
Lex Berezhny 412ace1c6f remove assert causing failures in console.py 2020-07-22 18:07:18 -04:00
Lex Berezhny 8100efb48c expose api as POST not GET 2020-07-20 23:48:27 -04:00
Lex Berezhny 8d164dfed3 skipIf fix 2020-07-20 23:38:21 -04:00
Lex Berezhny ffea76cdd5 skip event buffering tests on all OSes except linux 2020-07-20 23:36:47 -04:00
Lex Berezhny 72ddb0c195 use normalized instead of claim_name for claim search 2020-07-20 22:44:18 -04:00
Lex Berezhny c57080711e added 99 as special number for no processes 2020-07-20 10:46:59 -04:00
Lex Berezhny 96aea579ac temporary debug print statements 2020-07-20 09:57:52 -04:00
Lex Berezhny e2aae23575 commit db transaction after setting work_mem 2020-07-16 23:27:24 -04:00
Lex Berezhny 2ae700feb3 vaccum claim after updating channel stats 2020-07-16 10:53:27 -04:00
Lex Berezhny d1ac066c6d include claim_hash in txo_claim_chahges postgres index 2020-07-16 10:53:03 -04:00
Lex Berezhny b61424979d lint 2020-07-15 23:50:52 -04:00
Lex Berezhny ca10874006 add vacuuming for tx table 2020-07-14 17:11:06 -04:00
Lex Berezhny a4680878c4 fixed issues with database transaction isolation 2020-07-14 16:46:24 -04:00
Lex Berezhny 1c29ae7204 run vacuum after every sync 2020-07-14 13:26:46 -04:00
Andrey Beletsky 86069b10ca Bind API server to the configured host instead of localhost 2020-07-14 14:56:47 +07:00
Lex Berezhny 9c5e2a8c8d fix none exception 2020-07-14 00:11:30 -04:00
Lex Berezhny 622a3b77ef indent 2020-07-14 00:05:25 -04:00
Lex Berezhny 0dff82c31c save all claims, even bad ones, also fix cli 2020-07-13 23:58:49 -04:00
Lex Berezhny 8e683c9cd0 indexing fixes 2020-07-13 21:00:24 -04:00
Lex Berezhny 69c45d43d3 fixed multi block file test 2020-07-13 18:32:44 -04:00
Lex Berezhny fab7b5579c skip test unreliable on mac 2020-07-13 18:21:41 -04:00
Lex Berezhny 9e87394fca lower event stress test params since it fails too easily on CI, attempt 3 2020-07-13 16:57:34 -04:00
Lex Berezhny 8c6633de17 lower event stress test params since it fails too easily on CI, attempt 2 2020-07-13 16:53:02 -04:00
Lex Berezhny 3af71a2674 lower event stress test params since it fails too easily on CI 2020-07-13 16:45:35 -04:00
Lex Berezhny b792b134a2 require ubuntu 20.04 for integration tests 2020-07-13 16:41:08 -04:00
Lex Berezhny f50196d395 fix channel integration test 2020-07-13 15:50:00 -04:00
Lex Berezhny 248e04089b pylint 2020-07-13 15:45:21 -04:00
Lex Berezhny 8fd92cb649 more indexes 2020-07-13 14:29:38 -04:00
Lex Berezhny af4138ff51 minor progress refinement 2020-07-13 13:12:01 -04:00
Lex Berezhny 462daf4dc4 increase work_mem and add more indexes 2020-07-13 12:38:28 -04:00
Lex Berezhny e63151a370 merged channel and content syncs and no longer storing canonical_url in column 2020-07-13 09:30:32 -04:00
Lex Berezhny 09a2b2fa46 wider labels 2020-07-13 00:59:44 -04:00
Lex Berezhny a3d91329fe performance 2020-07-13 00:55:30 -04:00
Lex Berezhny 7bf96fd637 silence invalid name errors when creating short_url 2020-07-12 18:07:12 -04:00
Lex Berezhny 5157b2535b correctly update main bar 2020-07-12 17:57:41 -04:00
Lex Berezhny 0151ce8040 use correct last progress bar state value 2020-07-12 17:29:58 -04:00
Lex Berezhny 5328ed105e reduce TX_FLUSH_SIZE 2020-07-12 16:44:36 -04:00
Lex Berezhny 7f01b1cb84 flipped main and secondary progress bars 2020-07-12 16:43:44 -04:00
Lex Berezhny 862c51946a test robustness 2020-07-12 16:28:13 -04:00
Lex Berezhny 1790ee3018 bug fixes and performance improvements 2020-07-12 16:27:51 -04:00
Lex Berezhny 7a4e5dcb05 handle claims in abandoned channels 2020-07-12 12:02:58 -04:00
Lex Berezhny 24a88db595 console fix 2020-07-12 09:22:06 -04:00
Lex Berezhny 915233c96c fixes 2020-07-12 08:55:25 -04:00
Lex Berezhny aa9365f218 refactored sync 2020-07-11 18:18:33 -04:00
Lex Berezhny 15b8891fce make resolve command work from cli 2020-07-07 10:52:41 -04:00
Lex Berezhny f8a8a75ae9 regenerated interface metadata 2020-07-07 10:49:40 -04:00
Lex Berezhny d18ed6c19b wrap short url creation in try/except 2020-07-07 00:20:11 -04:00
Lex Berezhny 4aa44d3b5a set work_mem for postgres 2020-07-06 22:42:15 -04:00
Lex Berezhny 34a9dff141 fixed conflict with two cursors when one is reading and another is commiting 2020-07-06 09:18:22 -04:00
Lex Berezhny 7d9bf03574 refactored 2020-07-05 23:03:45 -04:00
Lex Berezhny 3fe1981657 faster inputs/outputs sync 2020-07-03 15:47:25 -04:00
Lex Berezhny 192c79c49c capture stderr before process pool starts 2020-07-01 18:32:20 -04:00
Lex Berezhny 5883c9bc6c hide error from protobuf 2020-06-30 23:13:38 -04:00
Lex Berezhny 4b50d1e329 vacuum before setting inputs 2020-06-30 19:37:00 -04:00
Lex Berezhny 39d8a20fd5 pgcopy COPY command 2020-06-30 17:32:51 -04:00
Lex Berezhny 9ccf00f56b add pgcopy to setup.py 2020-06-30 12:18:03 -04:00
Lex Berezhny 46662b55c7 clean up build workflow 2020-06-30 12:17:48 -04:00
Lex Berezhny b45a222f98 event subscription and publishing bug fixes 2020-06-29 18:10:26 -04:00
Lex Berezhny 434c1bc6b3 test_nulls 2020-06-27 23:23:57 -04:00
Lex Berezhny 2495df8859 null in claim description 2020-06-27 23:14:28 -04:00
Lex Berezhny 635aebfeeb fixed null claim name support for postgres 2020-06-27 23:00:12 -04:00
Lex Berezhny 81926a42f9 fix postgres db connection 2020-06-27 22:31:49 -04:00
Lex Berezhny f2ff4410dc re-enable postgres integration tests 2020-06-27 22:24:59 -04:00
Lex Berezhny 564018c937 better debug badrow 2020-06-27 11:46:49 -04:00
Lex Berezhny 4d1eafc0a4 debug badrow 2020-06-27 09:39:23 -04:00
Lex Berezhny 211f8b2e59 pause console elapsed timer when no blocks read/save 2020-06-27 09:22:52 -04:00
Lex Berezhny 9500be26fd no order by file 2020-06-26 23:37:10 -04:00
Lex Berezhny 017ef5b41a order by file 2020-06-26 23:35:11 -04:00
Lex Berezhny 4b19861a74 console improvements 2020-06-26 21:52:01 -04:00
Lex Berezhny 5a0a987f0c refactored and simplified the blockchain sync 2020-06-26 10:39:58 -04:00
Lex Berezhny 4cb4659489 removed Takeovers for now 2020-06-26 10:39:16 -04:00
Lex Berezhny e64b108404 console 2020-06-23 13:11:16 -04:00
Lex Berezhny 7870abaef4 script for testing console progress bars 2020-06-23 12:28:01 -04:00
Lex Berezhny 71e14c8e63 least 2020-06-22 20:30:05 -04:00
Lex Berezhny b3b6361429 fix test 2020-06-22 18:47:10 -04:00
Lex Berezhny 0d5441f3bf update 2020-06-22 11:44:27 -04:00
Lex Berezhny 9757c69189 pylint 2020-06-22 11:02:59 -04:00
Lex Berezhny d8fb31aedd addeed greatest sql function that supports postgres and sqlite 2020-06-22 10:52:17 -04:00
Lex Berezhny 54a0bf9290 fixups to make lbrynet work on cli with postgres 2020-06-21 23:21:43 -04:00
Lex Berezhny a3ef8d7411 pylint 2020-06-21 20:14:14 -04:00
Lex Berezhny 4810ff5f94 run more tests on github workflow 2020-06-21 19:55:13 -04:00
Lex Berezhny 2306edebf7 tests/integration/blockchain/test_blockchain.py 2020-06-21 19:54:59 -04:00
Lex Berezhny db5a33dc3f blockchain/test_database.py 2020-06-21 19:54:34 -04:00
Lex Berezhny 597bebb5be tests/unit/db 2020-06-21 19:53:18 -04:00
Lex Berezhny 73ff1d3b3a run more tests 2020-06-21 19:52:59 -04:00
Lex Berezhny 9198877098 include_is_spent no longer needed 2020-06-21 19:52:39 -04:00
Lex Berezhny 46da2584ca db 2020-06-21 19:51:09 -04:00
Lex Berezhny 53b7d0a58b blockchain 2020-06-21 19:50:53 -04:00
Lex Berezhny d1a243247d progress 2020-06-19 14:28:34 -04:00
Lex Berezhny 18dc5fbc9f run crypto and schema tests 2020-06-10 15:45:57 -04:00
Lex Berezhny 410212c17a update tests 2020-06-10 15:45:29 -04:00
Lex Berezhny a39f87b3c5 more resilient test 2020-06-10 15:25:45 -04:00
Lex Berezhny 147b9d5ad1 run another integration test 2020-06-09 23:52:08 -04:00
Lex Berezhny 1f210c0b0b skip event queue test until it can be more reliable 2020-06-09 23:42:21 -04:00
Lex Berezhny 86df4bdd11 run more unittests 2020-06-09 23:28:07 -04:00
Lex Berezhny 096f74d79b silence expected exception 2020-06-09 23:27:45 -04:00
Lex Berezhny ae8bc59c65 remove duplicate dewies test 2020-06-09 23:27:19 -04:00
Lex Berezhny 01dbbb4c3a skip blockchain sync unittest 2020-06-09 23:26:59 -04:00
Lex Berezhny 96d1926da4 upgrad pyyaml 2020-06-09 23:25:42 -04:00
Lex Berezhny 2c10e71774 coverage on windows 2020-06-09 20:59:32 -04:00
Lex Berezhny ea6be53071 run all steps 2020-06-09 20:55:16 -04:00
Lex Berezhny cb5250f630 integration coverage 2020-06-09 20:48:38 -04:00
Lex Berezhny 54e83daa59 back to pinned coverallsapp 2020-06-09 18:13:27 -04:00
Lex Berezhny df44d6ef56 back to bboe 2020-06-09 17:13:07 -04:00
Lex Berezhny aa4ef94e15 andremiras coveralls - spacing tests-unit 2020-06-09 17:00:26 -04:00
Lex Berezhny fad144cb96 andremiras coveralls - spacing 2020-06-09 16:59:34 -04:00
Lex Berezhny 1fe444bca2 andremiras coveralls 2020-06-09 16:58:22 -04:00
Lex Berezhny 55196ccb6b master github action 2020-06-09 16:46:59 -04:00
Lex Berezhny b3cb50aff0 back to bboe 2020-06-09 16:30:55 -04:00
Lex Berezhny ebf36f513c re-check default coveralls package 2020-06-09 14:03:26 -04:00
Lex Berezhny 47d207ff77 bboe coveralls 2020-06-09 13:56:01 -04:00
Lex Berezhny d99e4221f2 pin coveralls app github action 2020-06-09 13:43:26 -04:00
Lex Berezhny 56ff1342c4 workflow typo 2020-06-09 13:01:18 -04:00
Lex Berezhny a8c8614948 workflow indentation 2020-06-09 13:00:24 -04:00
Lex Berezhny 0e9184048c try coveralls again 2020-06-09 12:58:05 -04:00
Lex Berezhny c01dceebcd try codecov python lib 2020-06-06 21:00:38 -04:00
Lex Berezhny e3cc6ea224 moved codecov into tox 2020-06-06 20:54:34 -04:00
Lex Berezhny 44bbd9578d sans build 2020-06-06 20:19:09 -04:00
Lex Berezhny 7855f9c93f sans unit tests 2020-06-06 20:02:31 -04:00
Lex Berezhny 76e21f65df fix codecov test flags 2020-06-06 13:39:03 -04:00
Lex Berezhny 53c8876b5e fix codecov test flags 2020-06-06 13:27:59 -04:00
Lex Berezhny 2927875830 placeholder classes to pass pylint 2020-06-06 13:20:26 -04:00
Lex Berezhny b9d954a394 install coveragge 2020-06-06 13:10:42 -04:00
Lex Berezhny a0fb3424aa more codecov experiments 2020-06-06 13:01:53 -04:00
Lex Berezhny 269c0f714e trying without codecov github action 2020-06-06 12:52:54 -04:00
Lex Berezhny 99d2a3f42b debugging codecov 2020-06-06 12:44:54 -04:00
Lex Berezhny a1aa578bc0 codecov2 2020-06-06 12:23:32 -04:00
Lex Berezhny 82062b5601 trying codecov 2020-06-06 12:20:19 -04:00
Lex Berezhny db1f984558 moved fast_sync.py script 2020-06-05 19:21:39 -04:00
Lex Berezhny e66445b46e deleted lbry.extras 2020-06-05 19:18:00 -04:00
Lex Berezhny 82b69109bd pylint 2020-06-05 00:51:55 -04:00
Lex Berezhny ec4e36446c update lbrycrd download URL 2020-06-05 00:38:57 -04:00
Lex Berezhny 3ef83febc0 progress 2020-06-05 00:35:22 -04:00
Lex Berezhny ffecd02fbc deleted db/search 2020-06-05 00:31:26 -04:00
Lex Berezhny ebddb1f0f5 moved exchange rate manager 2020-06-04 18:48:44 -04:00
Lex Berezhny 9d0b9805b2 github actions install psycopg2 2020-06-04 18:17:37 -04:00
Lex Berezhny 8653839c16 no tests run on github actions, just build binaries for now 2020-06-04 17:52:10 -04:00
Lex Berezhny 29b1f93699 switch cli to use pre-generated interface 2020-06-04 17:51:01 -04:00
Lex Berezhny 97c285f22b removed redundant assert equal 2020-06-04 17:50:21 -04:00
Lex Berezhny c12e07de11 stricter check for old windows directory structure 2020-06-04 17:49:59 -04:00
Lex Berezhny 726bae97b0 generated lbry.service.metadata 2020-06-04 17:29:34 -04:00
Lex Berezhny 5efb36ffd2 windows & mac path tests 2020-06-04 17:29:34 -04:00
Lex Berezhny b4d6b14599 + tqdm 2020-06-04 17:29:34 -04:00
Lex Berezhny 9a40381f5a imports 2020-06-04 17:29:34 -04:00
Lex Berezhny 47f6d542c5 import fix 2020-06-04 17:29:34 -04:00
Lex Berezhny 8799caa0e4 github actions 2020-06-04 17:29:34 -04:00
Lex Berezhny a042377a7b run unittest on multiple platforms 2020-06-03 19:43:08 -04:00
Lex Berezhny 596ed08395 EventQueuePublisher uses a buffer to reduce number of tasks created 2020-05-22 18:40:21 -04:00
Lex Berezhny 2af29b892b new cli 2020-05-20 18:05:49 -04:00
Lex Berezhny 7ffb169376 Service.run -> Daemon.run 2020-05-20 18:05:13 -04:00
Lex Berezhny d11f4f9bed switched Client class from plain GET RPC to WebSockets 2020-05-20 18:04:18 -04:00
Lex Berezhny f2e844c476 ignore warnings when parsing old claims 2020-05-20 18:02:47 -04:00
Lex Berezhny 83f7eab0e7 tests for directory configurability 2020-05-20 18:02:05 -04:00
Lex Berezhny 4a9f9906a0 event streams can now be "closed" and thus you can listen for .last event 2020-05-20 18:01:34 -04:00
Lex Berezhny b7ff6569e4 added Database.temp_sqlite_regtest 2020-05-20 17:59:26 -04:00
Lex Berezhny 12915143b8 improved directory structure configurability 2020-05-20 17:57:42 -04:00
Lex Berezhny 06d93e667a event reporting from blockchain.sync 2020-05-20 17:54:38 -04:00
Lex Berezhny b341187b14 moved necessary code to get download directory to upstream appdirs and deleted winpaths.py 2020-05-19 17:05:03 -04:00
Lex Berezhny a996e65eff integration tests updated 2020-05-18 08:29:15 -04:00
Lex Berezhny 0886a7946e unit tests updated 2020-05-18 08:28:23 -04:00
Lex Berezhny be6ebf0047 refactored wallet and mnemonic 2020-05-18 08:26:36 -04:00
Lex Berezhny 7c4f943bcb updated to testcase 2020-05-18 08:24:44 -04:00
Lex Berezhny 5b5c45ea76 API is now typed and includes sharable argument lists 2020-05-18 08:24:15 -04:00
Lex Berezhny 6986211c1e lbry.blockchain fixes and cleanup 2020-05-18 08:22:23 -04:00
Lex Berezhny 8ac78990d8 updated setup.py and lbry/__init__.py now includes imports to solve cyclic import issues 2020-05-18 08:20:13 -04:00
Lex Berezhny 4f879bbbae new cli 2020-05-18 08:15:53 -04:00
Lex Berezhny e3f080a7ad extract singular tags, languages, locations from kwargs 2020-05-18 08:15:24 -04:00
Lex Berezhny d232eeaf81 plain execute function 2020-05-18 08:14:30 -04:00
Lex Berezhny 5d6388b366 added db_url to config 2020-05-18 08:13:29 -04:00
Lex Berezhny 955e44631d new API parser 2020-05-12 11:02:34 -04:00
Lex Berezhny 8dc5150dbe moved test fixture file 2020-05-12 10:35:34 -04:00
Lex Berezhny d488bfd9d4 refactored mnemonic.py 2020-05-09 18:33:23 -04:00
Lex Berezhny 391b95fd12 lbry.wallet.words minified removing line breaks and extra space and converted to tuples 2020-05-08 16:25:56 -04:00
Lex Berezhny 4b172e4180 updated testcase to use api instead of daemon 2020-05-08 10:34:57 -04:00
Lex Berezhny 29ef4425b0 fix import in managed_stream 2020-05-08 10:33:41 -04:00
Lex Berezhny 558b1aeadf made lbry.db.queries.add_channel_keys_to_txo_results async 2020-05-08 10:30:40 -04:00
Lex Berezhny 41ce3e4ad8 fix Output source offset calculation and made generat_channel_private_key async 2020-05-08 10:23:50 -04:00
Lex Berezhny 8c91777e5d refactored lbry.wallet 2020-05-06 10:53:31 -04:00
Lex Berezhny db89607e4e added create_default_wallet and create_default_account conf options 2020-05-06 10:50:29 -04:00
Lex Berezhny d476f08d13 convenience methods on ledger 2020-05-06 10:50:00 -04:00
Lex Berezhny 2a0c653c37 claim_search progress 2020-05-01 23:25:07 -04:00
Lex Berezhny 219c7cf37d removed lbry.wallet.server and lbry.wallet.orchstr8 2020-05-01 23:22:17 -04:00
Lex Berezhny 2f575a393f minor updates to stream_manager.py 2020-05-01 09:34:57 -04:00
Lex Berezhny 713c665588 unittests 2020-05-01 09:34:34 -04:00
Lex Berezhny 9554b66a37 wip lbry.service 2020-05-01 09:33:58 -04:00
Lex Berezhny 533f31cc89 wip lbry.wallet 2020-05-01 09:33:10 -04:00
Lex Berezhny fef09c1773 wip misc 2020-05-01 09:31:33 -04:00
Lex Berezhny 6a33d86bfe wip lbry.db 2020-05-01 09:29:44 -04:00
Lex Berezhny ccd32eae70 wip lbry.blockchain 2020-05-01 09:28:51 -04:00
Lex Berezhny c61c9726b0 lbry/blockchain/constants.py -> lbry/constants.py 2020-04-28 13:28:30 -04:00
Lex Berezhny fd5be69d55 lbry/wallet/server/db/{reader,fts} -> lbry/db 2020-04-28 12:26:17 -04:00
Lex Berezhny 1f72751a88 unit test __init__.py files 2020-04-25 08:58:18 -04:00
Lex Berezhny 362ab67186 more wallet unit test cleanup 2020-04-25 08:51:41 -04:00
Lex Berezhny ffe7fcf124 refactored wallet unit tests, moved most things to blockchain 2020-04-25 08:44:36 -04:00
Lex Berezhny 1f5dbc3eb8 renamed: wallet/stream.py -> event.py 2020-04-25 08:29:25 -04:00
Lex Berezhny 4dd85a169b cleaning up wallet directory, mostly moving things to blockchain 2020-04-25 08:25:29 -04:00
Lex Berezhny fe547f1b0e dropping custom wallet rpc implementation 2020-04-25 08:05:46 -04:00
Lex Berezhny ba154c799e increase timeout for wallet server payment service 2020-04-12 16:14:18 -04:00
Lex Berezhny e2ffd24d51 listen for wallet pay service payment earlier in test 2020-04-12 15:51:18 -04:00
Lex Berezhny 8545ab880b psycopg2 returns Decimal for SUM() 2020-04-12 15:30:25 -04:00
Lex Berezhny 4e85f34353 pylint 2020-04-12 12:05:03 -04:00
Lex Berezhny 777c6342f8 merged some old stashed code 2020-04-12 11:59:00 -04:00
Lex Berezhny 10c262a095 postgres 2020-04-12 11:06:05 -04:00
Lex Berezhny 74e3471bd9 setup.py 2020-04-12 09:26:59 -04:00
Lex Berezhny 4048cfb3e8 Update node.py 2020-04-11 21:20:02 -04:00
Lex Berezhny 8b7b284c0d Update setup.py 2020-04-11 21:07:07 -04:00
Lex Berezhny 2d4d51388b Update setup.py 2020-04-11 20:54:08 -04:00
Lex Berezhny 06b75d07dc sudo 2020-04-11 20:21:28 -04:00
Lex Berezhny 87a44fd41c syntax 2020-04-11 20:20:21 -04:00
Lex Berezhny 70693f4d1a apt-get update 2020-04-11 20:17:55 -04:00
Lex Berezhny 42224dadb6 pylint 2020-04-11 20:15:04 -04:00
Lex Berezhny 53fc94d688 pylint and sqlalchemy 1.4 2020-04-11 20:01:10 -04:00
Lex Berezhny 0b6d01fecc sqlalchemy 2020-04-11 17:31:20 -04:00
Lex Berezhny ae9d4af8c0 postgresql support added 2020-04-11 17:27:41 -04:00
Lex Berezhny 2309d6354c schema chaining 2020-04-08 19:03:01 -04:00
Lex Berezhny fd2f9846e9 parsing and inserting works 2020-04-08 19:03:01 -04:00
Lex Berezhny 66666e1167 script to generate blockchain test data 2020-04-08 19:03:01 -04:00
Lex Berezhny 4c0fbb84d6 reproduce issue with lbrycrd -reindex 2020-04-08 19:03:01 -04:00
Lex Berezhny fd3448ffb8 remove plyvel and add pyzmq 2020-04-08 19:03:01 -04:00
Lex Berezhny 6d93f97b51 Transaction._deserialize now optionally accepts stream and returns self 2020-04-08 19:03:01 -04:00
Lex Berezhny 2ee13ce39f BCDataStream can now also take file pointer as argument 2020-04-08 19:03:01 -04:00
Lex Berezhny bad4320ddf initial import of fast sync 2020-04-08 19:03:01 -04:00
Lex Berezhny a220736dea updated docs 2020-04-08 12:57:24 -04:00
Alex Grintsvayg 149d343201 drop a few unused conf vars 2020-04-07 16:11:23 -04:00
Lex Berezhny 496cc79ba8 v0.68.0 2020-04-06 12:55:21 -04:00
Jack Robison 3591768745 Merge pull request #2908 from lbryio/batched-resolve
Fix connection to wallet server breaking upon giant resolve requests
2020-04-06 12:52:51 -04:00
Jack Robison d615f6761a automatically batch large resolve requests 2020-04-06 12:03:33 -04:00
Jack Robison 5b29894048 add reset clients counter to prometheus 2020-04-06 12:03:33 -04:00
Lex Berezhny d17f0cfa40 Merge pull request #2909 from lbryio/none_for_unconfirmed_time
do not estimate timestamp for unconfirmed txs
2020-04-06 11:41:17 -04:00
Victor Shyba b2f70c7120 return none for unconfirmed time estimation 2020-04-06 06:03:35 -03:00
Jack Robison 238707bd93 Merge pull request #2897 from lbryio/reorg-claims
handle claim reorgs in the wallet server
2020-04-05 17:48:56 -04:00
Jack Robison 87cdf1e3a0 improve test_reorg_change_claim_height 2020-04-05 16:58:36 -04:00
Jack Robison 57fd47022e test_reorg_change_claim_height 2020-04-03 13:39:44 -04:00
Jack Robison 5eafd3bf6b feedback 2020-04-03 13:39:44 -04:00
Jack Robison 640b5b0ea9 delete_claims_above_height with thread lock 2020-04-03 13:39:44 -04:00
Jack Robison e4fb2f4680 test_reorg_dropping_claim 2020-04-03 13:39:44 -04:00
Jack Robison a4909f54e4 test reorg count metric 2020-04-03 13:39:44 -04:00
Jack Robison f7065c6f0c add reorg count metric to prometheus 2020-04-03 13:39:44 -04:00
Jack Robison 64f7f837e7 delete claims above reorg height from the database 2020-04-03 13:39:44 -04:00
Jack Robison 130d36acd9 Merge pull request #2905 from lbryio/testnet-headers
fix header checkpoints on testnet
2020-04-03 13:38:47 -04:00
Jack Robison e94c28cfa2 fix header checkpoints on testnet 2020-04-03 12:21:17 -04:00
Jack Robison a697dec30c Merge pull request #2902 from lbryio/bump_checkpoints
bump checkpoints
2020-04-03 12:20:38 -04:00
Victor Shyba bac09e9b9f bump checkpoints 2020-04-03 12:26:52 -03:00
Lex Berezhny b179f8f86a Merge pull request #2901 from lbryio/txo_is_spent_filter
replaced `--unspent` with `--is_not_spent` on several commands and added `--is_spent` to many claim/txo `*_list` commands, `support_list` got three new flags `--sent --received --staked`
2020-04-02 22:24:47 -04:00
Lex Berezhny 962d04ae17 fix txo_spend 2020-04-01 21:03:56 -04:00
Lex Berezhny f28e3bfe37 lint 2020-04-01 20:53:09 -04:00
Lex Berezhny 6474c86d32 cleaned up *_list commands 2020-04-01 20:44:34 -04:00
Lex Berezhny 769ea8cdfe added --is_spent filter to txo list/sum commands 2020-03-31 23:08:51 -04:00
Lex Berezhny 052e77dd5a v0.67.2 2020-03-31 17:47:19 -04:00
Lex Berezhny 1a4b0cb12a Merge pull request #2900 from lbryio/wallet_status_does_not_fail
`wallet_status` no longer fails if wallet component has not started
2020-03-31 17:45:51 -04:00
Lex Berezhny f9aa95c987 default to None for all values in wallet_status when wallet_manager not started yet 2020-03-31 17:22:13 -04:00
Lex Berezhny a5d06fb4a4 wallet_status no longer fails if wallet component has not started 2020-03-31 16:20:13 -04:00
Lex Berezhny d5e5d90bdc v0.67.1 2020-03-31 12:38:38 -04:00
Lex Berezhny e62678e4e6 Merge pull request #2898 from lbryio/duration_is_int
`ffmpeg` file analysis returns duration as integer now
2020-03-31 12:06:44 -04:00
Lex Berezhny 558ac24f7e fix test 2020-03-31 11:29:58 -04:00
Lex Berezhny 6f22f6a59f use ceil() on duration float() instead of int() directly 2020-03-31 10:57:37 -04:00
Lex Berezhny 5ec74f8abe ffmpeg file analysis returns duration as integer now 2020-03-31 10:28:04 -04:00
Lex Berezhny 16d7547e03 changed default txo_spend batch_size default to 500 from 1000 2020-03-30 22:48:05 -04:00
Lex Berezhny 767112dcda updated txo_spend docs 2020-03-30 21:45:58 -04:00
Lex Berezhny 81e23d1d8c v0.67.0 2020-03-30 19:43:45 -04:00
Lex Berezhny ace90a4354 Merge pull request #2890 from lbryio/faster-status
improve speed of `status`
2020-03-30 19:42:57 -04:00
Jack Robison 267e7096cc add test for rechecking ffmpeg installation 2020-03-30 19:11:04 -04:00
Jack Robison e7cded7511 check ffmpeg/ffmprobe paths in a thread 2020-03-30 19:11:04 -04:00
Jack Robison 33fbd715c0 don't block status on connectivity check 2020-03-30 19:11:04 -04:00
Jack Robison 25ba5b867c dont recheck ffmpeg installation in status 2020-03-30 19:11:04 -04:00
Lex Berezhny c8f431447c Merge pull request #2885 from lbryio/repair_tip_on_open
repair headers tip on open and let new headers come in during background fetching
2020-03-30 19:03:09 -04:00
Victor Shyba 1b83a1d09a test and fix verifying from middle 2020-03-30 19:42:27 -03:00
Victor Shyba d2fb7a7151 lock only when fetching, giving a chance for tip updates 2020-03-30 19:42:27 -03:00
Victor Shyba 7ad3447598 repair tip on open 2020-03-30 19:42:27 -03:00
Lex Berezhny 5e5bc8e705 Merge pull request #2887 from lbryio/txo_spend
added `txo_spend` command to support liquidating large number of txos (eg. tips) by batching them across several transactions
2020-03-30 18:41:05 -04:00
Lex Berezhny 886d1e8a19 added --include_full_tx option to txo_list 2020-03-30 18:15:13 -04:00
Lex Berezhny 6de7a035fa added preview/blocking back into doc string 2020-03-30 17:47:38 -04:00
Lex Berezhny 48d2497eb2 added txo_spend command to support liquidating large number of txos (eg. tips) 2020-03-30 17:47:38 -04:00
Lex Berezhny 6494754ab9 Merge pull request #2896 from lbryio/first_on_ready
fix for startup issues when connecting to wallet servers
2020-03-30 17:41:14 -04:00
Lex Berezhny a8153627c6 move on_read.first to earlier 2020-03-30 17:02:08 -04:00
Lex Berezhny ed38966edb add test to verify we listen to on_ready before it actually triggers 2020-03-30 15:26:04 -04:00
Lex Berezhny 85551d1e54 remove no-op test, segwit is always on now 2020-03-30 15:17:25 -04:00
Lex Berezhny ca31363180 listen for on_read.first before it is triggered 2020-03-30 14:53:52 -04:00
Lex Berezhny a8f11eb3c3 Merge pull request #2893 from lbryio/fix_float_timestamp
fix estimated timestamp to return integer instead of float
2020-03-30 10:58:32 -04:00
Lex Berezhny 151805121c increase wallet server payment test timeout 2020-03-29 20:00:23 -04:00
Lex Berezhny 0422d2a021 estimated timestamp should be integer 2020-03-29 19:39:37 -04:00
Lex Berezhny ffdf70257b Merge pull request #2888 from lbryio/m4a-content-type
add m4a content type
2020-03-27 17:54:45 -04:00
Thomas Zarebczan 6079d60aa1 add m4a content type 2020-03-26 23:08:52 -04:00
Jack Robison d46d012f8c Revert "delete live bittrex test"
This reverts commit f70343bb
2020-03-26 20:31:39 -04:00
Jack Robison baf926b360 v0.66.0 2020-03-26 17:59:02 -04:00
Jack Robison 82339869dc Revert "v0.66.0"
This reverts commit 4ab56ae3
2020-03-26 17:50:44 -04:00
Jack Robison f70343bb63 delete live bittrex test 2020-03-26 17:50:31 -04:00
Jack Robison 4ab56ae3d1 v0.66.0 2020-03-26 16:49:54 -04:00
Jack Robison 39b4031684 Merge pull request #2880 from lbryio/handle_mkv_files
Handle MKV files
2020-03-26 16:45:20 -04:00
Brannon King 71f8965393 re-use ffprobe info in stream_type
avoid duplicate args


handle review comment
2020-03-26 16:20:12 -04:00
Brannon King 66857e72a4 ensure only webm-type MKV files 2020-03-26 16:20:12 -04:00
Jack Robison 15eb5d47eb Merge pull request #2881 from lbryio/wallet_sync_status
add `wallet_syncing` to `status`, which is set to true when processing transaction updates
2020-03-26 15:28:20 -04:00
Jack Robison 86242139da update tests 2020-03-26 13:56:51 -04:00
Victor Shyba 460bdc4148 move wallet_syncing to wallet status is_syncing 2020-03-26 11:43:48 -04:00
Victor Shyba 1052126522 add wallet_syncing status 2020-03-26 11:43:48 -04:00
Jack Robison fdb42ac876 Merge pull request #2852 from lbryio/faster-transaction-sync
More efficient syncing of wallet transactions
2020-03-26 11:43:20 -04:00
Jack Robison 6486f986e8 bump min required server version 2020-03-26 11:41:49 -04:00
Jack Robison bc4075e2ed create_task instead of ensure_future 2020-03-26 11:41:49 -04:00
Jack Robison 11525c7d0d fix caching transaction_show 2020-03-26 11:41:49 -04:00
Jack Robison bb12ae9ce6 fix duplicate get_merkle requests 2020-03-26 11:41:49 -04:00
Jack Robison 66b4ad0c44 improve log 2020-03-26 11:41:49 -04:00
Jack Robison 4bff2d718e clean up test 2020-03-26 11:41:49 -04:00
Jack Robison af7a7b4dc6 fix maybe_migrate_certificates blocking the api while populating channel signing keys 2020-03-26 11:41:49 -04:00
Jack Robison ec20d9a2a8 faster wallet sync with get_transaction_and_merkle 2020-03-26 11:41:49 -04:00
Lex Berezhny b0aee3d335 Merge pull request #2884 from lbryio/txo_plot
added `txo_plot` command to allow plotting txo sums over time
2020-03-26 10:14:02 -04:00
Lex Berezhny 5e2ddbfd86 txo_plot returns lbc instead of dewies 2020-03-26 01:13:09 -04:00
Lex Berezhny f086ebbb8e lint 2020-03-26 00:42:11 -04:00
Lex Berezhny e5bf6a5bfc added txo_plot command to allow plotting txo sums over time 2020-03-26 00:37:13 -04:00
Lex Berezhny 76fa86d54b v0.65.0 2020-03-23 14:56:45 -04:00
Jack Robison 60d89506a5 Merge pull request #2877 from lbryio/on_ready_fixup
wait for tx sync to finish but not header sync
2020-03-23 13:46:47 -04:00
Victor Shyba 2893f1eb9e add other taskgroup for non-start-important tasks 2020-03-23 12:36:05 -03:00
Victor Shyba a08cbf412d on_ready.wait so we start after tx syncs 2020-03-23 12:26:07 -03:00
Lex Berezhny 0bd65356f9 increase timeout in usage payment test 2020-03-23 11:06:29 -04:00
Lex Berezhny 2e2b39455f Merge pull request #2875 from lbryio/1080p
default to 1080P for anything higher than 1080P
2020-03-23 10:44:18 -04:00
Lex Berezhny 1f9fbe34e4 Merge pull request #2863 from lbryio/async_headers
fetch headers on demand
2020-03-23 10:41:31 -04:00
Thomas Zarebczan 506d7ae50e default to 1080P for anything higher than 1080P
Previously would do 1440p
2020-03-23 10:20:13 -04:00
Victor Shyba 342cb00625 less concurrent repeated header checks 2020-03-23 01:19:34 -03:00
Victor Shyba 952fc01efd add script that generates checkpoints 2020-03-23 00:05:36 -03:00
Victor Shyba 3eebe301fe move checkpoints out of folder into file 2020-03-22 23:45:14 -03:00
Victor Shyba 19c0a81c42 fix bad usages of hash and some tests 2020-03-22 23:31:47 -03:00
Victor Shyba 9fc7f9904b fix tests, delete old code 2020-03-22 23:31:47 -03:00
Victor Shyba b04a516063 better locking, stop corrupting headers, fix some tests 2020-03-22 23:31:47 -03:00
Victor Shyba 241e946d91 first experimental version 2020-03-22 23:31:45 -03:00
Victor Shyba e45375dc26 more async parts 2020-03-22 23:30:11 -03:00
Victor Shyba ec8e243323 estimate timestamps instead of using block headers 2020-03-22 23:30:11 -03:00
Victor Shyba af0e9368d4 headers get now async 2020-03-22 23:30:11 -03:00
Lex Berezhny 285483b81a Merge pull request #2872 from lbryio/claim_list_include_received_tips
added `--include_received_tips` to `claim_list` and `txo_list` commands
2020-03-22 18:00:35 -04:00
Lex Berezhny 73a91d5569 added --include_received_tips to claim_list/txo_list 2020-03-22 17:22:15 -04:00
Lex Berezhny e24c78be32 Merge pull request #2871 from lbryio/wallet_txo_data_leak
fix to prevent transaction cache from leaking some information about outputs between unrelated wallets
2020-03-22 13:16:03 -04:00
Lex Berezhny b11184de68 fix purchase_create 2020-03-22 12:23:21 -04:00
Lex Berezhny 61d02fc5d7 fix to prevent transaction cache from leaking some information about outputs between unrelated wallets 2020-03-22 11:45:40 -04:00
Lex Berezhny f0217f6821 Merge pull request #2870 from lbryio/resolve_includes
added new flags to `resolve` (and a few to `claim_search`) commands: `--include_purchase_receipt`, `--include_is_my_output`, `--include_sent_supports`, `--include_sent_tips` and `--include_received_tips`
2020-03-22 02:08:28 -04:00
Lex Berezhny 09644914a6 added --include_is_my_output to claim_search 2020-03-22 01:51:09 -04:00
Lex Berezhny b5c24d6a48 fixing unit tests 2020-03-22 01:29:26 -04:00
Lex Berezhny bdd2ac2c25 renamed some flags 2020-03-22 01:13:26 -04:00
Lex Berezhny 76376f0d33 lint 2020-03-22 00:24:38 -04:00
Lex Berezhny 9749da46ae added flags to resolve: --include_purchase_receipt, --include_is_my_output, --include_my_supports, --include_my_tips 2020-03-21 23:44:57 -04:00
Lex Berezhny 1c05295e89 updated doc 2020-03-21 20:15:21 -04:00
Lex Berezhny 4d0f28215a added release-text-lines: to release script to support multi-line release notes 2020-03-21 20:09:39 -04:00
Lex Berezhny 0bba72bc5a Merge pull request #2862 from lbryio/txo_list_support_sent_outputs
`txo_list` adds many new ownership filters such as `--is_my_input`, `--is_my_output`, etc and some metadata filters such as `--channel_id`, `--reposted_claim_id`, etc and a new command `txo_sum` which takes the same arguments as `txo_list` and produces sum of outputs
2020-03-21 19:48:34 -04:00
Lex Berezhny 87089b8e83 fix support test 2020-03-21 18:48:06 -04:00
Lex Berezhny 6a58148a89 added support for --order_by=none 2020-03-21 18:16:25 -04:00
Lex Berezhny 15091052be added --no_totals to txo_list 2020-03-21 18:06:05 -04:00
Lex Berezhny 5cd7e9a9b8 increment scema version and force specific indexes to be used for get_txos() 2020-03-21 15:08:14 -04:00
Lex Berezhny 7cb530c334 added --channel_id and --order_by to txo_list 2020-03-20 23:19:26 -04:00
Lex Berezhny 5e0324cc91 added --reposted_claim_id to txo_list 2020-03-20 20:22:57 -04:00
Lex Berezhny 6293e227ea added txo_sum command 2020-03-20 19:08:47 -04:00
Lex Berezhny d6d83a5c76 integration test fix 2020-03-20 19:08:47 -04:00
Lex Berezhny 93fc883b90 fixing unit tests 2020-03-20 19:08:47 -04:00
Lex Berezhny dd21803598 working --is_my_input_or_output --is_my_input --is_my_output etc 2020-03-20 19:08:47 -04:00
Lex Berezhny af2f2282c2 txo_list returns txo funded by my account but sent to external address 2020-03-20 19:08:47 -04:00
Jack Robison 869a76c9bb Merge pull request #2851 from lbryio/transaction-info
Add `blockchain.transaction.info` and `blockchain.transaction.get_batch` to the wallet server for more efficient transaction fetching
2020-03-20 10:25:45 -04:00
Jack Robison 44b2964a6a add blockchain.transaction.get_batch 2020-03-19 19:20:30 -04:00
Jack Robison 8fddb57e0a reduce block lru cache sizes 2020-03-19 19:20:30 -04:00
Jack Robison c7a5a0cab0 add blockchain.transaction.info 2020-03-19 19:20:30 -04:00
Jack Robison c13aab3ffc Merge pull request #2823 from lbryio/multiple-db-readers
Use multiple processes for querying the db for api calls
2020-03-19 19:19:53 -04:00
Jack Robison 95a9a76598 windows 2020-03-19 19:19:02 -04:00
Jack Robison 923cfa3d50 another 2020-03-19 19:19:02 -04:00
Jack Robison 2c489168c2 feedback 2020-03-19 19:19:02 -04:00
Jack Robison f3292b4d34 feedback 2020-03-19 19:19:02 -04:00
Jack Robison 9769829b72 fix android executor 2020-03-19 19:19:02 -04:00
Jack Robison 36243d15cc fix peer_list 2020-03-19 19:19:02 -04:00
Jack Robison f4645f570c more read only calls 2020-03-19 19:19:02 -04:00
Jack Robison 61603ccfce write lock 2020-03-19 19:19:02 -04:00
Jack Robison a26cfc639c WAL 2020-03-19 19:19:02 -04:00
Jack Robison 90602931d8 multiple readers for transaction_list 2020-03-19 19:19:02 -04:00
Jack Robison d1b330028c multiple db reader processeses 2020-03-19 19:19:02 -04:00
Jack Robison 7a6b1930bf rename executor -> writer_executor 2020-03-19 19:19:02 -04:00
Lex Berezhny c271361552 Merge pull request #2865 from lbryio/lbrycrd-1744
upgrade lbrycrd to v0.17.4.4
2020-03-19 17:54:26 -04:00
Lex Berezhny 17789bc814 Merge pull request #2866 from lbryio/dont_shlex_windows
fixed issue with shell parsing on Windows
2020-03-19 17:53:47 -04:00
Brannon King d7c16e161a fixed issue with shell parsing on Windows 2020-03-19 13:11:57 -06:00
Lex Berezhny e71c17d7e7 upgrade lbrycrd to v0.17.4.4 2020-03-19 12:54:19 -04:00
Lex Berezhny 17d87eb157 Merge pull request #2861 from lbryio/ffmpeg_fixes
transcode in stream_update, re-enable transcode on Windows
2020-03-19 12:20:47 -04:00
Brannon King bf11bcc084 fixed transcoding tests for non-async execute
status ordering broke path check
2020-03-18 12:43:27 -06:00
Brannon King bd291109df addressing code review comments 2020-03-18 12:00:52 -06:00
Brannon King ac89ba9b8d don't require ProactorEventLoop on Windows
fix linter errors
2020-03-18 12:00:52 -06:00
Brannon King 5ab634e375 support search path for ffmpeg 2020-03-18 12:00:52 -06:00
Brannon King bb1978d976 ffmpeg now invoked via stream update 2020-03-18 12:00:52 -06:00
Lex Berezhny bea94ce8ac Merge pull request #2854 from lbryio/tracemalloc_api
Adds tracemalloc api for memory troubleshooting
2020-03-16 22:15:41 -04:00
Victor Shyba 9561f93594 disable tracemalloc after test finishes so it doesnt slow down everything else 2020-03-16 11:29:16 -03:00
Victor Shyba d44d5c3304 enable/disable instead of set 2020-03-16 06:40:22 -03:00
Victor Shyba ec541e2057 lint and improve tests 2020-03-11 19:56:37 -03:00
Victor Shyba 8169bf6b97 top objects api 2020-03-11 19:56:37 -03:00
Victor Shyba 56c8ad1221 start/stop tracemalloc over api 2020-03-11 19:56:37 -03:00
Lex Berezhny 6814f2e38c Merge pull request #2857 from eggplantbren/master
Trending algorithm with time delay and variable decay rate
2020-03-11 18:34:52 -04:00
Brendon J. Brewer 5f043b9a78 variable decay 2020-03-12 10:33:15 +13:00
Lex Berezhny c9092cd1c7 v0.64.0 2020-03-09 13:32:19 -04:00
Lex Berezhny 86cc65d894 Merge pull request #2848 from lbryio/claim_transaction_list_filtering
added new `txo_list` command with filtering for `--claim_id`, claim `--name` and `--is_received`/`--is_not_received`, also commands `claim_list`/`stream_list`/`channel_list`/`support_list` are based on `txo_list` and thus support most of the new filters
2020-03-08 23:33:38 -04:00
Lex Berezhny 8ef2647fa9 is_received 2020-03-08 23:11:03 -04:00
Lex Berezhny 3ff9e99416 added txo_list command 2020-03-08 16:22:49 -04:00
Lex Berezhny 1731046011 Merge pull request #2846 from lbryio/txo_is_spent
added `is_spent` attribute to transaction outputs
2020-03-06 21:10:30 -05:00
Lex Berezhny c3d96184b6 fix test 2020-03-06 20:35:26 -05:00
Lex Berezhny c255c606a7 added is_spent attribute to transaction outputs 2020-03-06 20:12:38 -05:00
Jack Robison cc69faa1fd v0.63.0 2020-03-06 15:58:34 -05:00
Jack Robison 0f53cd86c8 Merge branch 'debug_find_ffmpeg' 2020-03-06 15:54:24 -05:00
Brannon King 212c8f188d only change LD vars if running from bundle
missed a word


moved is_running_from_bundle
2020-03-06 15:53:09 -05:00
Brannon King dec248adec repair env modified by pyinstaller
see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
2020-03-06 15:53:09 -05:00
Brannon King 98e6a066f4 add unit test and debug output 2020-03-06 15:53:09 -05:00
Jack Robison c90db54a3d Revert "v0.63.0"
This reverts commit 927d1569
2020-03-06 15:51:42 -05:00
Jack Robison 927d156933 v0.63.0 2020-03-06 14:49:53 -05:00
Jack Robison c71f91073f Merge pull request #2845 from lbryio/temp-disable-windows-ffmpeg
fix DHT setup and disable ffmpeg on windows temporarily
2020-03-06 14:20:18 -05:00
Jack Robison ab28387692 remove ProactorEventLoop and disable ffmpeg on windows temporarily 2020-03-06 13:45:34 -05:00
Lex Berezhny c4905d02b9 Merge pull request #2793 from lbryio/dont_validate_non_video
don't validate and repair files that aren't videos; add maximum video bit rate setting
2020-03-04 09:16:37 -05:00
Brannon King ee39880fb5 fix items from review 2020-03-03 21:38:30 -07:00
Brannon King 926b3e56b9 take a hint, lint 2020-03-03 21:38:30 -07:00
Brannon King e060df5367 hide ValueError 2020-03-03 21:38:30 -07:00
Brannon King a3294d4a0d make bit_rate check support maximum 2020-03-03 21:38:30 -07:00
Brannon King 19ce0ab246 ogg -> ogv, ignore files that aren't video 2020-03-03 21:38:30 -07:00
Lex Berezhny 75a1cc0d33 Merge pull request #2840 from lbryio/status_returns_server_features
`status` command returns features available of currently connected wallet server, including trending algorithm
2020-03-03 20:47:04 -05:00
Lex Berezhny 05228529b0 fix tests 2020-03-03 20:26:07 -05:00
Lex Berezhny 3cbeadfbc3 lbrynet status command returns features available of currenty connected wallet server, including thetrending algorithm used 2020-03-03 20:04:57 -05:00
Lex Berezhny 3d4938c0e2 Merge pull request #2839 from lbryio/file_analysis_join_path
use `os.path.join()` instead of string addition when searching for file analysis binaries
2020-03-03 10:48:06 -05:00
Lex Berezhny 1b850b8a2b dont prepend path when doing which ffmpeg 2020-03-02 21:11:11 -05:00
Lex Berezhny ebd33f1869 use os.path.join instead of string addition when searching for file analysis binaries 2020-03-02 20:34:54 -05:00
Jack Robison ac846b4df3 v0.62.0 2020-03-01 20:29:32 -05:00
Jack Robison 0d427c9b90 Merge pull request #2836 from lbryio/wallet-server-lru-caches
Add LRU caches for blocks and block hashes in the wallet server
2020-03-01 15:19:28 -05:00
Jack Robison dc6194f862 perf counter 2020-03-01 15:17:21 -05:00
Jack Robison de50214e1f block lru cache 2020-03-01 15:17:21 -05:00
Jack Robison 5ec4a88c35 block hash lru cache 2020-03-01 15:17:05 -05:00
Jack Robison 31c141e757 Merge pull request #2835 from lbryio/wallet-server-threadpools
Improve wallet server performance with separate thread pools for leveldb, the block processor, and mempool
2020-03-01 15:15:20 -05:00
Jack Robison 9d5760d899 fix test 2020-03-01 14:41:14 -05:00
Jack Robison 7945e1ea3c mempool threadpool 2020-03-01 12:40:13 -05:00
Jack Robison d94c40e371 leveldb threadpool 2020-03-01 12:40:13 -05:00
Jack Robison 31f22122e8 block processor threadpool 2020-03-01 12:40:13 -05:00
Lex Berezhny 506582aa2b Merge pull request #2834 from lbryio/list_resolve_includes_local_data
when using `--resolve` with local claim `*_list` commands, update resolved result with local metadata (such as `is_mine`)
2020-02-29 15:48:49 -05:00
Lex Berezhny a02b251c9b when using --resolve with local claim list commands, update resolved result with local metadata (such as is_mine and is_mine) 2020-02-29 09:42:11 -05:00
Lex Berezhny 34cbb6fa79 Merge pull request #2831 from lbryio/claim_type_index_improvement
update `claim_type_*` db indexes to be more performant
2020-02-27 14:27:24 -05:00
Lex Berezhny 33679b56fd update claim_type_* indexes to be more peformant 2020-02-27 12:07:38 -05:00
Lex Berezhny 8897c4d560 Merge pull request #2816 from osilkin98/removes-anonymous
removes ability to comment anonymously
2020-02-27 10:37:47 -05:00
Oleg Silkin f0c18ccbe7 Removes testing for anonymous comments, fixes up existing asserts 2020-02-26 19:59:38 -05:00
Oleg Silkin e560d83c51 drops support for commenting without a channel 2020-02-26 19:58:03 -05:00
Lex Berezhny eca7addc67 v0.61.0 2020-02-24 10:01:03 -05:00
Lex Berezhny 263f3ba5c9 Merge pull request #2828 from lbryio/update_lbrycrd
update lbrycrd in tests to v0.17.4.3
2020-02-22 14:47:48 -05:00
Lex Berezhny f514123ef0 update lbrycrd 2020-02-22 13:42:28 -05:00
Lex Berezhny 9f461db0d0 Merge pull request #2827 from lbryio/repost_inherit_tags
reposts inherit the tags of the original claim, filtered/blocked claims cause their reposts to also be filtered/blocked
2020-02-22 13:04:31 -05:00
Lex Berezhny 7de8670616 blocked original repost causes all reposts to also be blocked 2020-02-22 12:49:20 -05:00
Lex Berezhny 8c7908b200 reposts inherit the tags of the original claim 2020-02-22 12:23:11 -05:00
Lex Berezhny e7722e039f Merge pull request #2826 from lbryio/fix_not_channel_ids
`claim_search --not_channel_ids` excludes the channel itself and not just claims in the channel
2020-02-21 23:01:00 -05:00
Lex Berezhny 039bc0208a fix tests 2020-02-21 22:22:28 -05:00
Lex Berezhny d57900a069 claim_search --not_channel_ids excludes the channel itself and not just claims in the channel 2020-02-21 21:59:46 -05:00
Jack Robison c1153302aa v0.60.1 2020-02-21 16:13:30 -05:00
Victor Shyba 1086a3297f check its broadcasted and use a retriable call 2020-02-21 17:51:46 -03:00
Jack Robison 5c613934ca Revert "v0.60.1"
This reverts commit 7f03b13579.
2020-02-21 15:05:33 -05:00
Jack Robison 6d7e9092f8 Merge pull request #2825 from lbryio/tx_from_stream_controller
use the stream controller tx instead of local history
2020-02-21 15:04:57 -05:00
Victor Shyba 8bb6f328dc use the stream controller tx instead of local history 2020-02-21 16:41:23 -03:00
Jack Robison 7f03b13579 v0.60.1 2020-02-21 14:14:15 -05:00
Jack Robison 09008ea991 Merge pull request #2824 from lbryio/skip_wallet_server_payment
max wallet server payment fee defaults to 0 and component doesnt start when its not a positive value
2020-02-21 13:27:58 -05:00
Jack Robison ee234212e6 Merge branch 'master' into skip_wallet_server_payment 2020-02-21 13:26:38 -05:00
Jack Robison 277b243f52 Merge pull request #2772 from lbryio/reflector-status
Add `is_fully_reflected` to `file_list` response
2020-02-21 13:21:36 -05:00
Jack Robison 0973ac753f add is_fully_reflected to file_list response 2020-02-21 12:49:19 -05:00
Victor Shyba f41cfbfb97 max wallet server payment fee defaults to 0 and component doesnt start in that case 2020-02-21 14:47:09 -03:00
Lex Berezhny a70980c81a reverting v0.60.1 release 2020-02-21 12:43:43 -05:00
Lex Berezhny 12a962f656 Merge pull request #2822 from lbryio/stream_controller_payment
use stream controller and documented errors for wallet server payments
2020-02-21 12:34:07 -05:00
Lex Berezhny 5936444f3e more reliable wait for wallet server payment test 2020-02-21 12:18:44 -05:00
Lex Berezhny c7d42f00c6 shorten error message for ServerPaymentFeeAboveMaxAllowed 2020-02-21 10:16:56 -05:00
Lex Berezhny b1318a9958 minor cleanup 2020-02-21 10:05:46 -05:00
Victor Shyba d2560d260c use stream controller and documented errors 2020-02-21 00:04:37 -03:00
Jack Robison f40a61cf9a v0.60.1 2020-02-20 17:56:54 -05:00
Jack Robison e0623578bf Merge pull request #2821 from lbryio/ffmpeg-find
add `ffmpeg_find` api to check ffmpeg installation status, don't recheck for ffmpeg in `status`
2020-02-20 17:48:36 -05:00
Jack Robison ead8daaa14 doc 2020-02-20 17:27:18 -05:00
Jack Robison 23b4b9e230 add ffmpeg_find api, don't recheck for it in status 2020-02-20 16:43:41 -05:00
Jack Robison 947017e334 Merge pull request #2820 from lbryio/disconnect-on-oversized
disconnect from client sending message larger than MAX_RECEIVE
2020-02-20 16:24:26 -05:00
Jack Robison 7fd0d6507f disconnect from client sending message larger than MAX_RECEIVE 2020-02-20 16:08:21 -05:00
Jack Robison ad1e9ef086 Merge pull request #2819 from lbryio/max-receive
add MAX_RECEIVE setting to wallet server
2020-02-20 14:38:11 -05:00
Jack Robison 2f1d08e417 add MAX_RECEIVE wallet server setting 2020-02-20 14:11:16 -05:00
Lex Berezhny 55f4eb80ba Merge pull request #2815 from lbryio/apply_share_usage_data_live
apply share_usage_data as its set, without restarting
2020-02-20 13:35:47 -05:00
Victor Shyba 7c160ff65e remove unused parameters 2020-02-20 14:38:13 -03:00
Victor Shyba fc5d5faaed use conf directly isntead of lambda 2020-02-20 14:27:39 -03:00
Victor Shyba 5394f1763c simplify external ip logic 2020-02-20 12:12:40 -03:00
Victor Shyba 050b67c9d6 apply share_usage_data as its set 2020-02-20 12:12:40 -03:00
Lex Berezhny 4b87cb45ee Merge pull request #2707 from lbryio/pay_wallet_servers
wallet servers requiring a daily fee will automatically get paid by client
2020-02-18 17:40:23 -05:00
Victor Shyba f0e1db319c make wallet server payments a component 2020-02-18 19:18:30 -03:00
Victor Shyba 3950715237 add max_wallet_server_fee conf to limit daily wallet server payments 2020-02-18 18:10:58 -03:00
Victor Shyba ae9ba14b59 use both donation and payment addresses separately 2020-02-18 18:10:58 -03:00
Victor Shyba ad6c6fbe35 fixes from review and add analytics 2020-02-18 18:10:58 -03:00
Victor Shyba ffa5c20c88 fix hanging test 2020-02-18 18:10:58 -03:00
Victor Shyba 5a6218eeca improve tests, fix types 2020-02-18 18:10:58 -03:00
Victor Shyba 200531dd96 wait for tx instead 2020-02-18 18:10:58 -03:00
Victor Shyba 15a2f048ac pay wallet server test and initial prototype 2020-02-18 18:10:58 -03:00
Victor Shyba d317a4042c change daily_fee to be lbc instead of dewies 2020-02-18 18:10:58 -03:00
Victor Shyba c2d717aba5 old code from #2683 2020-02-18 18:10:58 -03:00
Lex Berezhny ef74777df1 v0.60.0 2020-02-17 17:52:11 -05:00
Lex Berezhny f6dbf99fb5 Merge pull request #2811 from lbryio/index_fixes
wallet server indexes
2020-02-17 17:43:55 -05:00
Lex Berezhny 0b2d9e15b4 added order bys for fee_amount 2020-02-17 17:24:48 -05:00
Lex Berezhny d14e5e75e8 all three order bys for duration filter 2020-02-17 17:24:48 -05:00
Lex Berezhny df11ef34d2 index cleanup 2020-02-17 17:24:48 -05:00
Lex Berezhny 4fc619262c replacing indexes on trending_global, trending_mixed to be trending_group, trending_mixed 2020-02-17 17:24:48 -05:00
Lex Berezhny 2750eee86d Merge pull request #2810 from lbryio/no_default_order_by
removed default `order_by` for `claim_search`
2020-02-17 17:04:41 -05:00
Lex Berezhny 71cb03345b removed default order_by for claim_search 2020-02-17 16:26:29 -05:00
Lex Berezhny 22c8ca6c40 Merge pull request #2806 from lbryio/privacyDefault
change default privacy setting
2020-02-17 16:21:34 -05:00
Victor Shyba 237a8965eb fix tests 2020-02-17 18:00:23 -03:00
jessop 7d872c7863 change default privacy setting 2020-02-13 18:56:11 -05:00
Jack Robison 0d08858dfb v0.59.2 2020-02-13 17:22:21 -05:00
Jack Robison 5c6d63c93d Merge pull request #2803 from lbryio/fix-2802
fix claim apis failing with --resolve if no claims are present
2020-02-13 15:15:56 -05:00
Jack Robison c8c10d2bb0 fix claim apis failing with --resolve if no claims are present
fixes https://github.com/lbryio/lbry-sdk/issues/2802
2020-02-13 14:58:51 -05:00
Jack Robison f4d3e9ea6a add development wallet server deploy script 2020-02-13 13:54:46 -05:00
Jack Robison 3713d3488d Merge pull request #2800 from lbryio/fix-2746
Fix race condition setting/clearing the claim_search and resolve cache
2020-02-13 11:39:27 -05:00
Jack Robison 9b46d03c91 fix race between setting items in and clearing the resolve/claim_search cache 2020-02-13 11:17:21 -05:00
Jack Robison fe4e738580 Merge pull request #2798 from lbryio/repost-index
Add sqlite index for reposted claims
2020-02-13 10:33:36 -05:00
Jack Robison 205b0c4263 call get_platform only once 2020-02-13 10:04:50 -05:00
Jack Robison 78eef25f1a add reposted_claim_hash index
fixes https://github.com/lbryio/lbry-sdk/issues/2782
2020-02-12 17:39:57 -05:00
Lex Berezhny c9305fd070 Merge pull request #2796 from lbryio/xenial_build
linux build of LBRY SDK for Ubuntu 16.04 Xenial
2020-02-12 14:28:41 -05:00
Lex Berezhny 20774280b9 Merge pull request #2790 from lbryio/sql_in_for_single_value
SQL generation fix to handle IN operation for one value lists
2020-02-12 14:27:36 -05:00
Lex Berezhny 74bd2557af linux build of LBRY SDK for Ubuntu 16.05 Xenial 2020-02-12 14:20:06 -05:00
Lex Berezhny 605fc8ecd8 claim_search by --claim_type=repost support 2020-02-12 11:51:35 -05:00
Lex Berezhny 6b745c53dc make sure SQL IN works with python set 2020-02-12 11:41:32 -05:00
Lex Berezhny 98c60b38fc disable too-many-nested-blocks lint rule 2020-02-12 10:47:40 -05:00
Lex Berezhny be4515ec9a use sets for __in constraint values 2020-02-12 10:39:25 -05:00
Lex Berezhny dcb1b64696 SQL generation fix to handle IN operation for one value lists 2020-02-12 10:31:27 -05:00
Jack Robison 1190a60c7a v0.59.1 2020-02-11 16:56:37 -05:00
Jack Robison 82ecf207c6 Merge pull request #2789 from lbryio/update-default-servers
update default wallet servers
2020-02-11 16:36:31 -05:00
Jack Robison 84c7f1bf58 update test 2020-02-11 16:13:55 -05:00
Jack Robison 5ac453fe07 update default wallet servers 2020-02-11 13:51:15 -05:00
Lex Berezhny c2f39d653d v0.59.0 2020-02-10 12:08:24 -05:00
Lex Berezhny 748178f536 Merge pull request #2780 from lbryio/release_current
manually incremented version and added support to release script to r…
2020-02-09 20:41:00 -05:00
Lex Berezhny ffbef69dc7 manually incremented version and added support to release script to release current version 2020-02-09 20:26:15 -05:00
Lex Berezhny 3757124323 Merge pull request #2771 from lbryio/update_url_regex
update url regex to include missing cases
2020-02-09 18:17:35 -05:00
Victor Shyba e65f214b3c fix test using space in name 2020-02-09 17:55:29 -05:00
Victor Shyba f14004e56b update url regex to cases tom reported 2020-02-09 17:55:29 -05:00
Lex Berezhny dadc004dd6 Merge pull request #2765 from eggplantbren/master
fixed some issues with AR trending
2020-02-09 17:53:27 -05:00
Brendon J. Brewer 3697d9e1f0 Tweaked some parameters, back to 134 2020-02-09 17:38:34 -05:00
Brendon J. Brewer 94524f7330 lint 2020-02-09 17:38:34 -05:00
Brendon J. Brewer 17d8a3e5d6 Use claim_hash not claim_id 2020-02-09 17:38:24 -05:00
Brendon J. Brewer 39996d7612 Only create file during install if logging is turned on 2020-02-09 17:37:55 -05:00
Lex Berezhny 601ee4320f Merge pull request #2768 from mkroman/master
update cffi to v1.13.2
2020-02-09 17:35:06 -05:00
Mikkel Kroman 6e4f6fa92d Update cffi to v1.13.2
This fixes compilation issues for CPython 3.8+
2020-02-09 17:19:53 -05:00
Lex Berezhny b7103c29dd Merge pull request #2773 from lbryio/fix_video_analysis_error
fix video analysis error
2020-02-09 17:19:23 -05:00
Brannon King e53209e5ae handle early lbrycrd exit in tests 2020-02-09 17:02:16 -05:00
Brannon King 501fbd3114 handle strange file metadata 2020-02-09 17:02:16 -05:00
Lex Berezhny 5626f43e2b Merge pull request #2779 from lbryio/github_actions
build community PRs using GitHub Actions
2020-02-09 16:49:40 -05:00
Lex Berezhny 0185cb48fb run lint and tests in parallel 2020-02-09 16:34:04 -05:00
Lex Berezhny 9a49eb06da fix import path in test_transcoding.py 2020-02-09 16:15:20 -05:00
Lex Berezhny 3fbc012231 moved blockchain/test_transcoding.py -> other/test_transcoding.py 2020-02-09 16:15:20 -05:00
Lex Berezhny 3938cf33d8 take github actions for a spin 2020-02-09 16:15:04 -05:00
Lex Berezhny 38b3d0c87c Merge pull request #2777 from lbryio/claim_list_multi_type
`claim_list` argument `--claim_type` can now be repeated to pass a list of claim_types
2020-02-08 23:34:34 -05:00
Lex Berezhny 15a2fa6199 claim_list --claim_type argument can be repeated 2020-02-08 23:34:04 -05:00
Lex Berezhny e834209d40 Merge pull request #2776 from lbryio/resolve_includes_censoring_channels_url
`resolve` results which are censored now include full details of channel which did the censoring
2020-02-08 23:32:44 -05:00
Lex Berezhny 9607d21828 censored searches/resolves include metadata of channel which did the censoring 2020-02-07 18:50:29 -05:00
Lex Berezhny 6fbbf36143 pylint 2020-02-07 18:44:30 -05:00
Lex Berezhny db25f3282e lbry.error: save arguments on self and support for functions around arguments 2020-02-07 18:25:06 -05:00
Jack Robison ceb92838e7 v0.58.2 2020-02-06 11:37:09 -05:00
Jack Robison afb98f2cfc Merge pull request #2770 from lbryio/disable-save-content-claim
Add `save_resolved_claims` config setting for better performance when not using file_x commands
2020-02-06 11:32:49 -05:00
Jack Robison ed785fb087 add save_resolved_claims config setting to disable saving claims upon resolving them 2020-02-06 11:27:26 -05:00
Jack Robison 3921180d77 v0.58.1 2020-02-04 15:12:55 -05:00
Jack Robison ef36e0311e Merge pull request #2764 from lbryio/version-analytics
Add version tracking to prometheus metrics
2020-02-04 12:44:30 -05:00
Jack Robison bf4cbe1204 add version to session count metric 2020-02-04 12:18:32 -05:00
Jack Robison 28eee4da25 add version metrics to prometheus 2020-02-04 10:52:08 -05:00
Jack Robison 933ccf6deb add block updates to prometheus 2020-02-04 10:26:22 -05:00
Jack Robison 9ac41322e5 add cpu count to prometheus 2020-02-04 10:25:40 -05:00
Jack Robison c3233e03ef Merge pull request #2762 from lbryio/fix-2368
cancel reflector uploads upon file delete
2020-02-03 23:51:12 -05:00
Jack Robison 111871bb28 update test 2020-02-03 23:00:45 -05:00
Jack Robison bf5b5f43e3 non blocking analytics 2020-02-03 23:00:45 -05:00
Jack Robison 2ed8ebff09 handle ConnectionError and ValueError in blob sendfile 2020-02-03 23:00:45 -05:00
Jack Robison 34eb856d09 cancel reflector uploads upon file delete
-remove unnecessary db call in stream_update
2020-02-03 23:00:45 -05:00
Lex Berezhny 2d644bdfb0 v0.58.0 2020-02-03 22:33:03 -05:00
Lex Berezhny 741e5cf103 regenerated api docs 2020-02-03 22:30:39 -05:00
Lex Berezhny d745c04fe6 Merge pull request #2726 from lbryio/add_video_transcoding
added video file validation and optimization when publishing streams (using `--validate_file` and `--optimize_file` arguments)
2020-02-03 22:29:15 -05:00
Lex Berezhny 3a3c63956a removed unnecessary try/finally 2020-02-03 22:06:15 -05:00
Lex Berezhny 153bdf576a removed wheel from setup.py 2020-02-03 22:05:47 -05:00
Lex Berezhny 6525ee6510 moved __init__ to top of class 2020-02-03 22:05:23 -05:00
Brannon King 1780ddd329 added ffmpeg status, addressed items from code review
linter
2020-02-03 21:53:42 -05:00
Brannon King 85ad972ca8 return string instead of path object for compatibility with hachoir 2020-02-03 21:53:42 -05:00
Brannon King a90b60799a Fixed check_video.py on Windows
using a cross-platform workaround


fixed proactor use in the SDK


fixed linter
2020-02-03 21:53:42 -05:00
Brannon King 47e8f74da9 changed to list append, relative claim test 2020-02-03 21:53:42 -05:00
Brannon King fac28072ab added unit tests, other minor fixes
added universe


try again


try 4
2020-02-03 21:53:42 -05:00
Brannon King ab77541f36 in progress on video transcoding
works


Fixing lint tests


remove eval
2020-02-03 21:53:42 -05:00
Lex Berezhny 9eb769f340 Merge pull request #2760 from lbryio/channel_hash_wallet_server_idx
added database index for several `channel_hash` based queries
2020-02-03 20:56:48 -05:00
Lex Berezhny 1e81d6f848 added database index for various channel_hash based queries 2020-02-03 20:28:39 -05:00
Jack Robison dc393f4b77 Merge pull request #2761 from lbryio/catch-notification-timeout
catch TimeoutError when attempting to send notifications
2020-02-03 19:03:07 -05:00
Jack Robison dd1de530c3 catch TimeoutError when attempting to send notifications 2020-02-03 18:46:17 -05:00
Lex Berezhny 74cd887249 v0.57.0 2020-02-03 13:41:33 -05:00
Lex Berezhny b40bb35652 change MINIMUM_REQUIRED on client for servers it connects to 2020-02-03 13:41:05 -05:00
Lex Berezhny dfe257af86 Revert "test to make sure next release tests pass"
This reverts commit ebb2a09107.
2020-02-03 13:39:36 -05:00
Lex Berezhny ebb2a09107 test to make sure next release tests pass 2020-02-03 13:22:33 -05:00
Lex Berezhny c157909b4e Merge pull request #2758 from ykris45/patch-6
update copyright year in license
2020-02-02 20:19:33 -05:00
YULIUS KURNIAWAN KRISTIANTO a744239b22 update year 2020-02-03 04:04:51 +07:00
Lex Berezhny 9b4417be87 Merge pull request #2757 from lbryio/resolve_for_local_list
added `--resolve` to local `claim_list`/`stream_list`/`channel_list` commands which returns more metadata for local claims
2020-02-01 18:15:35 -05:00
Lex Berezhny 84f807e278 regenerate docs 2020-02-01 17:59:57 -05:00
Lex Berezhny 448635a945 added --resolve to local *_list commands 2020-02-01 17:59:10 -05:00
Lex Berezhny 638e3e6b3d Merge pull request #2756 from lbryio/blocked_resolve
resolve errors make distinction between truely not found claims and claims which were censored by wallet server
2020-02-01 14:00:37 -05:00
Lex Berezhny bfe711bd42 fixing tests 2020-02-01 13:29:55 -05:00
Lex Berezhny d5eed91e7f fix sql unit tests 2020-02-01 13:03:39 -05:00
Lex Berezhny 15abf49211 censored resolve responses return appropriate error 2020-02-01 12:53:39 -05:00
Lex Berezhny b7eec0586c added ResolveCensoredError 2020-02-01 12:49:01 -05:00
Lex Berezhny e639124a69 Merge pull request #2755 from eggplantbren/master
fixed initialization problem with ar trending algorithm
2020-01-30 13:44:25 -05:00
Brendon J. Brewer 22bca7a16e Length zero could also just mean there aren't any claims 2020-01-31 07:34:43 +13:00
Lex Berezhny 6f66b354e5 Merge pull request #2754 from lbryio/feat-claim_search_duration
add duration to claim search params
2020-01-29 16:43:34 -05:00
jessop f98e3320ae make review changes 2020-01-29 16:25:04 -05:00
jessop 73c958222b add duration to claim search params 2020-01-29 15:46:47 -05:00
Lex Berezhny e43230e46a v0.56.0 2020-01-28 19:58:28 -05:00
Lex Berezhny dd07bc29eb Merge pull request #2747 from eggplantbren/master
new trending algorithm contributed by Brendon Brewer, to enable on wallet server set `TRENDING_ALGORITHMS` environment variable on wallet server to `ar`
2020-01-28 19:36:52 -05:00
Lex Berezhny 790bbfcc99 integrate ar trending algorithm into configurable trending system 2020-01-28 18:11:32 -05:00
Brendon J. Brewer 994e70d43a ar trending algorithm 2020-01-28 18:11:32 -05:00
Jack Robison 7d5e19716a Merge pull request #2749 from lbryio/prometheus-interrupts
add metrics to wallet server prometheus
2020-01-28 16:31:10 -05:00
Jack Robison 11530c675d initialize_request_handlers, remove local rpc handlers 2020-01-28 15:54:22 -05:00
Jack Robison 1c474352fe add wallet server prometheus metrics
-sessions gauge
-requests counter
-response times histogram
-notification counter
-request errors counter
-interrupt counter
-operational error counter
-internal error counter
-reader executor times histogram
-pending queries gauge
-lbrycrd request counter
-client versions counter
2020-01-28 15:54:22 -05:00
Jack Robison 3b2d635390 Revert "Revert "configurable trending algorithms""
This reverts commit 55e08f96a6.
2020-01-28 15:53:57 -05:00
Jack Robison 055c532d31 Merge pull request #2752 from lbryio/revert-2750-configurable_trending
Revert "configurable trending algorithms"
2020-01-28 15:04:29 -05:00
Jack Robison 55e08f96a6 Revert "configurable trending algorithms" 2020-01-28 15:03:09 -05:00
Lex Berezhny 3e0a9180bc Merge pull request #2750 from lbryio/configurable_trending
configurable trending algorithms
2020-01-28 12:30:26 -05:00
Lex Berezhny 01ed71b243 configurable trending 2020-01-28 11:58:27 -05:00
Lex Berezhny 595c058517 moved trending.py to trending/zscore.py 2020-01-28 11:58:27 -05:00
Alex Grintsvayg e04b0a6995 add docker tag to prometheus version info 2020-01-27 14:16:18 -05:00
Jack Robison 8961848ab1 Merge pull request #2748 from lbryio/cached-wallet-id
faster wallet.id attribute, improves speed of selecting wallets during api calls
2020-01-27 13:58:23 -05:00
Jack Robison 6fb1c72b7d cached wallet id 2020-01-27 13:57:21 -05:00
Alex Grintsvayg 400d79d6ab prom port off by default 2020-01-27 13:55:44 -05:00
Lex Berezhny ce6018f387 v0.55.0 2020-01-27 12:20:41 -05:00
Alex Grintsvayg 65d9dca917 Merge branch 'docker'
* docker:
  fix gitlab ci
  add min_version and method name to metrics
  include docker tag in build info
  build_type.py -> build_info.py
  add docker build info to wallet server image
2020-01-23 13:13:33 -05:00
Alex Grintsvayg 2232b08351 fix gitlab ci 2020-01-23 13:13:20 -05:00
Alex Grintsvayg 0732ab6ad5 add min_version and method name to metrics 2020-01-23 13:13:20 -05:00
Alex Grintsvayg fb568768c5 include docker tag in build info 2020-01-23 13:13:19 -05:00
Alex Grintsvayg 7fd56e0add build_type.py -> build_info.py 2020-01-23 13:13:19 -05:00
Alex Grintsvayg 38a3f8cf4c add docker build info to wallet server image 2020-01-23 13:13:19 -05:00
Lex Berezhny 8d8a5b36b6 Merge pull request #2737 from lbryio/fix_repost_resolve_bug
fix bug where repost claims were not being fully resolved
2020-01-22 23:32:55 -05:00
Lex Berezhny a70cc7beb1 fix bug where repost claims were not being fully resolved 2020-01-22 22:59:05 -05:00
Alex Grintsvayg c5421699b1 add prometheus port to docker-compose.yml 2020-01-22 20:08:29 -05:00
Alex Grintsvayg d9f809864f include version and build info in prometheus output 2020-01-22 19:13:58 -05:00
Lex Berezhny bb23f509d7 rename BLOCKING_CHANNELS_IDS/FILTERING_CHANNELS_IDS to remove double S 2020-01-22 11:02:01 -05:00
Alex Grin 254e184677 Merge pull request #2734 from lbryio/prom2
Add prometheus metrics collection to client and server (take two)
2020-01-22 10:28:54 -05:00
Alex Grintsvayg 6c07141abd Add prometheus metrics collection to client and server 2020-01-22 10:28:17 -05:00
Lex Berezhny 5822fc1d5b v0.54.1 2020-01-22 08:40:35 -05:00
Lex Berezhny 2cd7ea257c Added support to differentiate between filtering and blocking for content censoring 2020-01-22 08:39:14 -05:00
Alex Grin 0a21b72f9c v0.54.0 2020-01-21 16:17:12 -05:00
Alex Grintsvayg 80280f6f4a Revert "v0.54.0"
This reverts commit bd3b4906d1.
2020-01-21 15:34:35 -05:00
Alex Grintsvayg 810a8e76d1 Pin pip to 19.3.1 on Windows
pip 20 made the following change:

> Remove interpreter-specific major version tag e.g. cp3-none-any from consideration.
> https://github.com/pypa/pip/pull/7355

The coincurve pip package only has a major version tag for its windows
wheel. pip20 will not allow us to install that wheel, and compiling
coincurve from source causes errors. So we cannot use pip 20 or above
until coincurve fixes its compatibility tags, we can compile coincurve
from source, we stop using coincurve, or we figure out another way to
install it.

Note that on osx and linux, coincurve has the correct tags.
2020-01-21 15:24:29 -05:00
Lex Berezhny bd3b4906d1 v0.54.0 2020-01-21 11:30:14 -05:00
Lex Berezhny d9c740f014 update tests to use 0.54.0 minimum version 2020-01-21 11:28:56 -05:00
Alex Grintsvayg ec63a18960 Revert "v0.54.0"
This reverts commit fab0618b6b.
2020-01-21 11:23:51 -05:00
Alex Grin fab0618b6b v0.54.0 2020-01-21 11:00:38 -05:00
Alex Grintsvayg 13e57e7aa8 Revert "v0.54.0"
This reverts commit 836b66e110.
2020-01-21 10:43:21 -05:00
Alex Grintsvayg 1299c9162c Revert "Add prometheus metrics collection to client and server"
This reverts commit 59a5bacb2e.
2020-01-21 10:30:49 -05:00
Alex Grintsvayg a64f33dbcc ignore coverage files 2020-01-21 10:30:48 -05:00
Alex Grin 836b66e110 v0.54.0 2020-01-20 14:26:36 -05:00
Alex Grintsvayg 802119d789 Revert "v0.54.0"
This reverts commit aec4d1dc60.
2020-01-20 14:24:30 -05:00
Alex Grintsvayg f94135cadd make pycharm happy about long lines 2020-01-20 14:18:00 -05:00
Alex Grintsvayg 474c0f980e skip randomly-failing dht test 2020-01-20 14:16:52 -05:00
Alex Grintsvayg 2cf0f791f2 remove unused import 2020-01-20 13:44:49 -05:00
Alex Grin aec4d1dc60 v0.54.0 2020-01-20 13:13:30 -05:00
Alex Grintsvayg 8d28bb5a66 bump min protocol version 2020-01-20 12:53:10 -05:00
Alex Grintsvayg 1d5e553f9c Merge branch 'blocked_search_metadata'
* blocked_search_metadata:
  add index for claim_type and release_time
  fix test
  fix json api generator
  add pagination for claim_search
  regenerate protobufs
  using multiprocessing.Manager to keep blocked content synced between readers
2020-01-20 12:51:02 -05:00
Lex Berezhny 9d79c52d20 add index for claim_type and release_time 2020-01-20 12:50:20 -05:00
Lex Berezhny 1ac03e2f1d fix test 2020-01-20 12:50:16 -05:00
Lex Berezhny 801f05f45e fix json api generator 2020-01-20 12:50:16 -05:00
Lex Berezhny 345196aa3b add pagination for claim_search 2020-01-20 12:49:42 -05:00
Lex Berezhny a01b52421e regenerate protobufs 2020-01-20 12:49:41 -05:00
Lex Berezhny 86cedfe8b2 using multiprocessing.Manager to keep blocked content synced between readers 2020-01-20 12:49:41 -05:00
Alex Grintsvayg 709128225b Merge branch 'prometheus'
* prometheus:
  Add prometheus metrics collection to client and server
2020-01-20 12:45:11 -05:00
Alex Grintsvayg 59a5bacb2e Add prometheus metrics collection to client and server 2020-01-20 12:43:49 -05:00
Jack Robison 3a21df31ee v0.53.3 2020-01-16 16:43:26 -05:00
Jack Robison 0e3407ec2f Merge pull request #2730 from lbryio/bump-aioupnp2
update aioupnp requirement to 0.0.17
2020-01-16 16:00:42 -05:00
Jack Robison fd6609e961 bump aioupnp requirement to 0.0.17 2020-01-16 15:40:29 -05:00
Jack Robison b0b5e045ff non blocking aioupnp success analytics 2020-01-16 15:40:24 -05:00
Jack Robison f293d7cccc pin windows pyinstaller 2020-01-16 15:02:15 -05:00
Jack Robison 974ed29c36 Merge pull request #2729 from lbryio/fix-headers-subscription-to-old
disconnect from server that returns rpc error for server.version
2020-01-15 17:59:17 -05:00
Jack Robison 53eb033034 disconnect from server that returns rpc error for server.version
-the server returns this error if it thinks we are incompatible with it
2020-01-15 17:06:13 -05:00
Jack Robison f2545b98ab Merge pull request #2728 from lbryio/check-server-version-from-client
check minimum server version from wallet client
2020-01-15 16:53:10 -05:00
Jack Robison 1bf51e855d check minimum server version from wallet client 2020-01-15 15:55:39 -05:00
Jack Robison e4da2a695b v0.53.2 2020-01-14 16:47:56 -05:00
Lex Berezhny 1a905d9a42 make --start-tag work again in release/script.py 2020-01-13 21:40:07 -05:00
Victor Shyba 7d9d0c15d3 fix test_direct_sync flakiness 2020-01-13 21:17:10 -05:00
Victor Shyba d34f5c2712 fix None in history, check records after wait 2020-01-13 21:17:10 -05:00
Victor Shyba a56dd66c98 separate cases and make it deterministic 2020-01-13 21:17:10 -05:00
Victor Shyba 3946bc6662 fix and document new edge case on tests 2020-01-13 21:17:10 -05:00
Victor Shyba 6647dd8f08 fix decrypting invalid bytes with valid padding 2020-01-13 21:17:10 -05:00
Jack Robison 9371122bed Merge pull request #2725 from lbryio/stream-manager-startup
faster stream manager startup
2020-01-13 18:09:49 -05:00
Jack Robison 734cd8ee08 logging 2020-01-13 17:51:05 -05:00
Jack Robison d85d9d05bb fast get_all_lbry_files 2020-01-13 17:51:05 -05:00
Jack Robison 1d7b87b3a9 fix inefficient loop in update_manually_removed_files_since_last_run 2020-01-13 17:51:02 -05:00
Lex Berezhny f0cfde36f2 v0.53.1 2020-01-13 14:12:30 -05:00
Lex Berezhny 7e220d2741 downgrade setuptools 2020-01-13 14:03:45 -05:00
Lex Berezhny c4b1351a43 stop building on travis 2020-01-13 14:03:45 -05:00
Lex Berezhny 90d2597bc3 pin to pyinstaller==3.5 2020-01-13 14:03:45 -05:00
Lex Berezhny c1508c94c2 v0.53.0 2020-01-13 12:23:41 -05:00
Jack Robison 1a0d805bad Merge pull request #2720 from lbryio/loggly-database-error
fix database error caused by logging tracebacks from within the sqlite transaction runner
2020-01-12 16:23:26 -05:00
Jack Robison 1a802469f5 missing fetchall() 2020-01-12 16:16:48 -05:00
Jack Robison 646ae7e2d1 catch RuntimeError in loggly handler to handle cases where there is not an event loop 2020-01-12 16:16:48 -05:00
Jack Robison 563d76ff9f Merge pull request #2719 from lbryio/refactor_header_progress
refactor header progress calculation, making it simpler to understand and not prone to division by zero
2020-01-12 16:16:23 -05:00
Victor Shyba 54d0473e85 refactor header progress 2020-01-12 00:56:43 -03:00
Jack Robison ec8a61c3af Merge pull request #2718 from lbryio/add-connected-to-wallet-status
add `connected` server to wallet status
2020-01-11 01:11:34 -05:00
Jack Robison 0315eaff8a add connected server to wallet status 2020-01-11 00:33:52 -05:00
Jack Robison b34f44c867 logging 2020-01-11 00:17:32 -05:00
Jack Robison ea7056835f Merge pull request #2716 from lbryio/batched-address-subscriptions
speed up wallet sync and startup by batching address history subscriptions
2020-01-10 21:32:12 -05:00
Jack Robison 0bb4cdadd9 use network.subscribe_address 2020-01-10 13:57:52 -05:00
Jack Robison 08f6520557 bare excepts 2020-01-10 12:27:56 -05:00
Jack Robison 38b108752e batched blockchain.address.subscribe 2020-01-09 23:40:04 -05:00
Jack Robison 0ee7870bdf defaultdict 2020-01-09 23:06:29 -05:00
Jack Robison 37d46ecdb2 fix looping over same things in _transaction_io 2020-01-09 23:06:29 -05:00
Oleg Silkin 2a7911c7d6 rename jsonrpc_comment_edit -> jsonrpc_comment_update 2020-01-09 19:28:45 -05:00
Oleg Silkin fc2d9b4fd2 restart travis build 2020-01-09 19:28:45 -05:00
Oleg Silkin 832020fa81 Makes claim_id and parent_id mutually exclusive for comment create 2020-01-09 19:28:45 -05:00
Oleg Silkin 78606ed4b8 Generates docs for comments 2020-01-09 19:28:45 -05:00
Oleg Silkin a04ba606e6 Adds test for comment_edit 2020-01-09 19:28:45 -05:00
Oleg Silkin ebf2e7ee70 Better validation logic; 2020-01-09 19:28:45 -05:00
Oleg Silkin b31881f424 Removes possibility of no channel being passed in & propogates errors 2020-01-09 19:28:45 -05:00
Oleg Silkin c0a6f6fd08 Adds validator methods 2020-01-09 19:28:45 -05:00
Oleg Silkin 7e1b1ca730 Adds comment_edit command 2020-01-09 19:28:45 -05:00
Jack Robison 163d176f0a Merge pull request #2710 from lbryio/fix-version-and-stop
Fix `version` and `stop` rpcs
2020-01-08 17:42:10 -05:00
Jack Robison 0f7eab5100 pylint 2020-01-08 12:44:22 -05:00
Jack Robison cbf78d474a fix version and stop rpcs 2020-01-08 12:44:22 -05:00
Victor Shyba 3615e0de34 cleanup leftover comment from review 2020-01-07 19:57:34 -05:00
Victor Shyba e73c081d36 remove bandwidth_limit env option 2020-01-07 19:57:34 -05:00
Victor Shyba 663bf6e3af remove bandwidth_limit from docker file 2020-01-07 19:57:34 -05:00
Victor Shyba 60194cbafd remove bw_limit on server 2020-01-07 19:57:34 -05:00
Lex Berezhny 7d2eb5faf7 v0.52.0 2020-01-06 12:21:49 -05:00
Lex Berezhny 5969f3d213 revert version 2020-01-06 12:20:05 -05:00
Lex Berezhny 6ddde88389 update release script with new directory structure 2020-01-06 12:18:26 -05:00
Lex Berezhny e9992ed6b4 fix test 2020-01-06 12:17:09 -05:00
Lex Berezhny 4db823a3be set version to 0.52.0 2020-01-06 12:17:09 -05:00
Lex Berezhny 78732513c8 set wallet server min/max 0.52.0-0.99.0 2020-01-06 12:17:09 -05:00
Jack Robison 79557bb878 Merge pull request #2704 from lbryio/disable-cryptonator
disable cryptonator feed
2020-01-03 16:18:05 -05:00
Jack Robison 4094b02ae9 update test 2020-01-03 16:00:00 -05:00
Jack Robison 9cc7c118a8 disable cryptonator feed 2020-01-03 15:28:29 -05:00
Lex Berezhny d3dad51c76 fix import error in cli.py 2020-01-03 03:08:15 -05:00
Lex Berezhny 5bf35de955 fix blockchain integration tests 2020-01-03 03:08:15 -05:00
Lex Berezhny f8928c654b do not reset callable_methods on Daemon class 2020-01-03 03:08:15 -05:00
Lex Berezhny b3903f4ffd fix import in unit test 2020-01-03 03:08:15 -05:00
Lex Berezhny 46c3f76edc run all integration tests on travis until we drop it 2020-01-03 03:08:15 -05:00
Victor Shyba 3488408b7a add jobs=8 back to pylint 2020-01-03 03:08:15 -05:00
Victor Shyba b7b164b84c fix comment alignment 2020-01-03 03:08:15 -05:00
Victor Shyba 7c353b7d76 too many \ 2020-01-03 03:08:15 -05:00
Lex Berezhny 3d9841b61a fix unit wallet tests 2020-01-03 03:08:15 -05:00
Lex Berezhny 87e2d2c7b0 re-enable pylint on CI 2020-01-03 03:08:15 -05:00
Lex Berezhny ff59619af4 more pylint 2020-01-03 03:08:15 -05:00
Lex Berezhny b79f2b86b5 renamed Daemon.py to daemon.py per pylint 2020-01-03 03:08:15 -05:00
Lex Berezhny 386fd7a459 updated imports after renaming component files 2020-01-03 03:08:15 -05:00
Lex Berezhny d4f41901ef lower cased component file names per pylint 2020-01-03 03:08:15 -05:00
Lex Berezhny f170da3e78 pylint fixes in lbry/extras/daemon 2020-01-03 03:08:15 -05:00
Lex Berezhny 5dc15be98a pylint lbry/extras/daemon/exchange_rate_manager.py 2020-01-03 03:08:15 -05:00
Lex Berezhny 88c7cfc745 pylint in progress in lbry/extras/daemon 2020-01-03 03:08:15 -05:00
Victor Shyba 867478697d lint blob/* 2020-01-03 03:08:15 -05:00
Lex Berezhny d27e8cf73a pylint fix in lbry/error/generate.py 2020-01-03 03:08:15 -05:00
Victor Shyba 4490caa4f7 lint storage 2020-01-03 03:08:15 -05:00
Lex Berezhny 1b25f2c531 pylint fix in lbry/utils.py 2020-01-03 03:08:15 -05:00
Lex Berezhny 9d911d1fa0 convince pylint about descriptor return type in lbry/conf.py 2020-01-03 03:08:15 -05:00
Victor Shyba 494feb9f6d lint: lbry/stream/* 2020-01-03 03:08:15 -05:00
Lex Berezhny 2a04943a67 add l variable for lists to allowed good-names in pylint 2020-01-03 03:08:15 -05:00
Lex Berezhny 731b29ce91 progress on pylint in lbry/conf.py 2020-01-03 03:08:15 -05:00
Victor Shyba 28fbb70858 blob_exchange lint 2020-01-03 03:08:15 -05:00
Victor Shyba 44f402c64e delete cryptoutils 2020-01-03 03:08:15 -05:00
Victor Shyba c7f391ca44 lint: finish dht parts 2020-01-03 03:08:15 -05:00
Lex Berezhny 5ed9c5e168 ignore pylint for winpaths.py 2020-01-03 03:08:15 -05:00
Lex Berezhny 51b1a86d19 pylint for lbry/connection_manager.py 2020-01-03 03:08:15 -05:00
Victor Shyba 20c46677d0 lint dht: datastore protocol 2020-01-03 03:08:15 -05:00
Lex Berezhny efb5f232f7 fixed pylint for lbry/utils.py 2020-01-03 03:08:15 -05:00
Lex Berezhny 9edf9561b2 pylint 2020-01-03 03:08:15 -05:00
Lex Berezhny 5d4eb018ee pylint for lbry/testcase.py 2020-01-03 03:08:15 -05:00
Victor Shyba 10fbce056b dht constants -> CONSTANTS (linting) 2020-01-03 03:08:15 -05:00
Lex Berezhny cbc6d6a572 pylint passes for lbry/wallet 2020-01-03 03:08:15 -05:00
Lex Berezhny fb1af9e3d2 update imports and more merging 2020-01-03 03:08:15 -05:00
Lex Berezhny c9e410a6f4 merged torba base classes with lbry sub-classes 2020-01-03 03:08:15 -05:00
Lex Berezhny c8d72b59c0 fix readme urls 2020-01-01 16:02:16 -05:00
Lex Berezhny cd71d4a2f7 remove double install of coverage package 2020-01-01 16:02:11 -05:00
Lex Berezhny ed0cc59e66 fix cyclic import error 2020-01-01 15:57:56 -05:00
Lex Berezhny d4f1b48f52 renamed integration tests to be clearer in gitlab UI 2020-01-01 15:57:56 -05:00
Lex Berezhny 08bec02170 added __init__ 2020-01-01 15:57:56 -05:00
Lex Berezhny c9f27b83e1 split integration tests into three jobs 2020-01-01 15:57:56 -05:00
Lex Berezhny 8327585b3b fix set_build.py and other things 2020-01-01 15:57:56 -05:00
Lex Berezhny 727815d6dd updating scripts after moving lbry up one level 2020-01-01 15:57:56 -05:00
Lex Berezhny 2968f74c6c moved lbry up one level 2020-01-01 15:57:56 -05:00
Lex Berezhny 431499f43f do not install torba 2020-01-01 15:57:56 -05:00
Lex Berezhny d809283d23 fix integrartion test 2020-01-01 15:57:56 -05:00
Lex Berezhny b49275fe6b added support for --no-logging argument to avoid leaking loggers in tests 2020-01-01 15:57:56 -05:00
Lex Berezhny 87c7ce588e disabled cryptonator.com in tests, removed references to torba 2020-01-01 15:57:56 -05:00
Lex Berezhny 26d0a7c742 fix generate_json_api 2020-01-01 15:57:56 -05:00
Lex Berezhny b01887c7ec fix two unit tests 2020-01-01 15:57:56 -05:00
Lex Berezhny 745031c020 orchstr8 works again, for now... 2020-01-01 15:57:56 -05:00
Lex Berezhny 3ccfc09e01 conslidated testcase 2020-01-01 15:57:56 -05:00
Lex Berezhny e8750275c9 skip code quality checks, will fix later 2020-01-01 15:57:56 -05:00
Lex Berezhny f0d7ea4cc6 updated files and scripts post torba merge 2020-01-01 15:57:56 -05:00
Lex Berezhny ea5322af82 removing bitcoin support 2020-01-01 15:57:56 -05:00
Lex Berezhny 0b23f68fb2 merged torba into lbry 2020-01-01 15:57:56 -05:00
Lex Berezhny 1c00129f76 v0.51.2 2019-12-30 16:34:52 -05:00
Lex Berezhny 454700af05 support json in setting_set 2019-12-30 16:31:14 -05:00
Jack Robison d9b482d90a Merge pull request #2699 from lbryio/faster-account-startup
don't log account details on startup if there are more than 10
2019-12-30 16:26:36 -05:00
Jack Robison 09c2c97069 only log account details on startup if there are not very many 2019-12-30 16:22:41 -05:00
Lex Berezhny 88faf0ce78 v0.51.1 2019-12-30 15:11:29 -05:00
Jack Robison aef83eec31 Merge pull request #2695 from lbryio/fix-12-to-13-migrator
Fix 12 to 13 migrator
2019-12-28 15:57:41 -05:00
Jack Robison 5bd222c266 less verbose log 2019-12-28 13:17:19 -05:00
Jack Robison d3e7f789b2 uncaught exception 2019-12-28 13:17:19 -05:00
Jack Robison 87c6e292f1 fix wallet server SessionManager._get_info 2019-12-28 13:17:19 -05:00
Jack Robison b597ad0b3f fix unique constraint failure in migrate12to13 2019-12-28 13:17:19 -05:00
Alex Grintsvayg 40dd99dea3 changelog note 2019-12-28 12:39:46 -05:00
Lex Berezhny 6ea2686219 v0.51.0 2019-12-23 12:39:48 -05:00
Alex Grintsvayg 6bca90c4f7 allow wallet snapshots using different compression algos 2019-12-21 19:18:20 -05:00
Alex Grintsvayg 20fa7bd852 better server range for boris 2019-12-21 13:48:07 -05:00
Alex Grintsvayg 6b80119eff drop travis badge 2019-12-21 13:29:09 -05:00
Alex Grintsvayg 1fc5469eb1 add gitlab badge 2019-12-21 13:26:38 -05:00
Alex Grintsvayg bf2769a0c2 disable travis docker builds. docker builds trigger from docker hub 2019-12-21 13:06:03 -05:00
Victor Shyba 325b601bea enable WAL 2019-12-21 07:49:21 -05:00
Lex Berezhny 126642912e lint 2019-12-20 16:52:56 -05:00
Lex Berezhny 57ee16d565 change returned error data structure to be JSONRPC standard compliant 2019-12-20 16:52:56 -05:00
Alex Grintsvayg 9f0fba063d ensure asset exists before upload 2019-12-20 13:01:51 -05:00
Victor Shyba 757fb51415 make block_expected public 2019-12-20 12:30:16 -05:00
Victor Shyba b1c5655138 __height -> _block_expected 2019-12-20 12:30:16 -05:00
Alex Grintsvayg 4d97dcaa93 remove extraneous apostrophe 2019-12-20 10:26:41 -05:00
Jack Robison 96db1b4685 v0.50.1 2019-12-19 22:09:19 -05:00
Jack Robison 9e21c52d04 Merge pull request #2685 from lbryio/fast_broadcast_notify
Update balance after a broadcast transaction is accepted to mempool without a delay, fix bug in `status` header progress
2019-12-19 22:07:16 -05:00
Victor Shyba 5fa801f9f2 broadcast_or_release uses None as timeout as opposed to tests 2019-12-19 23:42:31 -03:00
Victor Shyba 0301768b79 lint on tests 2019-12-19 22:58:36 -03:00
Victor Shyba 4c6dedfa4f fix last test 2019-12-19 22:34:10 -03:00
Victor Shyba 6652d55455 wip 2019-12-19 21:46:14 -03:00
Victor Shyba 3e03dd3e80 apply timeout and check for expected height on wait 2019-12-19 20:20:42 -03:00
Victor Shyba 84b8a22423 comment the broadcast interruption case 2019-12-19 16:20:12 -03:00
Victor Shyba ada03e12fc proper check for restrictions on gettx 2019-12-19 15:13:13 -03:00
Victor Shyba d66c801350 wake up mempool on broadcast 2019-12-19 15:13:13 -03:00
Victor Shyba 91846939f6 organize logic for when its downloading 2019-12-18 19:09:33 -03:00
Jack Robison c85127d76b fix division by zero error 2019-12-18 10:44:16 -05:00
Lex Berezhny 166bf65e88 torba test_database fix 2019-12-17 23:48:07 -05:00
Lex Berezhny f68bdc406f fix bug when two updates for the same claim happened in the same block 2019-12-17 23:48:07 -05:00
Lex Berezhny 8390482d6c aws does not use memoryview 2019-12-17 23:48:07 -05:00
Alex Grintsvayg 783f454f8e dont include dist dir in windows zip 2019-12-16 20:32:42 -05:00
Alex Grintsvayg 86a67bd245 autopublish release and notify in slack 2019-12-16 16:40:56 -05:00
Alex Grintsvayg 099b0b65c0 run torba unit tests the way lex runs them 2019-12-16 12:59:15 -05:00
Lex Berezhny 505652efa0 v0.50.0 2019-12-16 11:33:13 -05:00
Victor Shyba a20088330f use buffer write return as written 2019-12-16 11:31:28 -05:00
Jack Robison a20e2504bb disable timing bug in test_ping_queue_discover 2019-12-16 10:10:58 -05:00
Lex Berezhny b07bc150e1 do not registery fake component in real component registry 2019-12-15 02:39:17 -05:00
Lex Berezhny 46c6588aa1 fixup fake exchange rate manager for integration tests 2019-12-15 02:39:17 -05:00
Lex Berezhny e36c672c9a pylint 2019-12-15 02:39:17 -05:00
Lex Berezhny c044e1ea8c further refactor and simplification 2019-12-15 02:39:17 -05:00
Lex Berezhny 25b6c1b6ca renamed test_ExchangeRateManager to test_exchange_rate_manager 2019-12-15 02:39:17 -05:00
Lex Berezhny c25d72d911 minor error class and type checking fixes 2019-12-15 02:39:17 -05:00
Miroslav Kovar 73613d1583 Refactor exchange rate manager 2019-12-15 02:39:17 -05:00
Lex Berezhny 1e6542d12d add --blocklist_channel_ids to claim_search command 2019-12-14 19:19:33 -05:00
Lex Berezhny 3cffaa43f7 remove invalid claim_id from default channel filter list 2019-12-14 19:19:33 -05:00
Lex Berezhny 08792e794f a support create and abandon in same block no longer inserts support 2019-12-14 18:19:06 -05:00
Lex Berezhny 7fa26d59da fixes for various edge cases where claim creation, updating and abandoning is occuring in the same block 2019-12-14 18:19:06 -05:00
Lex Berezhny 74469829e5 pylint 2019-12-14 15:47:28 -05:00
Lex Berezhny ba414742a6 improved the data structure returned from RPC for errors by adding a error name and other metadata 2019-12-14 15:47:28 -05:00
Lex Berezhny f6fcfb6dde simplified and only log potentially transactional functions 2019-12-13 18:44:32 -05:00
Oleg Silkin c5b90c0144 Removes unused debug flag 2019-12-13 18:44:32 -05:00
Oleg Silkin f669881849 Adds function, param logging to all lbrynet handling 2019-12-13 18:44:32 -05:00
Alex Grintsvayg d0b70ea6df upload assets from gitlab CI to github on a tagged release
also disables travis asset uploading
2019-12-13 13:35:40 -05:00
Lex Berezhny 51af43d492 move none_check after fee update since that can be None 2019-12-13 13:23:23 -05:00
Lex Berezhny 8a49ad4586 catch invalid None value for all cases when creating/updating claims 2019-12-13 13:23:23 -05:00
gpjacobs 2ae068c2d2 Improving error handling for null values 2019-12-13 13:23:23 -05:00
Alex Grintsvayg 5e07e8dbbd copy some of lex's changes to boris 2019-12-13 10:12:47 -05:00
Lex Berezhny 8d2c68505e remove VERBOSITY from test 2019-12-12 13:25:27 -05:00
Lex Berezhny 44f1e7c401 test all possible P2SH and BECH32 payment combinations on wallet server and at least P2SH and BECH32 to P2PKH receipt on client address 2019-12-12 13:25:27 -05:00
Lex Berezhny fd2945469e pylint 2019-12-12 13:25:27 -05:00
Lex Berezhny b0f0827cff client side segwit support 2019-12-12 13:25:27 -05:00
Lex Berezhny c1e2c415c4 TaskGroup cannot work without even loop 2019-12-11 22:48:39 -05:00
Victor Shyba a258b96602 simplify test 2019-12-11 22:48:39 -05:00
Victor Shyba 82ef76b1cb set regardless, there is no need for the check 2019-12-11 22:48:39 -05:00
Victor Shyba aaa9982932 missing test file 2019-12-11 22:48:39 -05:00
Victor Shyba 1cdb4bc9e7 task group: set done if empty 2019-12-11 22:48:39 -05:00
Alex Grintsvayg fddb0c9e66 notify us when boris is killed with signals 2019-12-11 14:02:39 -05:00
Lex Berezhny 5c78d2b6d2 install wget in Dockerfile 2019-12-11 13:08:02 -05:00
Victor Shyba a4113cbb52 add apsw to pylint conf 2019-12-11 12:38:00 -05:00
Victor Shyba afa3bda3c8 update tests 2019-12-11 12:06:41 -05:00
Victor Shyba 61d7edd15f change api name to wallet reconnect 2019-12-11 12:06:41 -05:00
Victor Shyba c2106de7e1 refactor + tests 2019-12-11 12:06:41 -05:00
Victor Shyba 0bf1be6198 add wallet restart API 2019-12-11 12:06:41 -05:00
Lex Berezhny af7c20e440 downgrade multidict to prevent segfault 2019-12-11 09:44:37 -05:00
Lex Berezhny 17771a2b89 added --initial-headers argument to lbrynet start command 2019-12-11 09:44:37 -05:00
Lex Berezhny 8840097fe2 test that creator can spend purchases received 2019-12-09 22:32:14 -05:00
Lex Berezhny 7d333efd45 utxo list includes purchases of your claims 2019-12-09 22:32:14 -05:00
Lex Berezhny ec65815c57 add parsing for segwit outpoint 2019-12-09 20:59:31 -05:00
Lex Berezhny 640b0c71e6 v0.49.0 2019-12-09 13:05:08 -05:00
Victor Shyba 4208876ef3 add header sync to update tasks so it can be cancelled 2019-12-09 13:04:23 -05:00
Victor Shyba bff9117360 progress from 0 2019-12-09 13:04:23 -05:00
Victor Shyba 6b3c4c70d2 dont timeout if data being transferred 2019-12-09 13:04:23 -05:00
Lex Berezhny 0292fd8d91 fix lint error 2019-12-08 22:40:06 -05:00
Lex Berezhny 94a41270d0 add InvalidPasswordError handling to more places 2019-12-08 22:40:06 -05:00
Lex Berezhny 75d78bfa53 added InvalidPasswordError code when password is invalid 2019-12-08 22:40:06 -05:00
Victor Shyba 88263db831 moar variables limit 2019-12-08 17:00:41 -05:00
Lex Berezhny 3ef96404ee use make install in Dockerfile 2019-12-08 17:00:41 -05:00
Lex Berezhny 13f9370f8c fix wallet server db writer test 2019-12-08 17:00:41 -05:00
Lex Berezhny 1c349270bf switch to apsw 2019-12-08 17:00:41 -05:00
Lex Berezhny c28f0f6286 added apsw installation to tox 2019-12-08 17:00:41 -05:00
Lex Berezhny b20939d211 added apsw installation 2019-12-08 17:00:41 -05:00
Lex Berezhny b76c45fc2b drop jonathonf/backports sqlite3 PPA from .travis.yml since the package was removed 2019-12-08 17:00:41 -05:00
Lex Berezhny 1bbfccd082 refactored error codes generate script and error hierarchy 2019-12-06 10:45:23 -05:00
Lex Berezhny cee7e06832 add configurable log level to exceptions, join message and description columns in error markdown table 2019-12-06 10:45:23 -05:00
Jack Robison 03dff4b382 fix build 2019-12-05 20:56:57 -05:00
Jack Robison f546b20a01 v0.48.2 2019-12-04 13:10:40 -05:00
Jack Robison 0e8d9a264f Merge pull request #2601 from mirgee/fix-2490
Persist and restore routing table
2019-12-03 15:44:01 -05:00
Jack Robison 66a4c98bee update test 2019-12-03 13:11:34 -05:00
Miroslav Kovar 36101db500 Wait for routing table to clear with timeout 2019-12-03 13:06:50 -05:00
Miroslav Kovar 6bff298d1e Add migrator for the new peer table 2019-12-03 13:06:50 -05:00
Jack Robison c832f8ffbb fix mock_network_loop param 2019-12-03 13:06:50 -05:00
Jack Robison d7fe46dbde fix drop/reconnect in test_losing_connection 2019-12-03 13:06:50 -05:00
Miroslav Kovar 5951186463 Minor changes
Fix typos
2019-12-03 13:06:50 -05:00
Miroslav Kovar 7191042bb8 Add new test, remove old test, change error type 2019-12-03 13:06:50 -05:00
Miroslav Kovar a80fbcc252 Catch resolve timeouts 2019-12-03 13:06:50 -05:00
Miroslav Kovar c321758afd Rename, fix tests / deduplicate set_joined, resolve conditionally 2019-12-03 13:06:50 -05:00
Miroslav Kovar ca8f59a643 Persist and restore routing table 2019-12-03 13:06:50 -05:00
Jack Robison 8110882617 Merge pull request #2650 from lbryio/fix-blob-request-loop
Fix blob request loop
2019-12-03 12:58:27 -05:00
Jack Robison a283b66541 add timeout to file_save attempt starting
-remove test_unban_recovers_stream
2019-12-03 11:56:59 -05:00
Jack Robison 569de37e16 fix sendfile exceptions 2019-12-03 11:56:55 -05:00
Jack Robison 669f3394c7 fix requesting same blob over and over if only peer(s) say they don't have it 2019-12-03 11:56:52 -05:00
Alex Grin 87ec2c7378 v0.48.1 2019-12-02 12:39:41 -05:00
Jack Robison ea7a31cad1 Merge pull request #2649 from lbryio/invalid-ips
Fix handling reserved ips
2019-12-01 18:06:32 -05:00
Jack Robison c894ade25e fix integration tests 2019-12-01 17:28:51 -05:00
Jack Robison a3fe127a92 fix and test reserved ips 2019-12-01 17:04:03 -05:00
Jack Robison 7f526be879 Merge pull request #2648 from lbryio/fix-blob-udp-ping
Fix UDP ping port in blob peer accumulator
2019-12-01 17:00:12 -05:00
Jack Robison 71ae174e29 fix predicting udp port in _accumulate_peers_for_value
-add comments
-rename functions to be more descriptive of what they do
2019-12-01 16:21:15 -05:00
Jack Robison 2277d134cc Merge pull request #2647 from lbryio/drain-iterative-findvalue
Request all pages of peers for a blob
2019-12-01 15:52:28 -05:00
Jack Robison 880aa265f1 drain all pages of peers returned by find_value in IterativeValueFinder 2019-12-01 14:53:39 -05:00
Alex Grintsvayg ea639e64a4 add docker-compose file for wallet server 2019-11-27 10:18:39 -05:00
Lex Berezhny b60fdee5e2 upgrade SDK to use lbrycrd v0.17.3.2 2019-11-26 21:31:31 -05:00
Alex Grintsvayg 44f07fb81d move build-related files into top-level dir 2019-11-26 18:28:58 -05:00
Alex Grintsvayg 60a85fadd3 no more codecov 2019-11-26 18:28:58 -05:00
Alex Grintsvayg 68367125e1 gitlab builds on windows
what's left:
  - github build progress notifications (need to upgrade to Gitlab Silver for this)
  - github releases
  - uploading coverage results to codecov.io
  - what directories to cache between builds
2019-11-26 18:28:58 -05:00
Alex Grintsvayg 6112f60681 gitlab ci setup
what works:
- tests
- linux and mac builds
- uploading builds to s3

what's left to do:
- uploading coverage results to codecov.io
- windows build
- github releases
- what directories to cache between builds
2019-11-26 18:28:58 -05:00
Alex Grintsvayg ecc74e2ae5 if user has no Downloads dir on windows, use cwd 2019-11-26 18:28:58 -05:00
jessop c2dae43965 response 2019-11-26 14:45:41 -05:00
jessop 6e3a25d255 status: wallet returns all servers rather than just available 2019-11-26 14:45:41 -05:00
jessop 4e13cf1d85 adds wallet server counts on startup 2019-11-26 14:45:41 -05:00
Lex Berezhny 0fee1897eb v0.48.0 2019-11-25 20:39:36 -05:00
Victor Shyba 9dd51467fe move lbry specific version to its place 2019-11-25 20:21:13 -05:00
Victor Shyba cec6dca72f bump protocol version 2019-11-25 20:21:13 -05:00
Victor Shyba 58df2b9bcf ignore cancelled error 2019-11-25 16:07:03 -05:00
Victor Shyba cc6bdd6295 sync fts only once 2019-11-25 16:07:03 -05:00
Victor Shyba 7a044f9f78 opening dbs can raise too 2019-11-25 16:07:03 -05:00
Victor Shyba e77acde005 wip 2019-11-25 16:07:03 -05:00
Victor Shyba ece5082096 enable callback tracebacks 2019-11-25 16:07:03 -05:00
Alex Grintsvayg 0723026a56 drop codecov 2019-11-25 12:52:30 -05:00
Lex Berezhny 0b7346f4a1 droped OldWalletServerTransaction class in wallet server unit tests 2019-11-25 10:22:34 -05:00
Lex Berezhny a357db1ff6 use correct segwit deserializer 2019-11-25 10:22:34 -05:00
Jack Robison a2287f8d0a Merge pull request #2637 from lbryio/fix-unhandled-download-errors
don't log tracebacks for expected download errors
2019-11-25 10:11:37 -05:00
Jack Robison 076decd6e9 don't log tracebacks for expected download errors 2019-11-25 09:47:42 -05:00
jessop d40461de94 review changes 2019-11-22 22:28:55 -05:00
jessop 4b1492be0d return value and more tests 2019-11-22 22:28:55 -05:00
jessop 1676246743 cleanup 2019-11-22 22:28:55 -05:00
jessop f7c9f6d142 adds settings_clear method and integration test 2019-11-22 22:28:55 -05:00
Alex Grintsvayg 6022339091 Revert "dont show loop blocking warnings when running tests in CI"
This reverts commit 25d97afa60.
2019-11-22 20:27:10 -05:00
Alex Grintsvayg 25d97afa60 dont show loop blocking warnings when running tests in CI 2019-11-22 11:55:04 -05:00
Alex Grintsvayg df635b963c enable passing args to tests 2019-11-21 15:31:49 -05:00
Lex Berezhny 6d0a054170 regenerate latest docs with claim_id argument to stream_repost 2019-11-20 10:56:42 -05:00
Lex Berezhny 177d3af77f regenerate latest docs 2019-11-20 10:39:17 -05:00
Lex Berezhny 35f71c619b fix unit tests 2019-11-19 22:26:30 -05:00
Lex Berezhny 31a80ccce4 minor fixups 2019-11-19 22:26:30 -05:00
Lex Berezhny 3194cec8df added missing files 2019-11-19 22:26:30 -05:00
Lex Berezhny 57fd3c5801 updated code base to use generated errors 2019-11-19 22:26:30 -05:00
Lex Berezhny 86617f1bda added names to errors and script to generate error classes from README 2019-11-19 22:26:30 -05:00
Lex Berezhny 35f36c598c moved error.py to be a module 2019-11-19 22:26:30 -05:00
Lex Berezhny 36bd7816fe add handling for invalid claims in calculate_reposts 2019-11-18 17:07:46 -05:00
Lex Berezhny fd632392d4 full reposted_claim data returned for claim_search and resolve 2019-11-18 17:07:46 -05:00
Victor Shyba 423b48866f apply refactor suggested on review by removing .out calls 2019-11-18 17:07:46 -05:00
Victor Shyba 745bc24343 add a test for normal repost mixed with blocking repost, use inner queries 2019-11-18 17:07:46 -05:00
Victor Shyba 04a823c7d0 FILTERING_CHANNELS_IDS on claim search 2019-11-18 17:07:46 -05:00
Victor Shyba 56af4c2fcb minor fixes from rebase 2019-11-18 17:07:46 -05:00
Victor Shyba 77daf6812a add config option for filtering channels 2019-11-18 17:07:46 -05:00
Victor Shyba c7e964ec42 add reposted_claim_hash column, writer and reader w/ tests 2019-11-18 17:07:46 -05:00
Victor Shyba 694e2c2a4f improve test cases for reposts, add a new one for reverse search 2019-11-18 17:07:46 -05:00
Victor Shyba df6537fae8 add API for reposting 2019-11-18 17:07:46 -05:00
Lex Berezhny 71f6542571 v0.47.0 2019-11-18 10:57:58 -05:00
Victor Shyba 5a6294c08e test errors arent trapped 2019-11-18 10:57:24 -05:00
Victor Shyba f0390786d6 avoid trapping errors on client 2019-11-18 10:57:24 -05:00
Jack Robison 719d18c670 Merge pull request #2620 from lbryio/torrent-db-support
Add torrent support to SQLiteStorage
2019-11-15 18:34:00 -05:00
Jack Robison a116dcd3bb add SQLiteStorage.delete_torrent 2019-11-15 16:53:19 -05:00
Jack Robison e2dd3dcf88 get_claims_from_torrent_info_hashes 2019-11-15 16:53:19 -05:00
Jack Robison 822f0c3cb2 add migrate12to13 script 2019-11-15 16:53:14 -05:00
Jack Robison 63fb39016b add torrent db tables
-decouple file table from stream table
2019-11-15 15:25:33 -05:00
Jack Robison e79f8d58ca expose bt_infohash in Claim 2019-11-15 15:25:33 -05:00
Lex Berezhny a469dfb583 stop signing support tx too many times 2019-11-15 14:16:54 -05:00
Victor Shyba 6cbc545f84 fix .connect injected method 2019-11-15 13:59:28 -05:00
Victor Shyba ea1e24d8f9 refactor from review 2019-11-15 13:59:28 -05:00
Victor Shyba 6fcb4d93c9 header component now shows header synchronization progress 2019-11-15 13:59:28 -05:00
Victor Shyba 36031c9d0f fix test, off by 1 2019-11-15 13:59:28 -05:00
Victor Shyba a70f5e2440 update block status during download 2019-11-15 13:59:28 -05:00
Victor Shyba 6c42aab567 stop spurious warning from wrong check 2019-11-15 13:59:28 -05:00
Victor Shyba 27631a1746 test initial header sync separately 2019-11-15 13:59:28 -05:00
Victor Shyba 320f5cebc7 cleanup 2019-11-15 13:59:28 -05:00
Victor Shyba ea2a583803 wip 2019-11-15 13:59:28 -05:00
Victor Shyba 9965801258 base64 headers 2019-11-15 13:59:28 -05:00
Victor Shyba 24ed0521c7 checkpoints 2019-11-15 13:59:28 -05:00
Victor Shyba d8fed79810 remove headers component 2019-11-15 13:59:28 -05:00
Victor Shyba c30db15efa faster header download 2019-11-15 13:59:28 -05:00
Jack Robison 817869b915 Merge pull request #2623 from lbryio/aioupnp-0.0.16
bump aioupnp to 0.0.16, fixes UPnP support for TP-Link routers
2019-11-15 11:50:59 -05:00
Jack Robison 3038dd9440 bump aioupnp -> 0.0.16 2019-11-15 11:24:49 -05:00
Lex Berezhny 276b2c737a pylint 2019-11-14 18:53:20 -05:00
Lex Berezhny 62543b94c9 minor cleanup of collections implementation 2019-11-14 18:53:20 -05:00
jessop 4d607ed276 review response 2019-11-14 18:53:20 -05:00
jessop 26e2865e35 cleanup 2019-11-14 18:53:20 -05:00
jessop f7c94c9eb5 collection resolve endpoint 2019-11-14 18:53:20 -05:00
jessop 25a91f89fd tests 2019-11-14 18:53:20 -05:00
jessop 246f055145 enables collections 2019-11-14 18:53:20 -05:00
Lex Berezhny 8c3bf0d4cb try ubuntu bionic on travis 2019-11-14 17:19:10 -05:00
Lex Berezhny e49f24e95f added more tests to make sure stream update and abandon update the full text search indexes 2019-11-14 17:19:10 -05:00
Lex Berezhny 48a18ff771 full text search in wallet server 2019-11-14 17:19:10 -05:00
Alex Grintsvayg 37161057c6 plz explain errors 2019-11-14 09:52:30 -05:00
Alex Grintsvayg 68d1157495 catch the right error, boris 2019-11-13 17:58:47 -05:00
Alex Grintsvayg 19c3b2f4b6 add a latest-release tag to wallet server docker builds 2019-11-13 12:08:04 -05:00
480 changed files with 49518 additions and 52186 deletions

No files matched your search

+197
View File
@@ -0,0 +1,197 @@
name: ci
on: push
jobs:
lint:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: extract pip cache
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: |
pip install --user --upgrade pip wheel
pip install -e .[lint]
- run: make lint
tests-unit:
name: "tests / unit"
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: set pip cache dir
id: pip-cache
run: echo "::set-output name=dir::$(pip cache dir)"
- name: extract pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: |
pip install --user --upgrade pip wheel
pip install -e .[test]
- env:
HOME: /tmp
run: coverage run -m unittest discover -v tests.unit
- env:
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_PARALLEL: true
name: Submit to coveralls
run: |
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
coveralls
tests-integration:
name: "tests / integration"
runs-on: ubuntu-20.04
strategy:
matrix:
test:
# - datanetwork
- blockchain
# - other
db:
- sqlite
- postgres
services:
postgres:
image: postgres:12
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
- name: extract pip cache
uses: actions/cache@v2
with:
path: ./.tox
key: tox-integration-${{ matrix.test }}-${{ matrix.db }}-${{ hashFiles('setup.py') }}
restore-keys: txo-integration-${{ matrix.test }}-${{ matrix.db }}-
- run: pip install tox
- env:
TEST_DB: ${{ matrix.db }}
run: tox -e ${{ matrix.test }}
- env:
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_PARALLEL: true
name: Submit to coveralls
run: |
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
coverage combine tests
coveralls
coveralls-finished:
needs: ["tests-unit", "tests-integration"]
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@57daa114
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
parallel-finished: true
build:
needs: ["lint", "tests-unit", "tests-integration"]
name: "build"
strategy:
matrix:
os:
- ubuntu-16.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: set pip cache dir
id: pip-cache
run: echo "::set-output name=dir::$(pip cache dir)"
- name: extract pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- name: Setup
run: |
pip install --user --upgrade pip wheel
pip install pyinstaller@git+https://github.com/eukreign/pyinstaller.git@sqlalchemy
- if: startsWith(runner.os, 'linux')
run: |
sudo apt-get install libzmq3-dev
pip install -e .[postgres]
- if: startsWith(runner.os, 'mac')
run: |
brew install zeromq
pip install -e .
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
name: Build & Run (Unix)
run: |
pyinstaller --onefile --name lbrynet lbry/cli.py
chmod +x dist/lbrynet
dist/lbrynet --version
- if: startsWith(runner.os, 'windows')
name: Build & Run (Windows)
run: |
pip install pywin32
pip install -e .
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/cli.py
dist/lbrynet.exe --version
- uses: actions/upload-artifact@v2
with:
name: lbrynet-${{ matrix.os }}
path: dist/
docker:
needs: ["build"]
name: "build (docker)"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- name: fetch lbrynet binary
uses: actions/download-artifact@v2
with:
name: lbrynet-ubuntu-16.04
- run: |
chmod +x lbrynet
mv lbrynet docker
- name: build and push docker image
uses: docker/build-push-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
repository: lbry/lbrynet
path: docker
tag_with_ref: true
tag_with_sha: true
add_git_labels: true
+16
View File
@@ -1,2 +1,18 @@
/.idea
/.DS_Store
/build
/dist
/.tox
/.coverage*
/lbry-venv
/venv
lbry.egg-info
__pycache__
_trial_temp/
/tests/integration/commands/files
/tests/.coverage.*
/lbry/blockchain/bin
/lbry/wallet/bin
+210
View File
@@ -0,0 +1,210 @@
default:
image: python:3.7
#cache:
# directories:
# - $HOME/venv
# - $HOME/.cache/pip
# - $HOME/Library/Caches/pip
# - $HOME/Library/Caches/Homebrew
# - $TRAVIS_BUILD_DIR/.tox
stages:
- test
- build
- assets
- release
.tagged:
rules:
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
when: on_success
test:lint:
stage: test
script:
- make install tools
- make lint
test:unit:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
test:datanetwork-integration:
stage: test
script:
- pip install tox-travis
- tox -e datanetwork
test:blockchain-integration:
stage: test
script:
- pip install tox-travis
- tox -e blockchain
test:other-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e other
test:json-api:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
.build:
stage: build
artifacts:
expire_in: 1 day
paths:
- lbrynet-${OS}.zip
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller
- pip install -e .
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
- chmod +x dist/lbrynet
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
- dist/lbrynet --version
build:linux:
extends: .build
image: ubuntu:16.04
variables:
OS: linux
before_script:
- apt-get update
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get update
- apt-get install -y --no-install-recommends python3.7-dev
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
build:mac:
extends: .build
tags: [macos] # makes gitlab use the mac runner
variables:
OS: mac
GIT_DEPTH: 5
VENV: /tmp/gitlab-lbry-sdk-venv
before_script:
# - brew upgrade python || true
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
# - pip3 install --user --upgrade pip virtualenv
- pip3 --version | grep -q '\(python 3\.7\)'
- virtualenv --python=python3.7 "${VENV}"
- source "${VENV}/bin/activate"
after_script:
- rm -rf "${VENV}"
build:windows:
extends: .build
tags: [windows] # makes gitlab use the windows runner
variables:
OS: windows
GIT_DEPTH: 5
before_script:
- ./docker/install_choco.ps1
- choco install -y --x86 python3 7zip checksum
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
- pip --version # | findstr /E '\(python 3\.7\)'
- pip install virtualenv pywin32
- virtualenv venv
- venv/Scripts/activate.ps1
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
after_script:
- rmdir -Recurse venv
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.5
- pip install -e .
- python docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
- dist/lbrynet.exe --version
# s3 = upload asset to s3 (build.lbry.io)
.s3:
stage: assets
variables:
GIT_STRATEGY: none
script:
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
- pip install awscli
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
s3:linux:
extends: .s3
variables: {OS: linux}
needs: ["build:linux"]
s3:mac:
extends: .s3
variables: {OS: mac}
needs: ["build:mac"]
s3:windows:
extends: .s3
variables: {OS: windows}
needs: ["build:windows"]
# github = upload assets to github when there's a tagged release
.github:
extends: .tagged
stage: assets
variables:
GIT_STRATEGY: none
script:
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
- pip install githubrelease
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
github:linux:
extends: .github
variables: {OS: linux}
needs: ["build:linux"]
github:mac:
extends: .github
variables: {OS: mac}
needs: ["build:mac"]
github:windows:
extends: .github
variables: {OS: windows}
needs: ["build:windows"]
publish:
extends: .tagged
stage: release
variables:
GIT_STRATEGY: none
script:
- pip install githubrelease
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
- >
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
-146
View File
@@ -1,146 +0,0 @@
dist: xenial
language: python
python: "3.7"
jobs:
include:
- stage: code quality
name: "pylint & mypy"
install:
- make install
script: make lint
- stage: test
name: "LBRY Unit Tests"
install:
- make install
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
after_success:
- coverage combine lbry/
- bash <(curl -s https://codecov.io/bash)
- name: "LBRY Integration Tests"
install:
- pip install coverage tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: cd lbry && tox
after_success:
- coverage combine lbry
- bash <(curl -s https://codecov.io/bash)
- &torba-tests
name: "Torba Unit Tests"
env: TESTTYPE=unit
install:
- pip install coverage tox-travis
script: cd torba && tox
after_success:
- coverage combine torba/tests
- bash <(curl -s https://codecov.io/bash)
- <<: *torba-tests
name: "Torba Integration Tests"
env: TESTTYPE=integration
- name: "Run Examples"
install:
- make install
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
after_success:
- coverage combine lbry
- bash <(curl -s https://codecov.io/bash)
- &build
stage: build
name: "Linux"
env: OS=linux
install:
- pip install pyinstaller awscli
- cd torba && pip install -e . && cd ..
- cd lbry
- python scripts/set_build.py
- pip install -e .
script:
- pyinstaller -F -n lbrynet lbry/extras/cli.py
- cd dist
- chmod +x lbrynet
- zip -j lbrynet-${OS}.zip lbrynet
- shasum -a 256 -b lbrynet-${OS}.zip
- ./lbrynet --version
after_success:
- aws configure set aws_access_key_id $ARTIFACTS_KEY
- aws configure set aws_secret_access_key $ARTIFACTS_SECRET
- aws configure set region us-east-1
- export S3_PATH="daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})"
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
deploy:
provider: releases
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
file: lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
- <<: *build
name: "Mac"
os: osx
osx_image: xcode8.3
language: shell
env: OS=mac
before_install:
- brew uninstall mercurial
- brew upgrade python || true
- pip3 install --user --upgrade pip virtualenv
- /Users/travis/Library/Python/3.7/bin/virtualenv --clear $HOME/venv
- source $HOME/venv/bin/activate
before_cache:
- brew cleanup
- <<: *build
name: "Windows"
os: windows
language: shell
env:
- OS=windows
- PATH=/c/Python37:/c/Python37/Scripts:/C/Windows/System32/downlevel:$PATH
before_install:
- choco install python --version=3.7.4 --x86
- python -m pip install --upgrade pip
- pip install pywin32
script:
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- cd dist
- 7z a -tzip lbrynet-windows.zip lbrynet.exe
- sha256sum -b lbrynet-windows.zip
- ./lbrynet.exe --version
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f lbry/scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:master
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+1 -1018
View File
File diff suppressed because it is too large. Load diff
+1 -4
View File
@@ -88,9 +88,6 @@ To install on Windows:
> lbry-venv\Scripts\activate
Install packages:
> cd torba
> pip install -e .
> cd ../lbry
> pip install -e .
```
@@ -99,7 +96,7 @@ To install on Windows:
To run the unit tests from the repo directory:
```
python -m unittest discover -s lbry tests.unit
python -m unittest discover tests.unit
```
## Usage
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2019 LBRY Inc
Copyright (c) 2015-2020 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
+1 -1
View File
@@ -1,4 +1,4 @@
include README.md
include CHANGELOG.md
include LICENSE
recursive-include torba *.txt *.py
recursive-include lbry *.txt *.py
+13 -11
View File
@@ -1,18 +1,20 @@
install:
cd torba && pip install -e .
cd lbry && pip install -e .
pip install mypy==0.701
pip install coverage astroid pylint
.PHONY: tools lint test idea
lint:
cd lbry && pylint lbry
cd torba && pylint --rcfile=setup.cfg torba
cd torba && mypy --ignore-missing-imports torba
pylint --rcfile=setup.cfg lbry
#mypy --ignore-missing-imports lbry
test:
cd lbry && tox
cd torba && tox
tox
idea:
mkdir -p .idea
cp -r lbry/scripts/idea/* .idea
cp -r scripts/idea/* .idea
start:
dropdb lbry --if-exists
createdb lbry
lbrynet start node \
--db-url=postgresql:///lbry --workers=0 --console=advanced --no-spv-address-filters \
--lbrycrd-rpc-user=lbry --lbrycrd-rpc-pass=somethingelse \
--lbrycrd-dir=${HOME}/.lbrycrd --data-dir=/tmp/tmp-lbrynet
+11 -7
View File
@@ -1,15 +1,15 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Gitlab CI Badge](https://ci.lbry.tech/lbry/lbry-sdk/badges/master/pipeline.svg)](https://ci.lbry.tech/lbry/lbry-sdk)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/extras/daemon)).
## Installation
@@ -27,6 +27,10 @@ With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Recommended hardware
The minimum hardware for a full node is 16cpus, 92gb of RAM, and 160gb of NVMe storage. The recommended hardware is 32cpus, 128gb of RAM, and 160gb of NVMe storage.
## Running from source
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
@@ -53,4 +57,4 @@ The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/example_daemon_settings.yml).
+5
View File
@@ -0,0 +1,5 @@
FROM ubuntu:20.04
COPY lbrynet /bin
RUN lbrynet --version
ENTRYPOINT ["lbrynet"]
CMD ["start", "node"]
+8
View File
@@ -0,0 +1,8 @@
FROM ubuntu:20.04
RUN apt-get update && \
apt-get install -y wget unzip && \
wget -nv https://build.lbry.io/lbrycrd/block_info_fix_try2/lbrycrd-linux.zip && \
unzip -d /bin lbrycrd-linux.zip && \
rm -rf lbrycrd-linux.zip /var/lib/apt/lists/*
RUN lbrycrdd --version
ENTRYPOINT ["lbrycrdd"]
+41
View File
@@ -0,0 +1,41 @@
version: "3.8"
volumes:
lbrycrd-data:
services:
postgres:
image: postgres:12
environment:
POSTGRES_USER: lbry
POSTGRES_PASSWORD: lbry
lbrycrd:
image: lbry/lbrycrd
build:
context: .
dockerfile: Dockerfile.lbrycrd
volumes:
- lbrycrd-data:/root/.lbrycrd
command: >
-rpcbind=lbrycrd
-rpcallowip=0.0.0.0/0
-rpcuser=lbryuser
-rpcpassword=lbrypass
-zmqpubhashblock=tcp://lbrycrd:29000
lbrynet:
image: lbry/lbrynet:fast_wallet_server_sync
depends_on:
- postgres
- lbrycrd
volumes:
- lbrycrd-data:/lbrycrd
command: >
start
--full-node
--api=0.0.0.0:5279
--db-url=postgresql://lbry:lbry@postgres:5432/lbry
--workers=12
--console=basic
--no-spv-address-filters
--lbrycrd-rpc-host=lbrycrd
--lbrycrd-rpc-user=lbryuser
--lbrycrd-rpc-pass=lbrypass
--lbrycrd-dir=/lbrycrd
+42
View File
@@ -0,0 +1,42 @@
import sys
import os
import re
import logging
import lbry.build_info as build_info_mod
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def _check_and_set(d: dict, key: str, value: str):
try:
d[key]
except KeyError:
raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
d[key] = value
def main():
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
if commit_hash is None:
raise ValueError("Commit hash not found in env vars")
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
docker_tag = os.getenv('DOCKER_TAG')
if docker_tag:
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
_check_and_set(build_info, "BUILD", "docker")
else:
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
with open(build_info_mod.__file__, 'w') as f:
f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
if __name__ == '__main__':
sys.exit(main())
+4985
View File
File diff suppressed because it is too large. Load diff
File renamed without changes.
File renamed without changes.

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 361 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 5.3 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

View File
File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

-7
View File
@@ -1,7 +0,0 @@
.git
.tox
__pycache__
dist
lbry.egg-info
docs
tests
-1
View File
@@ -1 +0,0 @@
/CHANGELOG.md merge=union
-13
View File
@@ -1,13 +0,0 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbry.egg-info
__pycache__
_trial_temp/
/tests/integration/files
-441
View File
@@ -1,441 +0,0 @@
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS,schema
# Add files or directories matching the regex patterns to the
# blacklist. The regex matches against base names, not paths.
# `\.#.*` - add emacs tmp files to the blacklist
ignore-patterns=\.#.*
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=4
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
# extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then re-enable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=
anomalous-backslash-in-string,
arguments-differ,
attribute-defined-outside-init,
bad-continuation,
bare-except,
broad-except,
cell-var-from-loop,
consider-iterating-dictionary,
dangerous-default-value,
duplicate-code,
fixme,
invalid-name,
len-as-condition,
locally-disabled,
logging-not-lazy,
missing-docstring,
no-else-return,
no-init,
no-member,
no-self-use,
protected-access,
redefined-builtin,
redefined-outer-name,
redefined-variable-type,
relative-import,
signature-differs,
super-init-not-called,
too-few-public-methods,
too-many-arguments,
too-many-branches,
too-many-instance-attributes,
too-many-lines,
too-many-locals,
too-many-nested-blocks,
too-many-public-methods,
too-many-return-statements,
too-many-statements,
trailing-newlines,
undefined-loop-variable,
ungrouped-imports,
unnecessary-lambda,
unused-argument,
unused-variable,
wrong-import-order,
wrong-import-position,
deprecated-lambda,
simplifiable-if-statement,
unidiomatic-typecheck,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
useless-return,
assignment-from-none,
stop-iteration-return
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=_$|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[BASIC]
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,input
# Good variable names which should always be accepted, separated by a comma
# allow `d` as its used frequently for deferred callback chains
good-names=i,j,k,ex,Run,_,d
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
[ELIF]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=120
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=leveldb,distutils
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
# ignored-classes=
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=10
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=8
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
-56
View File
@@ -1,56 +0,0 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
Run `lbrynet start` to launch the API server.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Running from source
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
+8
View File
@@ -0,0 +1,8 @@
__version__ = "1.0.0"
from lbry.wallet import Account, Wallet, WalletManager
from lbry.blockchain import Ledger, RegTestLedger, TestNetLedger
from lbry.blockchain import Transaction, Output, Input
from lbry.blockchain import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
from lbry.service import API, Daemon, FullNode, LightClient
from lbry.db.database import Database
from lbry.conf import Config
+6
View File
@@ -0,0 +1,6 @@
from lbry.utils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
BLOBHASH_LENGTH = get_lbry_hash_obj().digest_size * 2
@@ -9,18 +9,20 @@ from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbry.cryptoutils import backend, get_lbry_hash_obj
from lbry.utils import get_lbry_hash_obj
from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbry.blob import MAX_BLOB_SIZE, blobhash_length
from lbry.blob import MAX_BLOB_SIZE, BLOBHASH_LENGTH
from lbry.blob.blob_info import BlobInfo
from lbry.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
_hexmatch = re.compile("^[a-f,0-9]+$")
HEXMATCH = re.compile("^[a-f,0-9]+$")
BACKEND = default_backend()
def is_valid_blobhash(blobhash: str) -> bool:
@@ -31,11 +33,11 @@ def is_valid_blobhash(blobhash: str) -> bool:
@return: True/False
"""
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
return len(blobhash) == BLOBHASH_LENGTH and HEXMATCH.match(blobhash)
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
padder = PKCS7(AES.block_size).padder()
encryptor = cipher.encryptor()
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
@@ -47,7 +49,7 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
if len(data) != length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
@@ -142,7 +144,7 @@ class AbstractBlob:
def close(self):
while self.writers:
peer, writer = self.writers.popitem()
_, writer = self.writers.popitem()
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
writer.finished.cancel()
while self.readers:
@@ -163,7 +165,10 @@ class AbstractBlob:
if not self.is_readable():
raise OSError('blob files cannot be read')
with self.reader_context() as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
try:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
except (ConnectionError, BrokenPipeError, RuntimeError, OSError, AttributeError):
return -1
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
File renamed without changes.
@@ -10,7 +10,11 @@ from lbry.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.dht.protocol.data_store import DictDataStore
from lbry.extras.daemon.storage import SQLiteStorage
class SQLiteStorage:
pass
log = logging.getLogger(__name__)
@@ -3,7 +3,7 @@ import logging
import asyncio
from io import BytesIO
from lbry.error import InvalidBlobHashError, InvalidDataError
from lbry.cryptoutils import get_lbry_hash_obj
from lbry.utils import get_lbry_hash_obj
log = logging.getLogger(__name__)
File renamed without changes.
@@ -95,7 +95,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]:
async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: # pylint: disable=too-many-return-statements
"""
:return: download success (bool), connected protocol (BlobExchangeClientProtocol)
"""
@@ -213,11 +213,11 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
def connection_lost(self, exc):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(exc),
str(type(exc)))
self.close()
@@ -27,7 +27,7 @@ class BlobDownloader:
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.failures: typing.Dict['KademliaPeer', int] = {}
self.connection_failures: typing.List['KademliaPeer'] = []
self.connection_failures: typing.Set['KademliaPeer'] = set()
self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
self.is_running = asyncio.Event(loop=self.loop)
@@ -48,7 +48,7 @@ class BlobDownloader:
connection_id=connection_id, connection_manager=self.blob_manager.connection_manager
)
if not bytes_received and not protocol and peer not in self.connection_failures:
self.connection_failures.append(peer)
self.connection_failures.add(peer)
if not protocol and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
@@ -87,32 +87,42 @@ class BlobDownloader:
if blob.get_is_verified():
return blob
self.is_running.set()
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
try:
while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set()
batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
while not self.peer_queue.empty():
batch.update(self.peer_queue.get_nowait())
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active, %s connections",
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
)
re_add: typing.Set['KademliaPeer'] = set()
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if peer in self.ignored or peer in self.active_connections:
if peer in self.ignored:
continue
if peer in tried_for_this_blob:
continue
if peer in self.active_connections:
if peer not in re_add:
re_add.add(peer)
continue
if not self.should_race_continue(blob):
break
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
just_probe = len(self.connections) == 0
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
tried_for_this_blob.add(peer)
if not re_add:
self.peer_queue.put_nowait(list(batch))
await self.new_peer_or_finished()
self.cleanup_active()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
blob.close()
if self.loop.is_running():
self.loop.call_soon(self.cleanup_active)
def close(self):
self.connection_failures.clear()
@@ -46,7 +46,7 @@ class BlobAvailabilityRequest(BlobMessage):
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
**kwargs) -> None:
assert len(requested_blobs)
assert len(requested_blobs) > 0
self.requested_blobs = requested_blobs
self.lbrycrd_address = lbrycrd_address
@@ -134,9 +134,9 @@ class BlobErrorResponse(BlobMessage):
}
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest, # pylint: disable=invalid-name
BlobPaymentAddressRequest]
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse, # pylint: disable=invalid-name
BlobErrorResponse, BlobPaymentAddressResponse]
@@ -157,10 +157,10 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
except ValueError:
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
@@ -179,7 +179,7 @@ class BlobRequest:
return d
def _get_request(self, request_type: blob_request_types):
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
request = tuple(filter(lambda r: type(r) == request_type, self.requests)) # pylint: disable=unidiomatic-typecheck
if request:
return request[0]
@@ -235,7 +235,7 @@ class BlobResponse:
return d
def _get_response(self, response_type: blob_response_types):
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
response = tuple(filter(lambda r: type(r) == response_type, self.responses)) # pylint: disable=unidiomatic-typecheck
if response:
return response[0]
@@ -280,4 +280,3 @@ class BlobResponse:
if response_type.key in response
])
return cls(requests, extra)
@@ -96,22 +96,30 @@ class BlobServerProtocol(asyncio.Protocol):
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
bh = blob.blob_hash[:8]
log.debug("send %s to %s:%i", bh, peer_address, peer_port)
blob_hash = blob.blob_hash[:8]
log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
self.started_transfer.set()
try:
sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
log.info("sent %s (%i bytes) to %s:%i", bh, sent, peer_address, peer_port)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError, asyncio.TimeoutError) as err:
if isinstance(err, asyncio.TimeoutError):
log.debug("timed out sending blob %s to %s", bh, peer_address)
if sent and sent > 0:
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
else:
log.debug("stopped sending %s to %s:%i", bh, peer_address, peer_port)
self.close()
log.debug("stopped sending %s to %s:%i", blob_hash, peer_address, peer_port)
return
except (OSError, ValueError, asyncio.TimeoutError) as err:
if isinstance(err, asyncio.TimeoutError):
log.debug("timed out sending blob %s to %s", blob_hash, peer_address)
else:
log.warning("could not read blob %s to send %s:%i", blob_hash, peer_address, peer_port)
self.close()
return
finally:
self.transfer_finished.set()
if responses:
else:
log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port)
if responses and not self.transport.is_closing():
self.send_response(responses)
def data_received(self, data):
@@ -122,7 +130,7 @@ class BlobServerProtocol(asyncio.Protocol):
return
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
_, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
+4
View File
@@ -0,0 +1,4 @@
from .ledger import Ledger, RegTestLedger, TestNetLedger, ledger_class_from_name
from .transaction import Transaction, Output, Input
from .bcd_data_stream import BCDataStream
from .dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
@@ -4,8 +4,11 @@ from io import BytesIO
class BCDataStream:
def __init__(self, data=None):
self.data = BytesIO(data)
def __init__(self, data=None, fp=None):
self.data = fp or BytesIO(data)
def tell(self):
return self.data.tell()
def reset(self):
self.data.seek(0)
+60
View File
@@ -0,0 +1,60 @@
import struct
from typing import NamedTuple, List
from chiabip158 import PyBIP158 # pylint: disable=no-name-in-module
from lbry.crypto.hash import double_sha256
from lbry.blockchain.transaction import Transaction
from lbry.blockchain.bcd_data_stream import BCDataStream
ZERO_BLOCK = bytes((0,)*32)
def create_address_filter(address_hashes: List[bytes]) -> bytes:
return bytes(PyBIP158([bytearray(a) for a in address_hashes]).GetEncoded())
def get_address_filter(address_filter: bytes) -> PyBIP158:
return PyBIP158(bytearray(address_filter))
class Block(NamedTuple):
height: int
version: int
file_number: int
block_hash: bytes
prev_block_hash: bytes
merkle_root: bytes
claim_trie_root: bytes
timestamp: int
bits: int
nonce: int
txs: List[Transaction]
@staticmethod
def from_data_stream(stream: BCDataStream, height: int, file_number: int):
header = stream.data.read(112)
version, = struct.unpack('<I', header[:4])
timestamp, bits, nonce = struct.unpack('<III', header[100:112])
tx_count = stream.read_compact_size()
return Block(
height=height,
version=version,
file_number=file_number,
block_hash=double_sha256(header),
prev_block_hash=header[4:36],
merkle_root=header[36:68],
claim_trie_root=header[68:100][::-1],
timestamp=timestamp,
bits=bits,
nonce=nonce,
txs=[
Transaction(height=height, position=i, timestamp=timestamp).deserialize(stream)
for i in range(tx_count)
]
)
@property
def is_first_block(self):
return self.prev_block_hash == ZERO_BLOCK
+743
View File
@@ -0,0 +1,743 @@
HASHES = {
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
}
@@ -1,6 +1,6 @@
import struct
import binascii
from torba.client.hash import double_sha256
from lbry.crypto.hash import double_sha256
class InvalidProofError(Exception):
+245
View File
@@ -0,0 +1,245 @@
import os.path
import asyncio
import sqlite3
from typing import List, Optional
from concurrent.futures import ThreadPoolExecutor
from lbry.schema.url import normalize_name
from .bcd_data_stream import BCDataStream
FILES = [
'claims',
'block_index',
]
def make_short_url(r):
try:
# TODO: we describe it as normalized but the old SDK didnt do that
name = r["name"].decode().replace("\x00", "")
return f'{name}#{r["shortestID"] or r["claimID"][::-1].hex()[0]}'
except UnicodeDecodeError:
# print(f'failed making short url due to name parse error for claim_id: {r["claimID"][::-1].hex()}')
return "INVALID NAME"
class FindShortestID:
__slots__ = 'short_id', 'new_id'
def __init__(self):
self.short_id = ''
self.new_id = None
def step(self, other_id, new_id):
other_id = other_id[::-1].hex()
if self.new_id is None:
self.new_id = new_id[::-1].hex()
for i in range(len(self.new_id)):
if other_id[i] != self.new_id[i]:
if i > len(self.short_id)-1:
self.short_id = self.new_id[:i+1]
break
def finalize(self):
return self.short_id
class BlockchainDB:
def __init__(self, directory: str):
self.directory = directory
self.connection: Optional[sqlite3.Connection] = None
self.executor: Optional[ThreadPoolExecutor] = None
async def run_in_executor(self, *args):
return await asyncio.get_running_loop().run_in_executor(self.executor, *args)
def sync_open(self):
self.connection = sqlite3.connect(
os.path.join(self.directory, FILES[0]+'.sqlite'),
timeout=60.0 * 5
)
for file in FILES[1:]:
self.connection.execute(
f"ATTACH DATABASE '{os.path.join(self.directory, file+'.sqlite')}' AS {file}"
)
self.connection.create_aggregate("find_shortest_id", 2, FindShortestID)
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_originalheight ON claim (originalheight);")
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_updateheight ON claim (updateheight);")
self.connection.execute("create index IF NOT EXISTS support_blockheight on support (blockheight);")
self.connection.row_factory = sqlite3.Row
async def open(self):
assert self.executor is None, "Database is already open."
self.executor = ThreadPoolExecutor(max_workers=1)
return await self.run_in_executor(self.sync_open)
def sync_close(self):
self.connection.close()
self.connection = None
async def close(self):
if self.executor is not None:
if self.connection is not None:
await self.run_in_executor(self.sync_close)
self.executor.shutdown()
self.executor = None
async def commit(self):
await self.run_in_executor(self.connection.commit)
def sync_execute(self, sql: str, *args):
return self.connection.execute(sql, *args)
async def execute(self, sql: str, *args):
return await self.run_in_executor(self.sync_execute, sql, *args)
def sync_execute_fetchall(self, sql: str, *args) -> List[dict]:
return self.connection.execute(sql, *args).fetchall()
async def execute_fetchall(self, sql: str, *args) -> List[dict]:
return await self.run_in_executor(self.sync_execute_fetchall, sql, *args)
def sync_get_best_height(self) -> int:
sql = "SELECT MAX(height) FROM block_info"
return self.connection.execute(sql).fetchone()[0]
async def get_best_height(self) -> int:
return await self.run_in_executor(self.sync_get_best_height)
def sync_get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
sql = """
SELECT
file as file_number,
COUNT(hash) as blocks,
SUM(txcount) as txs,
MAX(height) as best_height,
MIN(height) as start_height
FROM block_info
WHERE status&1 AND status&4 AND NOT status&32 AND NOT status&64
"""
args = ()
if file_number is not None and start_height is not None:
sql += "AND file = ? AND height >= ?"
args = (file_number, start_height)
return [dict(r) for r in self.sync_execute_fetchall(sql + " GROUP BY file ORDER BY file ASC;", args)]
async def get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
return await self.run_in_executor(
self.sync_get_block_files, file_number, start_height
)
def sync_get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
return [dict(r) for r in self.sync_execute_fetchall(
"""
SELECT datapos as data_offset, height, hash as block_hash, txCount as txs
FROM block_info
WHERE file = ? AND height >= ?
AND status&1 AND status&4 AND NOT status&32 AND NOT status&64
ORDER BY datapos ASC;
""", (block_file, start_height)
)]
async def get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
return await self.run_in_executor(self.sync_get_blocks_in_file, block_file, start_height)
def sync_get_claim_support_txo_hashes(self, at_height: int) -> set:
return {
r['txID'] + BCDataStream.uint32.pack(r['txN'])
for r in self.connection.execute(
"""
SELECT txID, txN FROM claim WHERE updateHeight = ?
UNION
SELECT txID, txN FROM support WHERE blockHeight = ?
""", (at_height, at_height)
).fetchall()
}
def sync_get_takeover_count(self, start_height: int, end_height: int) -> int:
sql = """
SELECT COUNT(*) FROM claim WHERE name IN (
SELECT name FROM takeover
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
)
""", (start_height, end_height)
return self.connection.execute(*sql).fetchone()[0]
async def get_takeover_count(self, start_height: int, end_height: int) -> int:
return await self.run_in_executor(self.sync_get_takeover_count, start_height, end_height)
def sync_get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
sql = """
SELECT name, claimID, MAX(height) AS height FROM takeover
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
GROUP BY name
""", (start_height, end_height)
return [{
'normalized': normalize_name(r['name'].decode()),
'claim_hash': r['claimID'],
'height': r['height']
} for r in self.sync_execute_fetchall(*sql)]
async def get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
return await self.run_in_executor(self.sync_get_takeovers, start_height, end_height)
def sync_get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
sql = "SELECT COUNT(*) FROM claim WHERE originalHeight BETWEEN ? AND ?"
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
async def get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
return await self.run_in_executor(self.sync_get_claim_metadata_count, start_height, end_height)
def sync_get_claim_metadata(self, claim_hashes) -> List[dict]:
sql = f"""
SELECT
name, claimID, activationHeight, expirationHeight, originalHeight,
(SELECT
CASE WHEN takeover.claimID = claim.claimID THEN takeover.height END
FROM takeover WHERE takeover.name = claim.nodename
ORDER BY height DESC LIMIT 1
) AS takeoverHeight,
(SELECT find_shortest_id(c.claimid, claim.claimid) FROM claim AS c
WHERE
c.nodename = claim.nodename AND
c.originalheight <= claim.originalheight AND
c.claimid != claim.claimid
) AS shortestID
FROM claim
WHERE claimID IN ({','.join(['?' for _ in claim_hashes])})
ORDER BY claimID
""", claim_hashes
return [{
"name": r["name"],
"claim_hash": r["claimID"],
"activation_height": r["activationHeight"],
"expiration_height": r["expirationHeight"],
"takeover_height": r["takeoverHeight"],
"creation_height": r["originalHeight"],
"short_url": make_short_url(r),
} for r in self.sync_execute_fetchall(*sql)]
async def get_claim_metadata(self, start_height: int, end_height: int) -> List[dict]:
return await self.run_in_executor(self.sync_get_claim_metadata, start_height, end_height)
def sync_get_support_metadata_count(self, start_height: int, end_height: int) -> int:
sql = "SELECT COUNT(*) FROM support WHERE blockHeight BETWEEN ? AND ?"
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
async def get_support_metadata_count(self, start_height: int, end_height: int) -> int:
return await self.run_in_executor(self.sync_get_support_metadata_count, start_height, end_height)
def sync_get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
sql = """
SELECT name, txid, txn, activationHeight, expirationHeight
FROM support WHERE blockHeight BETWEEN ? AND ?
""", (start_height, end_height)
return [{
"name": r['name'],
"txo_hash_pk": r['txID'] + BCDataStream.uint32.pack(r['txN']),
"activation_height": r['activationHeight'],
"expiration_height": r['expirationHeight'],
} for r in self.sync_execute_fetchall(*sql)]
async def get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
return await self.run_in_executor(self.sync_get_support_metadata, start_height, end_height)
@@ -1,10 +1,19 @@
import re
import textwrap
from torba.client.util import coins_to_satoshis, satoshis_to_coins
from decimal import Decimal
from lbry.constants import COIN
def lbc_to_dewies(lbc: str) -> int:
try:
return coins_to_satoshis(lbc)
if not isinstance(lbc, str):
raise ValueError("{coins} must be a string")
result = re.search(r'^(\d{1,10})\.(\d{1,8})$', lbc)
if result is not None:
whole, fractional = result.groups()
return int(whole + fractional.ljust(8, "0"))
raise ValueError(f"'{lbc}' is not a valid coin decimal")
except ValueError:
raise ValueError(textwrap.dedent(
f"""
@@ -30,13 +39,17 @@ def lbc_to_dewies(lbc: str) -> int:
def dewies_to_lbc(dewies) -> str:
return satoshis_to_coins(dewies)
coins = '{:.8f}'.format(dewies / COIN).rstrip('0')
if coins.endswith('.'):
return coins+'0'
else:
return coins
def dict_values_to_lbc(d):
lbc_dict = {}
for key, value in d.items():
if isinstance(value, int):
if isinstance(value, (int, Decimal)):
lbc_dict[key] = dewies_to_lbc(value)
elif isinstance(value, dict):
lbc_dict[key] = dict_values_to_lbc(value)
+63
View File
@@ -0,0 +1,63 @@
from binascii import hexlify, unhexlify
from lbry.constants import NULL_HASH32
class TXRef:
__slots__ = '_id', '_hash'
def __init__(self):
self._id = None
self._hash = None
@property
def id(self):
return self._id
@property
def hash(self):
return self._hash
@property
def height(self):
return -1
@property
def is_null(self):
return self.hash == NULL_HASH32
class TXRefImmutable(TXRef):
__slots__ = ('_height', '_timestamp')
def __init__(self):
super().__init__()
self._height = -1
self._timestamp = -1
@classmethod
def from_hash(cls, tx_hash: bytes, height: int, timestamp: int) -> 'TXRefImmutable':
ref = cls()
ref._hash = tx_hash
ref._id = hexlify(tx_hash[::-1]).decode()
ref._height = height
ref._timestamp = timestamp
return ref
@classmethod
def from_id(cls, tx_id: str, height: int, timestamp: int) -> 'TXRefImmutable':
ref = cls()
ref._id = tx_id
ref._hash = unhexlify(tx_id)[::-1]
ref._height = height
ref._timestamp = timestamp
return ref
@property
def height(self):
return self._height
@property
def timestamp(self):
return self._timestamp
+370
View File
@@ -0,0 +1,370 @@
import base64
import os
import struct
import asyncio
import logging
import zlib
from datetime import date
from concurrent.futures.thread import ThreadPoolExecutor
from io import BytesIO
from typing import Optional, Iterator, Tuple, Callable
from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.blockchain.util import ArithUint256
from .checkpoints import HASHES
log = logging.getLogger(__name__)
class InvalidHeader(Exception):
def __init__(self, height, message):
super().__init__(message)
self.message = message
self.height = height
class Headers:
header_size = 112
chunk_size = 10**16
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
target_timespan = 150
checkpoints = HASHES
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
timestamp_average_offset = 160.6855883050695 # calculated at 733447
validate_difficulty: bool = True
def __init__(self, path) -> None:
if path == ':memory:':
self.io = BytesIO()
self.path = path
self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.executor = ThreadPoolExecutor(1)
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self):
if not self.executor:
self.executor = ThreadPoolExecutor(1)
if self.path != ':memory:':
if not os.path.exists(self.path):
self.io = open(self.path, 'w+b')
else:
self.io = open(self.path, 'r+b')
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
if bytes_size % self.header_size:
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
await self.repair()
else:
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
await self.repair(start_height=max_checkpointed_height)
await self.ensure_checkpointed_size()
await self.get_all_missing_headers()
async def close(self):
if self.executor:
self.executor.shutdown()
self.executor = None
self.io.close()
@staticmethod
def serialize(header):
return b''.join([
struct.pack('<I', header['version']),
unhexlify(header['prev_block_hash'])[::-1],
unhexlify(header['merkle_root'])[::-1],
unhexlify(header['claim_trie_root'])[::-1],
struct.pack('<III', header['timestamp'], header['bits'], header['nonce'])
])
@staticmethod
def deserialize(height, header):
version, = struct.unpack('<I', header[:4])
timestamp, bits, nonce = struct.unpack('<III', header[100:112])
return {
'version': version,
'prev_block_hash': hexlify(header[4:36][::-1]),
'merkle_root': hexlify(header[36:68][::-1]),
'claim_trie_root': hexlify(header[68:100][::-1]),
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
'block_height': height,
}
def get_next_chunk_target(self, chunk: int) -> ArithUint256:
return ArithUint256(self.max_target)
def get_next_block_target(self, max_target: ArithUint256, previous: Optional[dict],
current: Optional[dict]) -> ArithUint256:
# https://github.com/lbryio/lbrycrd/blob/master/src/lbry.cpp
if previous is None and current is None:
return max_target
if previous is None:
previous = current
actual_timespan = current['timestamp'] - previous['timestamp']
modulated_timespan = self.target_timespan + int((actual_timespan - self.target_timespan) / 8)
minimum_timespan = self.target_timespan - int(self.target_timespan / 8) # 150 - 18 = 132
maximum_timespan = self.target_timespan + int(self.target_timespan / 2) # 150 + 75 = 225
clamped_timespan = max(minimum_timespan, min(modulated_timespan, maximum_timespan))
target = ArithUint256.from_compact(current['bits'])
new_target = min(max_target, (target * clamped_timespan) / self.target_timespan)
return new_target
def __len__(self) -> int:
return self._size
def __bool__(self):
return True
async def get(self, height) -> dict:
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
try:
return self.deserialize(height, await self.get_raw_header(height))
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height):
if height <= 0:
return
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_date(self, height):
return date.fromtimestamp(self.estimated_timestamp(height))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
def _read(self, height, count=1):
self.io.seek(height * self.header_size, os.SEEK_SET)
return self.io.read(self.header_size * count)
def chunk_hash(self, start, count):
self.io.seek(start * self.header_size, os.SEEK_SET)
return self.hash_header(self.io.read(count * self.header_size)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
if self.height < max_checkpointed_height:
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if await self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
async def fetch_chunk(self, height):
log.info("on-demand fetching height %s", height)
start = (height // 1000) * 1000
headers = await self.chunk_getter(start) # pylint: disable=not-callable
chunk = (
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
elif start not in self.checkpoints:
return # todo: fixme
raise Exception(
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
async def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
def _has_header(height):
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
def _io_checkall():
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
@property
def height(self) -> int:
return len(self)-1
@property
def bytes_size(self):
return len(self) * self.header_size
async def hash(self, height=None) -> bytes:
return self.hash_header(
await self.get_raw_header(height if height is not None else self.height)
)
@staticmethod
def hash_header(header: bytes) -> bytes:
if header is None:
return b'0' * 64
return hexlify(double_sha256(header)[::-1])
async def connect(self, start: int, headers: bytes) -> int:
added = 0
bail = False
for height, chunk in self._iterate_chunks(start, headers):
try:
# validate_chunk() is CPU bound and reads previous chunks from file system
await self.validate_chunk(height, chunk)
except InvalidHeader as e:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
if chunk:
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
if bail:
break
return added
def _write(self, height, verified_chunk):
self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(verified_chunk) // self.header_size
# self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = max(self._size or 0, self.io.tell() // self.header_size)
return written
async def validate_chunk(self, height, chunk):
previous_hash, previous_header, previous_previous_header = None, None, None
if height > 0:
raw = await self.get_raw_header(height-1)
previous_header = self.deserialize(height-1, raw)
previous_hash = self.hash_header(raw)
if height > 1:
previous_previous_header = await self.get(height-2)
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
for current_hash, current_header in self._iterate_headers(height, chunk):
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
self.validate_header(height, current_hash, current_header, previous_hash, block_target)
previous_previous_header = previous_header
previous_header = current_header
previous_hash = current_hash
def validate_header(self, height: int, current_hash: bytes,
header: dict, previous_hash: bytes, target: ArithUint256):
if previous_hash is None:
if self.genesis_hash is not None and self.genesis_hash != current_hash:
raise InvalidHeader(
height, f"genesis header doesn't match: {current_hash.decode()} "
f"vs expected {self.genesis_hash.decode()}")
return
if header['prev_block_hash'] != previous_hash:
raise InvalidHeader(
height, "previous hash mismatch: {} vs expected {}".format(
header['prev_block_hash'].decode(), previous_hash.decode())
)
if self.validate_difficulty:
if header['bits'] != target.compact:
raise InvalidHeader(
height, "bits mismatch: {} vs expected {}".format(
header['bits'], target.compact)
)
proof_of_work = self.get_proof_of_work(current_hash)
if proof_of_work > target:
raise InvalidHeader(
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
)
async def repair(self, start_height=0):
previous_header_hash = fail = None
batch_size = 36
for height in range(start_height, self.height, batch_size):
headers = await asyncio.get_running_loop().run_in_executor(
self.executor, self._read, height, batch_size
)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(height, headers):
height = header['block_height']
if previous_header_hash:
if header['prev_block_hash'] != previous_header_hash:
fail = True
elif height == 0:
if header_hash != self.genesis_hash:
fail = True
else:
# for sanity and clarity, since it is the only way we can end up here
assert start_height > 0 and height == start_height
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
def __truncate(at_height):
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
previous_header_hash = header_hash
@classmethod
def get_proof_of_work(cls, header_hash: bytes):
return ArithUint256(int(b'0x' + cls.header_hash_to_pow_hash(header_hash), 16))
def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]:
assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}"
start = 0
end = (self.chunk_size - height % self.chunk_size) * self.header_size
while start < end:
yield height + (start // self.header_size), headers[start:end]
start = end
end = min(len(headers), end + self.chunk_size * self.header_size)
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
assert len(headers) % self.header_size == 0, len(headers)
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]
yield self.hash_header(header), self.deserialize(height+idx, header)
@staticmethod
def header_hash_to_pow_hash(header_hash: bytes):
header_hash_bytes = unhexlify(header_hash)[::-1]
h = sha512(header_hash_bytes)
pow_hash = double_sha256(
ripemd160(h[:len(h) // 2]) +
ripemd160(h[len(h) // 2:])
)
return hexlify(pow_hash[::-1])
class UnvalidatedHeaders(Headers):
validate_difficulty = False
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
checkpoints = {}
+346
View File
@@ -0,0 +1,346 @@
import os
import struct
import shutil
import asyncio
import logging
import zipfile
import tempfile
import urllib.request
from typing import Optional
from binascii import hexlify
import aiohttp
import zmq
import zmq.asyncio
from lbry.conf import Config
from lbry.event import EventController
from lbry.error import LbrycrdEventSubscriptionError, LbrycrdUnauthorizedError
from .database import BlockchainDB
from .ledger import Ledger, RegTestLedger
log = logging.getLogger(__name__)
DOWNLOAD_URL = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
)
class Process(asyncio.SubprocessProtocol):
IGNORE_OUTPUT = [
b'keypool keep',
b'keypool reserve',
b'keypool return',
]
def __init__(self):
self.ready = asyncio.Event()
self.stopped = asyncio.Event()
def pipe_data_received(self, fd, data):
if not any(ignore in data for ignore in self.IGNORE_OUTPUT):
if b'Error:' in data:
log.error(data.decode())
else:
for line in data.decode().splitlines():
log.debug(line.rstrip())
if b'Error:' in data:
self.ready.set()
raise SystemError(data.decode())
if b'Done loading' in data:
self.ready.set()
def process_exited(self):
self.stopped.set()
self.ready.set()
ZMQ_BLOCK_EVENT = 'pubhashblock'
class Lbrycrd:
def __init__(self, ledger: Ledger):
self.ledger, self.conf = ledger, ledger.conf
self.data_dir = self.actual_data_dir = ledger.conf.lbrycrd_dir
if self.is_regtest:
self.actual_data_dir = os.path.join(self.data_dir, 'regtest')
self.blocks_dir = os.path.join(self.actual_data_dir, 'blocks')
self.bin_dir = os.path.join(os.path.dirname(__file__), 'bin')
self.daemon_bin = os.path.join(self.bin_dir, 'lbrycrdd')
self.cli_bin = os.path.join(self.bin_dir, 'lbrycrd-cli')
self.protocol = None
self.transport = None
self.subscribed = False
self.subscription: Optional[asyncio.Task] = None
self.default_generate_address = None
self._on_block_hash_controller = EventController()
self.on_block_hash = self._on_block_hash_controller.stream
self.on_block_hash.listen(lambda e: log.info('%s %s', hexlify(e['hash']), e['msg']))
self._on_tx_hash_controller = EventController()
self.on_tx_hash = self._on_tx_hash_controller.stream
self.db = BlockchainDB(self.actual_data_dir)
self._session: Optional[aiohttp.ClientSession] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
@property
def session(self) -> aiohttp.ClientSession:
if self._session is None:
self._session = aiohttp.ClientSession()
return self._session
@classmethod
def temp_regtest(cls):
return cls(RegTestLedger(
Config.with_same_dir(tempfile.mkdtemp()).set(
blockchain="regtest",
lbrycrd_rpc_port=9245 + 2, # avoid conflict with default rpc port
lbrycrd_peer_port=9246 + 2, # avoid conflict with default peer port
lbrycrd_zmq="tcp://127.0.0.1:29002"
)
))
@staticmethod
def get_block_file_name(block_file_number):
return f'blk{block_file_number:05}.dat'
def get_block_file_path(self, block_file_number):
return os.path.join(
self.actual_data_dir, 'blocks',
self.get_block_file_name(block_file_number)
)
@property
def is_regtest(self):
return isinstance(self.ledger, RegTestLedger)
@property
def rpc_url(self):
return (
f'http://{self.conf.lbrycrd_rpc_user}:{self.conf.lbrycrd_rpc_pass}'
f'@{self.conf.lbrycrd_rpc_host}:{self.conf.lbrycrd_rpc_port}/'
)
@property
def exists(self):
return (
os.path.exists(self.cli_bin) and
os.path.exists(self.daemon_bin)
)
async def download(self):
downloaded_file = os.path.join(
self.bin_dir, DOWNLOAD_URL[DOWNLOAD_URL.rfind('/')+1:]
)
if not os.path.exists(self.bin_dir):
os.mkdir(self.bin_dir)
if not os.path.exists(downloaded_file):
log.info('Downloading: %s', DOWNLOAD_URL)
async with aiohttp.ClientSession() as session:
async with session.get(DOWNLOAD_URL) as response:
with open(downloaded_file, 'wb') as out_file:
while True:
chunk = await response.content.read(4096)
if not chunk:
break
out_file.write(chunk)
with urllib.request.urlopen(DOWNLOAD_URL) as response:
with open(downloaded_file, 'wb') as out_file:
shutil.copyfileobj(response, out_file)
log.info('Extracting: %s', downloaded_file)
with zipfile.ZipFile(downloaded_file) as dotzip:
dotzip.extractall(self.bin_dir)
# zipfile bug https://bugs.python.org/issue15795
os.chmod(self.cli_bin, 0o755)
os.chmod(self.daemon_bin, 0o755)
return self.exists
async def ensure(self):
return self.exists or await self.download()
def get_start_command(self, *args):
if self.is_regtest:
args += ('-regtest',)
if self.conf.lbrycrd_zmq:
args += (
f'-zmqpubhashblock={self.conf.lbrycrd_zmq}',
f'-zmqpubhashtx={self.conf.lbrycrd_zmq}',
)
return (
self.daemon_bin,
f'-datadir={self.data_dir}',
f'-port={self.conf.lbrycrd_peer_port}',
f'-rpcport={self.conf.lbrycrd_rpc_port}',
f'-rpcuser={self.conf.lbrycrd_rpc_user}',
f'-rpcpassword={self.conf.lbrycrd_rpc_pass}',
'-server', '-printtoconsole',
*args
)
async def open(self):
await self.db.open()
async def close(self):
await self.db.close()
await self.close_session()
async def close_session(self):
if self._session is not None:
await self._session.close()
self._session = None
async def start(self, *args):
loop = asyncio.get_running_loop()
command = self.get_start_command(*args)
log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(Process, *command)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
await self.open()
async def stop(self, cleanup=True):
try:
await self.close()
self.transport.terminate()
await self.protocol.stopped.wait()
assert self.transport.get_returncode() == 0, "lbrycrd daemon exit with error"
self.transport.close()
finally:
if cleanup:
await self.cleanup()
async def cleanup(self):
await asyncio.get_running_loop().run_in_executor(
None, shutil.rmtree, self.data_dir, True
)
async def ensure_subscribable(self):
zmq_notifications = await self.get_zmq_notifications()
subs = {e['type']: e['address'] for e in zmq_notifications}
if ZMQ_BLOCK_EVENT not in subs:
raise LbrycrdEventSubscriptionError(ZMQ_BLOCK_EVENT)
if not self.conf.lbrycrd_zmq:
self.conf.lbrycrd_zmq = subs[ZMQ_BLOCK_EVENT]
async def subscribe(self):
if not self.subscribed:
self.subscribed = True
ctx = zmq.asyncio.Context.instance()
sock = ctx.socket(zmq.SUB) # pylint: disable=no-member
sock.connect(self.conf.lbrycrd_zmq)
sock.subscribe("hashblock")
sock.subscribe("hashtx")
self.subscription = asyncio.create_task(self.subscription_handler(sock))
async def subscription_handler(self, sock):
try:
while self.subscribed:
msg = await sock.recv_multipart()
if msg[0] == b'hashtx':
await self._on_tx_hash_controller.add({
'hash': msg[1],
'msg': struct.unpack('<I', msg[2])[0]
})
elif msg[0] == b'hashblock':
await self._on_block_hash_controller.add({
'hash': msg[1],
'msg': struct.unpack('<I', msg[2])[0]
})
except asyncio.CancelledError:
sock.close()
raise
def unsubscribe(self):
if self.subscribed:
self.subscribed = False
self.subscription.cancel()
self.subscription = None
def sync_run(self, coro):
if self._loop is None:
try:
self._loop = asyncio.get_event_loop()
except RuntimeError:
self._loop = asyncio.new_event_loop()
return self._loop.run_until_complete(coro)
async def rpc(self, method, params=None):
if self._session is not None and self._session.closed:
raise Exception("session is closed! RPC attempted during shutting down.")
message = {
"jsonrpc": "1.0",
"id": "1",
"method": method,
"params": params or []
}
async with self.session.post(self.rpc_url, json=message) as resp:
if resp.status == 401:
raise LbrycrdUnauthorizedError()
try:
result = await resp.json()
except aiohttp.ContentTypeError as e:
raise Exception(await resp.text()) from e
if not result['error']:
return result['result']
else:
result['error'].update(method=method, params=params)
raise Exception(result['error'])
async def get_zmq_notifications(self):
return await self.rpc("getzmqnotifications")
async def generate(self, blocks):
if self.default_generate_address is None:
self.default_generate_address = await self.get_new_address()
return await self.generate_to_address(blocks, self.default_generate_address)
async def get_new_address(self):
return await self.rpc("getnewaddress")
async def generate_to_address(self, blocks, address):
return await self.rpc("generatetoaddress", [blocks, address])
async def send_to_address(self, address, amount):
return await self.rpc("sendtoaddress", [address, amount])
async def get_block(self, block_hash):
return await self.rpc("getblock", [block_hash])
async def get_raw_mempool(self):
return await self.rpc("getrawmempool")
async def get_raw_transaction(self, txid):
return await self.rpc("getrawtransaction", [txid])
async def fund_raw_transaction(self, tx):
return await self.rpc("fundrawtransaction", [tx])
async def sign_raw_transaction_with_wallet(self, tx):
return await self.rpc("signrawtransactionwithwallet", [tx])
async def send_raw_transaction(self, tx):
return await self.rpc("sendrawtransaction", [tx])
async def claim_name(self, name, data, amount):
return await self.rpc("claimname", [name, data, amount])
async def update_claim(self, txid, data, amount):
return await self.rpc("updateclaim", [txid, data, amount])
async def abandon_claim(self, txid, address):
return await self.rpc("abandonclaim", [txid, address])
async def support_claim(self, name, claim_id, amount, value="", istip=False):
return await self.rpc("supportclaim", [name, claim_id, amount, value, istip])
async def abandon_support(self, txid, address):
return await self.rpc("abandonsupport", [txid, address])
+179
View File
@@ -0,0 +1,179 @@
from binascii import unhexlify
from string import hexdigits
from typing import TYPE_CHECKING, Type
from lbry.crypto.hash import hash160, double_sha256
from lbry.crypto.base58 import Base58
from lbry.schema.url import URL
from .header import Headers, UnvalidatedHeaders
from .checkpoints import HASHES
from .dewies import lbc_to_dewies
if TYPE_CHECKING:
from lbry.conf import Config
class Ledger:
name = 'LBRY Credits'
symbol = 'LBC'
network_name = 'mainnet'
headers_class = Headers
secret_prefix = bytes((0x1c,))
pubkey_address_prefix = bytes((0x55,))
script_address_prefix = bytes((0x7a,))
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
genesis_bits = 0x1f00ffff
target_timespan = 150
fee_per_byte = 50
fee_per_name_char = 200000
checkpoints = HASHES
def __init__(self, conf: 'Config'):
self.conf = conf
self.coin_selection_strategy = None
@classmethod
def get_id(cls):
return '{}_{}'.format(cls.symbol.lower(), cls.network_name.lower())
@staticmethod
def address_to_hash160(address) -> bytes:
return Base58.decode(address)[1:21]
@classmethod
def pubkey_hash_to_address(cls, h160):
raw_address = cls.pubkey_address_prefix + h160
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
@classmethod
def public_key_to_address(cls, public_key):
return cls.pubkey_hash_to_address(hash160(public_key))
@classmethod
def script_hash_to_address(cls, h160):
raw_address = cls.script_address_prefix + h160
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
@staticmethod
def private_key_to_wif(private_key):
return b'\x1c' + private_key + b'\x01'
@classmethod
def is_valid_address(cls, address):
decoded = Base58.decode_check(address)
return decoded[0] == cls.pubkey_address_prefix[0]
@classmethod
def valid_address_or_error(cls, address):
try:
assert cls.is_valid_address(address)
except:
raise Exception(f"'{address}' is not a valid address")
@staticmethod
def valid_claim_id(claim_id: str):
if not len(claim_id) == 40:
raise Exception(f"Incorrect claimid length: {len(claim_id)}")
if set(claim_id).difference(hexdigits):
raise Exception("Claim id is not hex encoded")
@staticmethod
def valid_channel_name_or_error(name: str):
try:
if not name:
raise Exception("Channel name cannot be blank.")
parsed = URL.parse(name)
if not parsed.has_channel:
raise Exception("Channel names must start with '@' symbol.")
if parsed.channel.name != name:
raise Exception("Channel name has invalid character")
except (TypeError, ValueError):
raise Exception("Invalid channel name.")
@staticmethod
def valid_stream_name_or_error(name: str):
try:
if not name:
raise Exception('Stream name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
raise Exception(
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
raise Exception('Stream name has invalid characters.')
except (TypeError, ValueError):
raise Exception("Invalid stream name.")
@staticmethod
def valid_collection_name_or_error(name: str):
try:
if not name:
raise Exception('Collection name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
raise Exception(
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
raise Exception('Collection name has invalid characters.')
except (TypeError, ValueError):
raise Exception("Invalid collection name.")
@staticmethod
def get_dewies_or_error(argument: str, lbc: str, positive_value=False):
try:
dewies = lbc_to_dewies(lbc)
if positive_value and dewies <= 0:
raise ValueError(f"'{argument}' value must be greater than 0.0")
return dewies
except ValueError as e:
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
if 'fee_address' in kwargs:
self.valid_address_or_error(kwargs['fee_address'])
return kwargs['fee_address']
if 'fee_currency' in kwargs or 'fee_amount' in kwargs:
return claim_address
class TestNetLedger(Ledger):
network_name = 'testnet'
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
checkpoints = {}
class RegTestLedger(Ledger):
network_name = 'regtest'
headers_class = UnvalidatedHeaders
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
genesis_bits = 0x207fffff
target_timespan = 1
checkpoints = {}
def ledger_class_from_name(name) -> Type[Ledger]:
return {
Ledger.network_name: Ledger,
TestNetLedger.network_name: TestNetLedger,
RegTestLedger.network_name: RegTestLedger
}[name]
@@ -1,10 +1,14 @@
from typing import List
from itertools import chain
from binascii import hexlify
from collections import namedtuple
from typing import List
from torba.client.bcd_data_stream import BCDataStream
from torba.client.util import subclass_tuple
from .bcd_data_stream import BCDataStream
def subclass_tuple(name, base):
return type(name, (base,), {'__slots__': ()})
# bitcoin opcodes
OP_0 = 0x00
@@ -24,6 +28,15 @@ OP_RETURN = 0x6a
OP_2DROP = 0x6d
OP_DROP = 0x75
# lbry custom opcodes
# checks
OP_PRICECHECK = 0xb0 # checks that the BUY output is >= SELL price
# tx types
OP_CLAIM_NAME = 0xb5
OP_SUPPORT_CLAIM = 0xb6
OP_UPDATE_CLAIM = 0xb7
OP_SELL_CLAIM = 0xb8
OP_BUY_CLAIM = 0xb9
# template matching opcodes (not real opcodes)
# base class for PUSH_DATA related opcodes
@@ -284,20 +297,25 @@ class Template:
class Script:
__slots__ = 'source', '_template', '_values', '_template_hint'
__slots__ = 'source', 'offset', '_template', '_values', '_template_hint'
templates: List[Template] = []
NO_SCRIPT = Template('no_script', None) # special case
def __init__(self, source=None, template=None, values=None, template_hint=None):
def __init__(self, source=None, template=None, values=None, template_hint=None, offset=None):
self.source = source
self.offset = offset
self._template = template
self._values = values
self._template_hint = template_hint
if source is None and template and values:
self.generate()
@property
def length(self):
return len(self.source)
@property
def template(self):
if self._template is None:
@@ -337,8 +355,7 @@ class Script:
self.source = self.template.generate(self._values)
class BaseInputScript(Script):
""" Input / redeem script templates (aka scriptSig) """
class InputScript(Script):
__slots__ = ()
@@ -386,7 +403,7 @@ class BaseInputScript(Script):
})
class BaseOutputScript(Script):
class OutputScript(Script):
__slots__ = ()
@@ -400,15 +417,88 @@ class BaseOutputScript(Script):
PAY_SCRIPT_HASH = Template('pay_script_hash', (
OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL
))
PAY_SEGWIT = Template('pay_script_hash+segwit', (
OP_0, PUSH_SINGLE('script_hash')
))
RETURN_DATA = Template('return_data', (
OP_RETURN, PUSH_SINGLE('data')
))
CLAIM_NAME_OPCODES = (
OP_CLAIM_NAME, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim'),
OP_2DROP, OP_DROP
)
CLAIM_NAME_PUBKEY = Template('claim_name+pay_pubkey_hash', (
CLAIM_NAME_OPCODES + PAY_PUBKEY_HASH.opcodes
))
CLAIM_NAME_SCRIPT = Template('claim_name+pay_script_hash', (
CLAIM_NAME_OPCODES + PAY_SCRIPT_HASH.opcodes
))
SUPPORT_CLAIM_OPCODES = (
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'),
OP_2DROP, OP_DROP
)
SUPPORT_CLAIM_PUBKEY = Template('support_claim+pay_pubkey_hash', (
SUPPORT_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes
))
SUPPORT_CLAIM_SCRIPT = Template('support_claim+pay_script_hash', (
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
))
SUPPORT_CLAIM_DATA_OPCODES = (
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
OP_2DROP, OP_2DROP
)
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
))
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
))
UPDATE_CLAIM_OPCODES = (
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
OP_2DROP, OP_2DROP
)
UPDATE_CLAIM_PUBKEY = Template('update_claim+pay_pubkey_hash', (
UPDATE_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes
))
UPDATE_CLAIM_SCRIPT = Template('update_claim+pay_script_hash', (
UPDATE_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
))
SELL_SCRIPT = Template('sell_script', (
OP_VERIFY, OP_DROP, OP_DROP, OP_DROP, PUSH_INTEGER('price'), OP_PRICECHECK
))
SELL_CLAIM = Template('sell_claim+pay_script_hash', (
OP_SELL_CLAIM, PUSH_SINGLE('claim_id'), PUSH_SUBSCRIPT('sell_script', SELL_SCRIPT),
PUSH_SUBSCRIPT('receive_script', InputScript.REDEEM_SCRIPT), OP_2DROP, OP_2DROP
) + PAY_SCRIPT_HASH.opcodes)
BUY_CLAIM = Template('buy_claim+pay_script_hash', (
OP_BUY_CLAIM, PUSH_SINGLE('sell_id'),
PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim_version'),
PUSH_SINGLE('owner_pubkey_hash'), PUSH_SINGLE('negotiation_signature'),
OP_2DROP, OP_2DROP, OP_2DROP,
) + PAY_SCRIPT_HASH.opcodes)
templates = [
PAY_PUBKEY_FULL,
PAY_PUBKEY_HASH,
PAY_SCRIPT_HASH,
RETURN_DATA
PAY_SEGWIT,
RETURN_DATA,
CLAIM_NAME_PUBKEY,
CLAIM_NAME_SCRIPT,
SUPPORT_CLAIM_PUBKEY,
SUPPORT_CLAIM_SCRIPT,
SUPPORT_CLAIM_DATA_PUBKEY,
SUPPORT_CLAIM_DATA_SCRIPT,
UPDATE_CLAIM_PUBKEY,
UPDATE_CLAIM_SCRIPT,
SELL_CLAIM, SELL_SCRIPT,
BUY_CLAIM,
]
@classmethod
@@ -433,6 +523,65 @@ class BaseOutputScript(Script):
def is_pay_pubkey(self):
return self.template.name.endswith('pay_pubkey_full')
@classmethod
def pay_claim_name_pubkey_hash(cls, claim_name, claim, pubkey_hash):
return cls(template=cls.CLAIM_NAME_PUBKEY, values={
'claim_name': claim_name,
'claim': claim,
'pubkey_hash': pubkey_hash
})
@classmethod
def pay_update_claim_pubkey_hash(cls, claim_name, claim_id, claim, pubkey_hash):
return cls(template=cls.UPDATE_CLAIM_PUBKEY, values={
'claim_name': claim_name,
'claim_id': claim_id,
'claim': claim,
'pubkey_hash': pubkey_hash
})
@classmethod
def pay_support_pubkey_hash(cls, claim_name: bytes, claim_id: bytes, pubkey_hash: bytes):
return cls(template=cls.SUPPORT_CLAIM_PUBKEY, values={
'claim_name': claim_name,
'claim_id': claim_id,
'pubkey_hash': pubkey_hash
})
@classmethod
def pay_support_data_pubkey_hash(
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
'claim_name': claim_name,
'claim_id': claim_id,
'support': support,
'pubkey_hash': pubkey_hash
})
@classmethod
def sell_script(cls, price):
return cls(template=cls.SELL_SCRIPT, values={
'price': price,
})
@classmethod
def sell_claim(cls, claim_id, price, signatures, pubkeys):
return cls(template=cls.SELL_CLAIM, values={
'claim_id': claim_id,
'sell_script': OutputScript.sell_script(price),
'receive_script': InputScript.redeem_script(signatures, pubkeys)
})
@classmethod
def buy_claim(cls, sell_id, claim_id, claim_version, owner_pubkey_hash, negotiation_signature):
return cls(template=cls.BUY_CLAIM, values={
'sell_id': sell_id,
'claim_id': claim_id,
'claim_version': claim_version,
'owner_pubkey_hash': owner_pubkey_hash,
'negotiation_signature': negotiation_signature,
})
@property
def is_pay_pubkey_hash(self):
return self.template.name.endswith('pay_pubkey_hash')
@@ -444,3 +593,34 @@ class BaseOutputScript(Script):
@property
def is_return_data(self):
return self.template.name.endswith('return_data')
@property
def is_claim_name(self):
return self.template.name.startswith('claim_name+')
@property
def is_update_claim(self):
return self.template.name.startswith('update_claim+')
@property
def is_support_claim(self):
return self.template.name.startswith('support_claim+')
@property
def is_support_claim_data(self):
return self.template.name.startswith('support_claim+data+')
@property
def is_sell_claim(self):
return self.template.name.startswith('sell_claim+')
@property
def is_buy_claim(self):
return self.template.name.startswith('buy_claim+')
@property
def is_claim_involved(self):
return any((
self.is_claim_name, self.is_support_claim, self.is_update_claim,
self.is_sell_claim, self.is_buy_claim
))
+1
View File
@@ -0,0 +1 @@
from .synchronizer import BlockchainSync
+340
View File
@@ -0,0 +1,340 @@
import logging
from binascii import hexlify, unhexlify
from typing import Tuple, List
from sqlalchemy import table, text, func, union, between
from sqlalchemy.future import select
from sqlalchemy.schema import CreateTable
from lbry.db.tables import (
Block as BlockTable, BlockFilter,
TX, TXFilter, TXO, TXI, Claim, Tag, Support
)
from lbry.db.tables import (
pg_add_block_constraints_and_indexes,
pg_add_block_filter_constraints_and_indexes,
pg_add_tx_constraints_and_indexes,
pg_add_tx_filter_constraints_and_indexes,
pg_add_txo_constraints_and_indexes,
pg_add_txi_constraints_and_indexes,
)
from lbry.db.query_context import ProgressContext, event_emitter, context
from lbry.db.sync import set_input_addresses, update_spent_outputs
from lbry.blockchain.transaction import Transaction
from lbry.blockchain.block import Block, create_address_filter
from lbry.blockchain.bcd_data_stream import BCDataStream
from .context import get_or_initialize_lbrycrd
from .filter_builder import FilterBuilder
log = logging.getLogger(__name__)
def get_best_block_height_for_file(file_number):
return context().fetchone(
select(func.coalesce(func.max(BlockTable.c.height), -1).label('height'))
.where(BlockTable.c.file_number == file_number)
)['height']
@event_emitter("blockchain.sync.blocks.file", "blocks", "txs", throttle=100)
def sync_block_file(
file_number: int, start_height: int, txs: int, flush_size: int, p: ProgressContext
):
chain = get_or_initialize_lbrycrd(p.ctx)
new_blocks = chain.db.sync_get_blocks_in_file(file_number, start_height)
if not new_blocks:
return -1
file_name = chain.get_block_file_name(file_number)
p.start(len(new_blocks), txs, progress_id=file_number, label=file_name)
block_file_path = chain.get_block_file_path(file_number)
done_blocks = done_txs = 0
last_block_processed, loader = -1, p.ctx.get_bulk_loader()
with open(block_file_path, "rb") as fp:
stream = BCDataStream(fp=fp)
for done_blocks, block_info in enumerate(new_blocks, start=1):
block_height = block_info["height"]
fp.seek(block_info["data_offset"])
block = Block.from_data_stream(stream, block_height, file_number)
loader.add_block(block)
if len(loader.txs) >= flush_size:
done_txs += loader.flush(TX)
p.step(done_blocks, done_txs)
last_block_processed = block_height
if p.ctx.stop_event.is_set():
return last_block_processed
if loader.txs:
done_txs += loader.flush(TX)
p.step(done_blocks, done_txs)
return last_block_processed
@event_emitter("blockchain.sync.blocks.indexes", "steps")
def blocks_constraints_and_indexes(p: ProgressContext):
p.start(1 + len(pg_add_block_constraints_and_indexes))
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE block;"))
p.step()
for constraint in pg_add_block_constraints_and_indexes:
if p.ctx.is_postgres:
p.ctx.execute(text(constraint))
p.step()
@event_emitter("blockchain.sync.blocks.vacuum", "steps")
def blocks_vacuum(p: ProgressContext):
p.start(1)
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM block;"))
p.step()
@event_emitter("blockchain.sync.spends.main", "steps")
def sync_spends(initial_sync: bool, p: ProgressContext):
if initial_sync:
p.start(
7 +
len(pg_add_tx_constraints_and_indexes) +
len(pg_add_txi_constraints_and_indexes) +
len(pg_add_txo_constraints_and_indexes)
)
# 1. tx table stuff
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE tx;"))
p.step()
for constraint in pg_add_tx_constraints_and_indexes:
if p.ctx.is_postgres:
p.ctx.execute(text(constraint))
p.step()
# A. Update TXIs to have the address of TXO they are spending.
# 2. txi table reshuffling
p.ctx.execute(text("ALTER TABLE txi RENAME TO old_txi;"))
p.ctx.execute(CreateTable(TXI, include_foreign_key_constraints=[]))
if p.ctx.is_postgres:
p.ctx.execute(text("ALTER TABLE txi DROP CONSTRAINT txi_pkey;"))
p.step()
# 3. insert
old_txi = table("old_txi", *(c.copy() for c in TXI.columns)) # pylint: disable=not-an-iterable
columns = [c for c in old_txi.columns if c.name != "address"] + [TXO.c.address]
join_txi_on_txo = old_txi.join(TXO, old_txi.c.txo_hash == TXO.c.txo_hash)
select_txis = select(*columns).select_from(join_txi_on_txo)
insert_txis = TXI.insert().from_select(columns, select_txis)
p.ctx.execute(insert_txis)
p.step()
# 4. drop old txi and vacuum
p.ctx.execute(text("DROP TABLE old_txi;"))
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE txi;"))
p.step()
for constraint in pg_add_txi_constraints_and_indexes:
if p.ctx.is_postgres:
p.ctx.execute(text(constraint))
p.step()
# B. Update TXOs to have the height at which they were spent (if they were).
# 5. txo table reshuffling
p.ctx.execute(text("ALTER TABLE txo RENAME TO old_txo;"))
p.ctx.execute(CreateTable(TXO, include_foreign_key_constraints=[]))
if p.ctx.is_postgres:
p.ctx.execute(text("ALTER TABLE txo DROP CONSTRAINT txo_pkey;"))
p.step()
# 6. insert
old_txo = table("old_txo", *(c.copy() for c in TXO.columns)) # pylint: disable=not-an-iterable
columns = [c for c in old_txo.columns if c.name != "spent_height"]
insert_columns = columns + [TXO.c.spent_height]
select_columns = columns + [func.coalesce(TXI.c.height, 0).label("spent_height")]
join_txo_on_txi = old_txo.join(TXI, old_txo.c.txo_hash == TXI.c.txo_hash, isouter=True)
select_txos = select(*select_columns).select_from(join_txo_on_txi)
insert_txos = TXO.insert().from_select(insert_columns, select_txos)
p.ctx.execute(insert_txos)
p.step()
# 7. drop old txo
p.ctx.execute(text("DROP TABLE old_txo;"))
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE txo;"))
p.step()
for constraint in pg_add_txo_constraints_and_indexes:
if p.ctx.is_postgres:
p.ctx.execute(text(constraint))
p.step()
else:
p.start(5)
# 1. Update spent TXOs setting spent_height
update_spent_outputs(p.ctx)
p.step()
# 2. Update TXIs to have the address of TXO they are spending.
set_input_addresses(p.ctx)
p.step()
# 3. Update tx visibility map, which speeds up index-only scans.
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM tx;"))
p.step()
# 4. Update txi visibility map, which speeds up index-only scans.
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM txi;"))
p.step()
# 4. Update txo visibility map, which speeds up index-only scans.
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM txo;"))
p.step()
@event_emitter("blockchain.sync.mempool.clear", "txs")
def clear_mempool(p: ProgressContext):
delete_all_the_things(-1, p)
@event_emitter("blockchain.sync.mempool.main", "txs")
def sync_mempool(p: ProgressContext) -> List[str]:
chain = get_or_initialize_lbrycrd(p.ctx)
mempool = chain.sync_run(chain.get_raw_mempool())
current = [hexlify(r['tx_hash'][::-1]).decode() for r in p.ctx.fetchall(
select(TX.c.tx_hash).where(TX.c.height < 0)
)]
loader = p.ctx.get_bulk_loader()
added = []
for txid in mempool:
if txid not in current:
raw_tx = chain.sync_run(chain.get_raw_transaction(txid))
loader.add_transaction(
None, Transaction(unhexlify(raw_tx), height=-1)
)
added.append(txid)
if p.ctx.stop_event.is_set():
return
loader.flush(TX)
return added
@event_emitter("blockchain.sync.filters.generate", "blocks", throttle=100)
def sync_filters(start, end, p: ProgressContext):
fp = FilterBuilder(start, end)
p.start((end-start)+1, progress_id=start, label=f"generate filters {start}-{end}")
with p.ctx.connect_streaming() as c:
loader = p.ctx.get_bulk_loader()
tx_hash, height, addresses, last_added = None, None, set(), None
address_to_hash = p.ctx.ledger.address_to_hash160
for row in c.execute(get_block_tx_addresses_sql(*fp.query_heights)):
if tx_hash != row.tx_hash:
if tx_hash is not None:
last_added = tx_hash
fp.add(tx_hash, height, addresses)
tx_hash, height, addresses = row.tx_hash, row.height, set()
addresses.add(address_to_hash(row.address))
if all([last_added, tx_hash]) and last_added != tx_hash: # pickup last tx
fp.add(tx_hash, height, addresses)
for tx_hash, height, addresses in fp.tx_filters:
loader.add_transaction_filter(
tx_hash, height, create_address_filter(list(addresses))
)
for height, addresses in fp.block_filters.items():
loader.add_block_filter(
height, 1, create_address_filter(list(addresses))
)
for group_filter in fp.group_filters:
for height, addresses in group_filter.groups.items():
loader.add_block_filter(
height, group_filter.factor, create_address_filter(list(addresses))
)
p.add(loader.flush(BlockFilter))
@event_emitter("blockchain.sync.filters.indexes", "steps")
def filters_constraints_and_indexes(p: ProgressContext):
is_postgres = p.ctx.is_postgres and p.ctx.pg_has_pk_constraint('block_filter')
constraints = (
pg_add_tx_filter_constraints_and_indexes +
pg_add_block_filter_constraints_and_indexes
)
p.start(2 + len(constraints))
if is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE block_filter;"))
p.step()
if is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE tx_filter;"))
p.step()
for constraint in constraints:
if is_postgres:
p.ctx.execute(text(constraint))
p.step()
@event_emitter("blockchain.sync.filters.vacuum", "steps")
def filters_vacuum(p: ProgressContext):
p.start(2)
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM block_filter;"))
p.step()
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM tx_filter;"))
p.step()
def get_block_range_without_filters() -> Tuple[int, int]:
sql = (
select(
func.coalesce(func.min(BlockTable.c.height), -1).label('start_height'),
func.coalesce(func.max(BlockTable.c.height), -1).label('end_height'),
)
.select_from(
BlockTable.join(
BlockFilter,
(BlockTable.c.height == BlockFilter.c.height) &
(BlockFilter.c.factor == 1),
isouter=True
)
)
.where(BlockFilter.c.height.is_(None))
)
result = context().fetchone(sql)
return result['start_height'], result['end_height']
def get_block_tx_addresses_sql(start_height, end_height):
return union(
select(TXO.c.tx_hash, TXO.c.height, TXO.c.address).where(
(TXO.c.address.isnot(None)) & between(TXO.c.height, start_height, end_height)
),
select(TXI.c.tx_hash, TXI.c.height, TXI.c.address).where(
(TXI.c.address.isnot(None)) & between(TXI.c.height, start_height, end_height)
),
).order_by('height', 'tx_hash')
@event_emitter("blockchain.sync.rewind.main", "steps")
def rewind(height: int, p: ProgressContext):
delete_all_the_things(height, p)
def delete_all_the_things(height: int, p: ProgressContext):
def constrain(col):
if height == -1:
return col == -1
return col >= height
deletes = [
BlockTable.delete().where(constrain(BlockTable.c.height)),
TXI.delete().where(constrain(TXI.c.height)),
TXO.delete().where(constrain(TXO.c.height)),
TX.delete().where(constrain(TX.c.height)),
Tag.delete().where(
Tag.c.claim_hash.in_(
select(Claim.c.claim_hash).where(constrain(Claim.c.height))
)
),
Claim.delete().where(constrain(Claim.c.height)),
Support.delete().where(constrain(Support.c.height)),
]
if height > 0:
deletes.extend([
# TODO: block and tx filters need better where() clauses (below actually breaks things)
BlockFilter.delete().where(BlockFilter.c.height >= height),
TXFilter.delete(),
])
for delete in p.iter(deletes):
p.ctx.execute(delete)
+335
View File
@@ -0,0 +1,335 @@
import logging
from typing import Tuple
from sqlalchemy import case, func, text
from sqlalchemy.future import select
from lbry.db.queries.txio import (
minimum_txo_columns, row_to_txo,
where_unspent_txos, where_claims_with_changed_supports,
count_unspent_txos, where_channels_with_changed_content,
where_abandoned_claims, count_channels_with_changed_content,
where_claims_with_changed_reposts, where_claims_with_stale_signatures
)
from lbry.db.query_context import ProgressContext, event_emitter
from lbry.db.tables import (
TX, TXO, Claim, Support, CensoredClaim,
pg_add_claim_and_tag_constraints_and_indexes
)
from lbry.db.utils import least
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
from lbry.schema.result import Censor
from lbry.blockchain.transaction import Output
from .context import get_or_initialize_lbrycrd
log = logging.getLogger(__name__)
def channel_content_count_calc(signable):
return (
select(func.count(signable.c.claim_hash))
.where((signable.c.channel_hash == Claim.c.claim_hash) & signable.c.is_signature_valid)
.scalar_subquery()
)
support = TXO.alias('support')
def staked_support_aggregation(aggregate):
return (
select(aggregate).where(
(support.c.txo_type == TXO_TYPES['support']) &
(support.c.spent_height == 0)
).scalar_subquery()
)
def staked_support_amount_calc(other):
return (
staked_support_aggregation(func.coalesce(func.sum(support.c.amount), 0))
.where(support.c.claim_hash == other.c.claim_hash)
)
def staked_support_count_calc(other):
return (
staked_support_aggregation(func.coalesce(func.count('*'), 0))
.where(support.c.claim_hash == other.c.claim_hash)
)
def reposted_claim_count_calc(other):
repost = TXO.alias('repost')
return (
select(func.coalesce(func.count(repost.c.reposted_claim_hash), 0))
.where(
(repost.c.reposted_claim_hash == other.c.claim_hash) &
(repost.c.spent_height == 0)
).scalar_subquery()
)
def make_label(action, blocks):
if blocks[0] == blocks[-1]:
return f"{action} {blocks[0]:>6}"
else:
return f"{action} {blocks[0]:>6}-{blocks[-1]:>6}"
def select_claims_for_saving(
blocks: Tuple[int, int],
missing_in_claims_table=False,
missing_or_stale_in_claims_table=False,
):
channel_txo = TXO.alias('channel_txo')
return select(
*minimum_txo_columns, TXO.c.claim_hash,
staked_support_amount_calc(TXO).label('staked_support_amount'),
staked_support_count_calc(TXO).label('staked_support_count'),
reposted_claim_count_calc(TXO).label('reposted_count'),
TXO.c.signature, TXO.c.signature_digest,
case([(
TXO.c.channel_hash.isnot(None),
select(channel_txo.c.public_key).select_from(channel_txo).where(
(channel_txo.c.spent_height == 0) &
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
(channel_txo.c.claim_hash == TXO.c.channel_hash)
).limit(1).scalar_subquery()
)]).label('channel_public_key')
).where(
where_unspent_txos(
CLAIM_TYPE_CODES, blocks,
missing_in_claims_table=missing_in_claims_table,
missing_or_stale_in_claims_table=missing_or_stale_in_claims_table,
)
).select_from(TXO.join(TX))
def row_to_claim_for_saving(row) -> Tuple[Output, dict]:
return row_to_txo(row), {
'staked_support_amount': int(row.staked_support_amount),
'staked_support_count': int(row.staked_support_count),
'reposted_count': int(row.reposted_count),
'signature': row.signature,
'signature_digest': row.signature_digest,
'channel_public_key': row.channel_public_key
}
@event_emitter("blockchain.sync.claims.insert", "claims")
def claims_insert(
blocks: Tuple[int, int],
missing_in_claims_table: bool,
flush_size: int,
p: ProgressContext
):
chain = get_or_initialize_lbrycrd(p.ctx)
p.start(
count_unspent_txos(
CLAIM_TYPE_CODES, blocks,
missing_in_claims_table=missing_in_claims_table,
), progress_id=blocks[0], label=make_label("add claims", blocks)
)
with p.ctx.connect_streaming() as c:
loader = p.ctx.get_bulk_loader()
cursor = c.execute(select_claims_for_saving(
blocks, missing_in_claims_table=missing_in_claims_table
).order_by(TXO.c.claim_hash))
for rows in cursor.partitions(900):
claim_metadata = chain.db.sync_get_claim_metadata(
claim_hashes=[row['claim_hash'] for row in rows]
)
i = 0
for row in rows:
metadata = claim_metadata[i] if i < len(claim_metadata) else {}
if metadata and metadata['claim_hash'] == row.claim_hash:
i += 1
txo, extra = row_to_claim_for_saving(row)
extra.update({
'short_url': metadata.get('short_url'),
'creation_height': metadata.get('creation_height'),
'activation_height': metadata.get('activation_height'),
'expiration_height': metadata.get('expiration_height'),
'takeover_height': metadata.get('takeover_height'),
})
loader.add_claim(txo, **extra)
if len(loader.claims) >= flush_size:
p.add(loader.flush(Claim))
p.add(loader.flush(Claim))
@event_emitter("blockchain.sync.claims.indexes", "steps")
def claims_constraints_and_indexes(p: ProgressContext):
should_run = p.ctx.is_postgres and not p.ctx.pg_has_pk_constraint('claim')
p.start(2 + len(pg_add_claim_and_tag_constraints_and_indexes))
if should_run:
p.ctx.execute_notx(text("VACUUM ANALYZE claim;"))
p.step()
if should_run:
p.ctx.execute_notx(text("VACUUM ANALYZE tag;"))
p.step()
for constraint in pg_add_claim_and_tag_constraints_and_indexes:
if should_run:
p.ctx.execute(text(constraint))
p.step()
@event_emitter("blockchain.sync.claims.vacuum", "steps")
def claims_vacuum(p: ProgressContext):
p.start(2)
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM claim;"))
p.step()
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM tag;"))
p.step()
@event_emitter("blockchain.sync.claims.update", "claims")
def claims_update(blocks: Tuple[int, int], p: ProgressContext):
p.start(
count_unspent_txos(CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True),
progress_id=blocks[0], label=make_label("mod claims", blocks)
)
with p.ctx.connect_streaming() as c:
loader = p.ctx.get_bulk_loader()
cursor = c.execute(select_claims_for_saving(
blocks, missing_or_stale_in_claims_table=True
))
for row in cursor:
txo, extra = row_to_claim_for_saving(row)
loader.update_claim(txo, **extra)
if len(loader.update_claims) >= 25:
p.add(loader.flush(Claim))
p.add(loader.flush(Claim))
@event_emitter("blockchain.sync.claims.delete", "claims")
def claims_delete(claims, p: ProgressContext):
p.start(claims, label="del claims")
deleted = p.ctx.execute(Claim.delete().where(where_abandoned_claims()))
p.step(deleted.rowcount)
@event_emitter("blockchain.sync.claims.takeovers", "claims")
def update_takeovers(blocks: Tuple[int, int], takeovers, p: ProgressContext):
p.start(takeovers, label=make_label("mod winner", blocks))
chain = get_or_initialize_lbrycrd(p.ctx)
with p.ctx.engine.begin() as c:
for takeover in chain.db.sync_get_takeovers(start_height=blocks[0], end_height=blocks[-1]):
update_claims = (
Claim.update()
.where(Claim.c.normalized == takeover['normalized'])
.values(
is_controlling=case(
[(Claim.c.claim_hash == takeover['claim_hash'], True)],
else_=False
),
takeover_height=case(
[(Claim.c.claim_hash == takeover['claim_hash'], takeover['height'])],
else_=None
),
activation_height=least(Claim.c.activation_height, takeover['height']),
)
)
result = c.execute(update_claims)
p.add(result.rowcount)
@event_emitter("blockchain.sync.claims.stakes", "claims")
def update_stakes(blocks: Tuple[int, int], claims: int, p: ProgressContext):
p.start(claims)
sql = (
Claim.update()
.where(where_claims_with_changed_supports(blocks))
.values(
staked_support_amount=staked_support_amount_calc(Claim),
staked_support_count=staked_support_count_calc(Claim),
)
)
result = p.ctx.execute(sql)
p.step(result.rowcount)
@event_emitter("blockchain.sync.claims.reposts", "claims")
def update_reposts(blocks: Tuple[int, int], claims: int, p: ProgressContext):
p.start(claims)
sql = (
Claim.update()
.where(where_claims_with_changed_reposts(blocks))
.values(reposted_count=reposted_claim_count_calc(Claim))
)
result = p.ctx.execute(sql)
p.step(result.rowcount)
@event_emitter("blockchain.sync.claims.invalidate", "claims")
def update_stale_signatures(blocks: Tuple[int, int], claims: int, p: ProgressContext):
p.start(claims)
with p.ctx.connect_streaming() as c:
loader = p.ctx.get_bulk_loader()
stream = Claim.alias('stream')
sql = (
select_claims_for_saving(None)
.where(TXO.c.claim_hash.in_(
where_claims_with_stale_signatures(
select(stream.c.claim_hash), blocks, stream
)
))
)
cursor = c.execute(sql)
for row in cursor:
txo, extra = row_to_claim_for_saving(row)
loader.update_claim(txo, public_key_height=blocks[1], **extra)
if len(loader.update_claims) >= 25:
p.add(loader.flush(Claim))
p.add(loader.flush(Claim))
@event_emitter("blockchain.sync.claims.channels", "channels")
def update_channel_stats(blocks: Tuple[int, int], initial_sync: int, p: ProgressContext):
update_sql = Claim.update().values(
signed_claim_count=channel_content_count_calc(Claim.alias('content')),
signed_support_count=channel_content_count_calc(Support),
)
if initial_sync:
p.start(p.ctx.fetchtotal(Claim.c.claim_type == TXO_TYPES['channel']), label="channel stats")
update_sql = update_sql.where(Claim.c.claim_type == TXO_TYPES['channel'])
elif blocks:
p.start(count_channels_with_changed_content(blocks), label="channel stats")
update_sql = update_sql.where(where_channels_with_changed_content(blocks))
else:
return
result = p.ctx.execute(update_sql)
if result.rowcount and p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM claim;"))
p.step(result.rowcount)
def select_reposts(channel_hashes, filter_type):
return (
select(Claim.c.reposted_claim_hash, filter_type, Claim.c.channel_hash).where(
(Claim.c.channel_hash.in_(channel_hashes)) &
(Claim.c.reposted_claim_hash.isnot(None))
)
)
@event_emitter("blockchain.sync.claims.filters", "claim_filters")
def update_claim_filters(resolve_censor_channel_hashes, search_censor_channel_hashes, p: ProgressContext):
p.ctx.execute(CensoredClaim.delete())
# order matters: first we insert the resolve filters; then the search ones.
# a claim that's censored in resolve is automatically also censored in search results.
p.ctx.execute(CensoredClaim.insert().from_select(
['claim_hash', 'censor_type', 'censoring_channel_hash'],
select_reposts(resolve_censor_channel_hashes, Censor.RESOLVE)
))
p.ctx.execute(p.ctx.insert_or_ignore(CensoredClaim).from_select(
['claim_hash', 'censor_type', 'censoring_channel_hash'],
select_reposts(search_censor_channel_hashes, Censor.SEARCH)
))
+25
View File
@@ -0,0 +1,25 @@
from contextvars import ContextVar
from lbry.db import query_context
from lbry.blockchain.lbrycrd import Lbrycrd
_chain: ContextVar[Lbrycrd] = ContextVar('chain')
def get_or_initialize_lbrycrd(ctx=None) -> Lbrycrd:
chain = _chain.get(None)
if chain is not None:
return chain
chain = Lbrycrd((ctx or query_context.context()).ledger)
chain.db.sync_open()
_chain.set(chain)
return chain
def uninitialize():
chain = _chain.get(None)
if chain is not None:
chain.db.sync_close()
chain.sync_run(chain.close_session())
_chain.set(None)
+80
View File
@@ -0,0 +1,80 @@
from typing import Dict
def split_range_into_batches(start, end, batch_size=100_000):
batch = [start, end]
batches = [batch]
for block in range(start, end+1):
if 0 < block != batch[0] and block % batch_size == 0:
batch = [block, block]
batches.append(batch)
else:
batch[1] = block
return batches
class GroupFilter:
"""
Collects addresses into buckets of specific sizes defined by 10 raised to power of factor.
eg. a factor of 2 (10**2) would create block buckets 100-199, 200-299, etc
a factor of 3 (10**3) would create block buckets 1000-1999, 2000-2999, etc
"""
def __init__(self, start, end, factor):
self.start = start
self.end = end
self.factor = factor
self.resolution = resolution = 10**factor
last_height_in_group, groups = resolution-1, {}
for block in range(start, end+1):
if block % resolution == last_height_in_group:
groups[block-last_height_in_group] = set()
self.last_height_in_group = last_height_in_group
self.groups: Dict[int, set] = groups
@property
def coverage(self):
return list(self.groups.keys())
def add(self, height, addresses):
group = self.groups.get(height - (height % self.resolution))
if group is not None:
group.update(addresses)
class FilterBuilder:
"""
Creates filter groups, calculates the necessary block range to fulfill creation
of filter groups and collects tx filters, block filters and group filters.
"""
def __init__(self, start, end):
self.start = start
self.end = end
self.group_filters = [
GroupFilter(start, end, 5),
GroupFilter(start, end, 4),
GroupFilter(start, end, 3),
GroupFilter(start, end, 2),
]
self.start_tx_height, self.end_tx_height = self._calculate_tx_heights_for_query()
self.tx_filters = []
self.block_filters: Dict[int, set] = {}
def _calculate_tx_heights_for_query(self):
for group_filter in self.group_filters:
if group_filter.groups:
return group_filter.coverage[0], self.end
return self.start, self.end
@property
def query_heights(self):
return self.start_tx_height, self.end_tx_height
def add(self, tx_hash, height, addresses):
if self.start <= height <= self.end:
self.tx_filters.append((tx_hash, height, addresses))
block_filter = self.block_filters.get(height)
if block_filter is None:
block_filter = self.block_filters[height] = set()
block_filter.update(addresses)
for group_filter in self.group_filters:
group_filter.add(height, addresses)
+95
View File
@@ -0,0 +1,95 @@
import logging
from typing import Tuple
from sqlalchemy import case, desc, text
from sqlalchemy.future import select
from lbry.db.tables import TX, TXO, Support, pg_add_support_constraints_and_indexes
from lbry.db.query_context import ProgressContext, event_emitter
from lbry.db.queries import row_to_txo
from lbry.db.constants import TXO_TYPES
from lbry.db.queries.txio import (
minimum_txo_columns,
where_unspent_txos, where_abandoned_supports,
count_unspent_txos,
)
from .claims import make_label
log = logging.getLogger(__name__)
@event_emitter("blockchain.sync.supports.insert", "supports")
def supports_insert(
blocks: Tuple[int, int],
missing_in_supports_table: bool,
flush_size: int,
p: ProgressContext
):
p.start(
count_unspent_txos(
TXO_TYPES['support'], blocks,
missing_in_supports_table=missing_in_supports_table,
), progress_id=blocks[0], label=make_label("add supprt", blocks)
)
channel_txo = TXO.alias('channel_txo')
select_supports = select(
*minimum_txo_columns, TXO.c.claim_hash,
TXO.c.signature, TXO.c.signature_digest,
case([(
TXO.c.channel_hash.isnot(None),
select(channel_txo.c.public_key).select_from(channel_txo).where(
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
(channel_txo.c.claim_hash == TXO.c.channel_hash) &
(channel_txo.c.height <= TXO.c.height)
).order_by(desc(channel_txo.c.height)).limit(1).scalar_subquery()
)]).label('channel_public_key'),
).select_from(
TXO.join(TX)
).where(
where_unspent_txos(
TXO_TYPES['support'], blocks,
missing_in_supports_table=missing_in_supports_table,
)
)
with p.ctx.connect_streaming() as c:
loader = p.ctx.get_bulk_loader()
for row in c.execute(select_supports):
txo = row_to_txo(row)
loader.add_support(
txo,
signature=row.signature,
signature_digest=row.signature_digest,
channel_public_key=row.channel_public_key
)
if len(loader.supports) >= flush_size:
p.add(loader.flush(Support))
p.add(loader.flush(Support))
@event_emitter("blockchain.sync.supports.delete", "supports")
def supports_delete(supports, p: ProgressContext):
p.start(supports, label="del supprt")
deleted = p.ctx.execute(Support.delete().where(where_abandoned_supports()))
p.step(deleted.rowcount)
@event_emitter("blockchain.sync.supports.indexes", "steps")
def supports_constraints_and_indexes(p: ProgressContext):
p.start(1 + len(pg_add_support_constraints_and_indexes))
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM ANALYZE support;"))
p.step()
for constraint in pg_add_support_constraints_and_indexes:
if p.ctx.is_postgres:
p.ctx.execute(text(constraint))
p.step()
@event_emitter("blockchain.sync.supports.vacuum", "steps")
def supports_vacuum(p: ProgressContext):
p.start(1)
if p.ctx.is_postgres:
p.ctx.execute_notx(text("VACUUM support;"))
p.step()
+422
View File
@@ -0,0 +1,422 @@
import os
import asyncio
import logging
from binascii import unhexlify
from typing import Optional, Tuple, Set, List, Coroutine
from concurrent.futures import ThreadPoolExecutor
from lbry.db import Database, trending
from lbry.db import queries as q
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
from lbry.db.query_context import Event, Progress
from lbry.event import BroadcastSubscription, EventController
from lbry.service.base import Sync, BlockEvent
from lbry.blockchain.lbrycrd import Lbrycrd
from lbry.error import LbrycrdEventSubscriptionError
from . import blocks as block_phase, claims as claim_phase, supports as support_phase
from .context import uninitialize
from .filter_builder import split_range_into_batches
log = logging.getLogger(__name__)
BLOCKS_INIT_EVENT = Event.add("blockchain.sync.blocks.init", "steps")
BLOCKS_MAIN_EVENT = Event.add("blockchain.sync.blocks.main", "blocks", "txs")
FILTER_INIT_EVENT = Event.add("blockchain.sync.filters.init", "steps")
FILTER_MAIN_EVENT = Event.add("blockchain.sync.filters.main", "blocks")
CLAIMS_INIT_EVENT = Event.add("blockchain.sync.claims.init", "steps")
CLAIMS_MAIN_EVENT = Event.add("blockchain.sync.claims.main", "claims")
TRENDS_INIT_EVENT = Event.add("blockchain.sync.trends.init", "steps")
TRENDS_MAIN_EVENT = Event.add("blockchain.sync.trends.main", "blocks")
SUPPORTS_INIT_EVENT = Event.add("blockchain.sync.supports.init", "steps")
SUPPORTS_MAIN_EVENT = Event.add("blockchain.sync.supports.main", "supports")
class BlockchainSync(Sync):
TX_FLUSH_SIZE = 25_000 # flush to db after processing this many TXs and update progress
CLAIM_FLUSH_SIZE = 25_000 # flush to db after processing this many claims and update progress
SUPPORT_FLUSH_SIZE = 25_000 # flush to db after processing this many supports and update progress
FILTER_FLUSH_SIZE = 10_000 # flush to db after processing this many filters and update progress
def __init__(self, chain: Lbrycrd, db: Database):
super().__init__(chain.ledger, db)
self.chain = chain
self.pid = os.getpid()
self._on_block_controller = EventController()
self.on_block = self._on_block_controller.stream
self.conf.events.register("blockchain.block", self.on_block)
self._on_mempool_controller = EventController()
self.on_mempool = self._on_mempool_controller.stream
self.on_block_hash_subscription: Optional[BroadcastSubscription] = None
self.on_tx_hash_subscription: Optional[BroadcastSubscription] = None
self.advance_loop_task: Optional[asyncio.Task] = None
self.block_hash_event = asyncio.Event()
self.tx_hash_event = asyncio.Event()
self.mempool = []
self.search_censor_channel_hashes = {
unhexlify(channel_id)[::-1] for channel_id in self.conf.search_censor_channel_ids
}
self.resolve_censor_channel_hashes = {
unhexlify(channel_id)[::-1] for channel_id in self.conf.resolve_censor_channel_ids
}
async def wait_for_chain_ready(self):
while True:
try:
return await self.chain.ensure_subscribable()
except asyncio.CancelledError:
raise
except LbrycrdEventSubscriptionError as e:
log.warning(
"Lbrycrd is misconfigured. Please double check if"
" zmqpubhashblock is properly set on lbrycrd.conf"
)
raise
except Exception as e:
log.warning("Blockchain not ready, waiting for it: %s", str(e))
await asyncio.sleep(1)
async def start(self):
self.db.stop_event.clear()
await self.wait_for_chain_ready()
self.advance_loop_task = asyncio.create_task(self.advance())
await self.advance_loop_task
await self.chain.subscribe()
self.advance_loop_task = asyncio.create_task(self.advance_loop())
self.on_block_hash_subscription = self.chain.on_block_hash.listen(
lambda e: self.block_hash_event.set()
)
self.on_tx_hash_subscription = self.chain.on_tx_hash.listen(
lambda e: self.tx_hash_event.set()
)
async def stop(self):
self.chain.unsubscribe()
self.db.stop_event.set()
for subscription in (
self.on_block_hash_subscription,
self.on_tx_hash_subscription,
self.advance_loop_task
):
if subscription is not None:
subscription.cancel()
if isinstance(self.db.executor, ThreadPoolExecutor):
await self.db.run(uninitialize)
async def run_tasks(self, tasks: List[Coroutine]) -> Optional[Set[asyncio.Future]]:
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_EXCEPTION
)
if pending:
self.db.stop_event.set()
for future in pending:
future.cancel()
for future in done:
future.result()
return
return done
async def get_block_headers(self, start_height: int, end_height: int = None):
return await self.db.get_block_headers(start_height, end_height)
async def get_best_block_height(self) -> int:
return await self.db.get_best_block_height()
async def get_best_block_height_for_file(self, file_number) -> int:
return await self.db.run(
block_phase.get_best_block_height_for_file, file_number
)
async def sync_blocks(self) -> Optional[Tuple[int, int]]:
tasks = []
starting_height = None
tx_count = block_count = 0
with Progress(self.db.message_queue, BLOCKS_INIT_EVENT) as p:
ending_height = await self.chain.db.get_best_height()
for chain_file in p.iter(await self.chain.db.get_block_files()):
# block files may be read and saved out of order, need to check
# each file individually to see if we have missing blocks
our_best_file_height = await self.get_best_block_height_for_file(
chain_file['file_number']
)
if our_best_file_height == chain_file['best_height']:
# we have all blocks in this file, skipping
continue
if -1 < our_best_file_height < chain_file['best_height']:
# we have some blocks, need to figure out what we're missing
# call get_block_files again limited to this file and current_height
chain_file = (await self.chain.db.get_block_files(
file_number=chain_file['file_number'], start_height=our_best_file_height+1,
))[0]
tx_count += chain_file['txs']
block_count += chain_file['blocks']
file_start_height = chain_file['start_height']
starting_height = min(
file_start_height if starting_height is None else starting_height,
file_start_height
)
tasks.append(self.db.run(
block_phase.sync_block_file, chain_file['file_number'], file_start_height,
chain_file['txs'], self.TX_FLUSH_SIZE
))
with Progress(self.db.message_queue, BLOCKS_MAIN_EVENT) as p:
p.start(block_count, tx_count, extra={
"starting_height": starting_height,
"ending_height": ending_height,
"files": len(tasks),
"claims": await self.chain.db.get_claim_metadata_count(starting_height, ending_height),
"supports": await self.chain.db.get_support_metadata_count(starting_height, ending_height),
})
completed = await self.run_tasks(tasks)
if completed:
if starting_height == 0:
await self.db.run(block_phase.blocks_constraints_and_indexes)
else:
await self.db.run(block_phase.blocks_vacuum)
best_height_processed = max(f.result() for f in completed)
return starting_height, best_height_processed
async def sync_filters(self):
with Progress(self.db.message_queue, FILTER_INIT_EVENT) as p:
p.start(2)
initial_sync = not await self.db.has_filters()
p.step()
if initial_sync:
blocks = [0, await self.db.get_best_block_height()]
else:
blocks = await self.db.run(block_phase.get_block_range_without_filters)
if blocks != (-1, -1):
batches = split_range_into_batches(*blocks)
p.step()
else:
p.step()
return
with Progress(self.db.message_queue, FILTER_MAIN_EVENT) as p:
p.start((blocks[1]-blocks[0])+1)
await self.run_tasks([
self.db.run(block_phase.sync_filters, *batch) for batch in batches
])
if initial_sync:
await self.db.run(block_phase.filters_constraints_and_indexes)
else:
await self.db.run(block_phase.filters_vacuum)
async def sync_spends(self, blocks_added):
if blocks_added:
await self.db.run(block_phase.sync_spends, blocks_added[0] == 0)
async def count_unspent_txos(
self,
txo_types: Tuple[int, ...],
blocks: Tuple[int, int] = None,
missing_in_supports_table: bool = False,
missing_in_claims_table: bool = False,
missing_or_stale_in_claims_table: bool = False,
) -> int:
return await self.db.run(
q.count_unspent_txos, txo_types, blocks,
missing_in_supports_table,
missing_in_claims_table,
missing_or_stale_in_claims_table,
)
async def distribute_unspent_txos(
self,
txo_types: Tuple[int, ...],
blocks: Tuple[int, int] = None,
missing_in_supports_table: bool = False,
missing_in_claims_table: bool = False,
missing_or_stale_in_claims_table: bool = False,
) -> int:
return await self.db.run(
q.distribute_unspent_txos, txo_types, blocks,
missing_in_supports_table,
missing_in_claims_table,
missing_or_stale_in_claims_table,
self.db.workers
)
async def count_abandoned_supports(self) -> int:
return await self.db.run(q.count_abandoned_supports)
async def count_abandoned_claims(self) -> int:
return await self.db.run(q.count_abandoned_claims)
async def count_claims_with_changed_supports(self, blocks) -> int:
return await self.db.run(q.count_claims_with_changed_supports, blocks)
async def count_claims_with_changed_reposts(self, blocks) -> int:
return await self.db.run(q.count_claims_with_changed_reposts, blocks)
async def count_claims_with_stale_signatures(self, blocks) -> int:
return await self.db.run(q.count_claims_with_stale_signatures, blocks)
async def count_channels_with_changed_content(self, blocks) -> int:
return await self.db.run(q.count_channels_with_changed_content, blocks)
async def count_takeovers(self, blocks) -> int:
return await self.chain.db.get_takeover_count(
start_height=blocks[0], end_height=blocks[-1]
)
async def sync_claims(self, blocks) -> bool:
delete_claims = takeovers = claims_with_changed_supports =\
claims_with_changed_reposts = claims_with_stale_signatures = 0
initial_sync = not await self.db.has_claims()
with Progress(self.db.message_queue, CLAIMS_INIT_EVENT) as p:
if initial_sync:
total, batches = await self.distribute_unspent_txos(CLAIM_TYPE_CODES)
elif blocks:
p.start(6)
# 1. content claims to be inserted or updated
total = await self.count_unspent_txos(
CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True
)
batches = [blocks] if total else []
p.step()
# 2. claims to be deleted
delete_claims = await self.count_abandoned_claims()
total += delete_claims
p.step()
# 3. claims to be updated with new support totals
claims_with_changed_supports = await self.count_claims_with_changed_supports(blocks)
total += claims_with_changed_supports
p.step()
# 4. claims to be updated with new repost totals
claims_with_changed_reposts = await self.count_claims_with_changed_reposts(blocks)
total += claims_with_changed_reposts
p.step()
# 5. claims to be updated due to name takeovers
takeovers = await self.count_takeovers(blocks)
total += takeovers
p.step()
# 6. claims where channel signature changed and claim was not re-signed in time
claims_with_stale_signatures = await self.count_claims_with_stale_signatures(blocks)
total += claims_with_stale_signatures
p.step()
else:
return initial_sync
with Progress(self.db.message_queue, CLAIMS_MAIN_EVENT) as p:
p.start(total)
if batches:
await self.run_tasks([
self.db.run(claim_phase.claims_insert, batch, not initial_sync, self.CLAIM_FLUSH_SIZE)
for batch in batches
])
if not initial_sync:
await self.run_tasks([
self.db.run(claim_phase.claims_update, batch) for batch in batches
])
if delete_claims:
await self.db.run(claim_phase.claims_delete, delete_claims)
if takeovers:
await self.db.run(claim_phase.update_takeovers, blocks, takeovers)
if claims_with_changed_supports:
await self.db.run(claim_phase.update_stakes, blocks, claims_with_changed_supports)
if claims_with_changed_reposts:
await self.db.run(claim_phase.update_reposts, blocks, claims_with_changed_reposts)
if claims_with_stale_signatures:
await self.db.run(claim_phase.update_stale_signatures, blocks, claims_with_stale_signatures)
if initial_sync:
await self.db.run(claim_phase.claims_constraints_and_indexes)
else:
await self.db.run(claim_phase.claims_vacuum)
return initial_sync
async def sync_supports(self, blocks):
delete_supports = 0
initial_sync = not await self.db.has_supports()
with Progress(self.db.message_queue, SUPPORTS_INIT_EVENT) as p:
if initial_sync:
total, support_batches = await self.distribute_unspent_txos(TXO_TYPES['support'])
elif blocks:
p.start(2)
# 1. supports to be inserted
total = await self.count_unspent_txos(
TXO_TYPES['support'], blocks, missing_in_supports_table=True
)
support_batches = [blocks] if total else []
p.step()
# 2. supports to be deleted
delete_supports = await self.count_abandoned_supports()
total += delete_supports
p.step()
else:
return
with Progress(self.db.message_queue, SUPPORTS_MAIN_EVENT) as p:
p.start(total)
if support_batches:
await self.run_tasks([
self.db.run(
support_phase.supports_insert, batch, not initial_sync, self.SUPPORT_FLUSH_SIZE
) for batch in support_batches
])
if delete_supports:
await self.db.run(support_phase.supports_delete, delete_supports)
if initial_sync:
await self.db.run(support_phase.supports_constraints_and_indexes)
else:
await self.db.run(support_phase.supports_vacuum)
async def sync_channel_stats(self, blocks, initial_sync):
await self.db.run(claim_phase.update_channel_stats, blocks, initial_sync)
async def sync_trends(self):
ending_height = await self.chain.db.get_best_height()
if ending_height is not None:
await self.db.run(trending.calculate_trending, ending_height)
async def sync_claim_filtering(self):
await self.db.run(
claim_phase.update_claim_filters,
self.resolve_censor_channel_hashes,
self.search_censor_channel_hashes
)
async def advance(self):
blocks_added = await self.sync_blocks()
await self.sync_spends(blocks_added)
await self.sync_filters()
initial_claim_sync = await self.sync_claims(blocks_added)
await self.sync_supports(blocks_added)
await self.sync_channel_stats(blocks_added, initial_claim_sync)
await self.sync_trends()
await self.sync_claim_filtering()
if blocks_added:
await self._on_block_controller.add(BlockEvent(blocks_added[-1]))
async def sync_mempool(self):
added = await self.db.run(block_phase.sync_mempool)
await self.sync_spends([-1])
await self.db.run(claim_phase.claims_insert, [-1, -1], True, self.CLAIM_FLUSH_SIZE)
await self.db.run(claim_phase.claims_update, [-1, -1])
await self.db.run(claim_phase.claims_vacuum)
self.mempool.extend(added)
await self._on_mempool_controller.add(added)
async def clear_mempool(self):
self.mempool.clear()
await self.db.run(block_phase.clear_mempool)
async def advance_loop(self):
while True:
try:
await asyncio.wait([
self.tx_hash_event.wait(),
self.block_hash_event.wait(),
], return_when=asyncio.FIRST_COMPLETED)
if self.block_hash_event.is_set():
self.block_hash_event.clear()
#await self.clear_mempool()
await self.advance()
self.tx_hash_event.clear()
#await self.sync_mempool()
except asyncio.CancelledError:
return
except Exception as e:
log.exception(e)
await self.stop()
async def rewind(self, height):
await self.db.run(block_phase.rewind, height)
+837
View File
@@ -0,0 +1,837 @@
import struct
import hashlib
import logging
import asyncio
from datetime import date
from binascii import hexlify, unhexlify
from typing import List, Iterable, Optional, Union
import ecdsa
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import load_der_public_key
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
from cryptography.exceptions import InvalidSignature
from lbry.crypto.hash import hash160, sha256
from lbry.crypto.base58 import Base58
from lbry.schema.url import normalize_name
from lbry.schema.claim import Claim
from lbry.schema.base import Signable
from lbry.schema.purchase import Purchase
from lbry.schema.support import Support
from .script import InputScript, OutputScript
from .bcd_data_stream import BCDataStream
from .hash import TXRef, TXRefImmutable
from .util import ReadOnlyList
log = logging.getLogger()
class TXRefMutable(TXRef):
__slots__ = ('tx',)
def __init__(self, tx: 'Transaction') -> None:
super().__init__()
self.tx = tx
@property
def id(self):
if self._id is None:
self._id = hexlify(self.hash[::-1]).decode()
return self._id
@property
def hash(self):
if self._hash is None:
self._hash = sha256(sha256(self.tx.raw_sans_segwit))
return self._hash
@property
def height(self):
return self.tx.height
@property
def timestamp(self):
return self.tx.timestamp
def reset(self):
self._id = None
self._hash = None
class TXORef:
__slots__ = 'tx_ref', 'position'
def __init__(self, tx_ref: TXRef, position: int) -> None:
self.tx_ref = tx_ref
self.position = position
@property
def id(self):
return f'{self.tx_ref.id}:{self.position}'
@property
def hash(self):
return self.tx_ref.hash + BCDataStream.uint32.pack(self.position)
@property
def is_null(self):
return self.tx_ref.is_null
@property
def txo(self) -> Optional['Output']:
return None
class TXORefResolvable(TXORef):
__slots__ = ('_txo',)
def __init__(self, txo: 'Output') -> None:
assert txo.tx_ref is not None
assert txo.position is not None
super().__init__(txo.tx_ref, txo.position)
self._txo = txo
@property
def txo(self):
return self._txo
class InputOutput:
__slots__ = 'tx_ref', 'position'
def __init__(self, tx_ref: Union[TXRef, TXRefImmutable] = None, position: int = None) -> None:
self.tx_ref = tx_ref
self.position = position
@property
def size(self) -> int:
""" Size of this input / output in bytes. """
stream = BCDataStream()
self.serialize_to(stream)
return len(stream.get_bytes())
def get_fee(self, ledger):
return self.size * ledger.fee_per_byte
def serialize_to(self, stream, alternate_script=None):
raise NotImplementedError
class Input(InputOutput):
NULL_SIGNATURE = b'\x00'*72
NULL_PUBLIC_KEY = b'\x00'*33
NULL_HASH32 = b'\x00'*32
__slots__ = 'txo_ref', 'sequence', 'coinbase', 'script'
def __init__(self, txo_ref: TXORef, script: InputScript, sequence: int = 0xFFFFFFFF,
tx_ref: TXRef = None, position: int = None) -> None:
super().__init__(tx_ref, position)
self.txo_ref = txo_ref
self.sequence = sequence
self.coinbase = script if txo_ref.is_null else None
self.script = script if not txo_ref.is_null else None
@property
def is_coinbase(self):
return self.coinbase is not None
@classmethod
def spend(cls, txo: 'Output') -> 'Input':
""" Create an input to spend the output."""
assert txo.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.'
script = InputScript.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY)
return cls(txo.ref, script)
@classmethod
def create_coinbase(cls) -> 'Input':
tx_ref = TXRefImmutable.from_hash(cls.NULL_HASH32, 0, 0)
txo_ref = TXORef(tx_ref, 0)
return cls(txo_ref, b'beef')
@property
def amount(self) -> int:
""" Amount this input adds to the transaction. """
if self.txo_ref.txo is None:
raise ValueError('Cannot resolve output to get amount.')
return self.txo_ref.txo.amount
@property
def is_my_input(self) -> Optional[bool]:
""" True if the output this input spends is yours. """
if self.txo_ref.txo is None:
return False
return self.txo_ref.txo.is_my_output
@classmethod
def deserialize_from(cls, stream):
tx_ref = TXRefImmutable.from_hash(stream.read(32), -1, -1)
position = stream.read_uint32()
script = stream.read_string()
sequence = stream.read_uint32()
return cls(
TXORef(tx_ref, position),
InputScript(script) if not tx_ref.is_null else script,
sequence
)
def serialize_to(self, stream, alternate_script=None):
stream.write(self.txo_ref.tx_ref.hash)
stream.write_uint32(self.txo_ref.position)
if alternate_script is not None:
stream.write_string(alternate_script)
else:
if self.is_coinbase:
stream.write_string(self.coinbase)
else:
stream.write_string(self.script.source)
stream.write_uint32(self.sequence)
class Output(InputOutput):
__slots__ = (
'amount', 'script', 'is_internal_transfer', 'spent_height', 'is_my_output', 'is_my_input',
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims', '_signable'
)
def __init__(self, amount: int, script: OutputScript,
tx_ref: TXRef = None, position: int = None,
is_internal_transfer: Optional[bool] = None, spent_height: Optional[bool] = None,
is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None,
sent_supports: Optional[int] = None, sent_tips: Optional[int] = None,
received_tips: Optional[int] = None,
channel: Optional['Output'] = None, private_key: Optional[str] = None
) -> None:
super().__init__(tx_ref, position)
self.amount = amount
self.script = script
self.is_internal_transfer = is_internal_transfer
self.spent_height = spent_height
self.is_my_output = is_my_output
self.is_my_input = is_my_input
self.sent_supports = sent_supports
self.sent_tips = sent_tips
self.received_tips = received_tips
self.channel = channel
self.private_key = private_key
self.purchase: 'Output' = None # txo containing purchase metadata
self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
self.reposted_claim: 'Output' = None # txo representing claim being reposted
self.claims: List['Output'] = None # resolved claims for collection
self._signable: Optional[Signable] = None
self.meta = {}
def update_annotations(self, annotated: 'Output'):
if annotated is None:
self.is_internal_transfer = None
self.spent_height = None
self.is_my_output = None
self.is_my_input = None
self.sent_supports = None
self.sent_tips = None
self.received_tips = None
else:
self.is_internal_transfer = annotated.is_internal_transfer
self.spent_height = annotated.spent_height
self.is_my_output = annotated.is_my_output
self.is_my_input = annotated.is_my_input
self.sent_supports = annotated.sent_supports
self.sent_tips = annotated.sent_tips
self.received_tips = annotated.received_tips
self.channel = annotated.channel if annotated else None
self.private_key = annotated.private_key if annotated else None
@property
def ref(self):
return TXORefResolvable(self)
@property
def id(self):
return self.ref.id
@property
def hash(self):
return self.ref.hash
@property
def is_spent(self):
if self.spent_height is not None:
return self.spent_height > 0
@property
def pubkey_hash(self):
pubkey_hash = self.script.values.get('pubkey_hash')
if pubkey_hash:
return pubkey_hash
return hash160(self.script.values['pubkey'])
@property
def has_address(self):
return (
'pubkey_hash' in self.script.values or
'script_hash' in self.script.values or
'pubkey' in self.script.values
)
def get_address(self, ledger):
if 'script_hash' in self.script.values:
return ledger.script_hash_to_address(self.script.values['script_hash'])
return ledger.pubkey_hash_to_address(self.pubkey_hash)
@classmethod
def pay_pubkey_hash(cls, amount, pubkey_hash):
return cls(amount, OutputScript.pay_pubkey_hash(pubkey_hash))
@classmethod
def deserialize_from(cls, stream, transaction_offset: int = 0):
amount = stream.read_uint64()
length = stream.read_compact_size()
offset = stream.tell()-transaction_offset
script = OutputScript(stream.read(length), offset=offset)
return cls(amount=amount, script=script)
def serialize_to(self, stream, alternate_script=None):
stream.write_uint64(self.amount)
stream.write_string(self.script.source)
def get_fee(self, ledger):
name_fee = 0
if self.script.is_claim_name:
name_fee = len(self.script.values['claim_name']) * ledger.fee_per_name_char
return max(name_fee, super().get_fee(ledger))
@property
def is_claim(self) -> bool:
return self.script.is_claim_name or self.script.is_update_claim
@property
def is_support(self) -> bool:
return self.script.is_support_claim
@property
def is_support_data(self) -> bool:
return self.script.is_support_claim_data
@property
def claim_hash(self) -> bytes:
if self.script.is_claim_name:
return hash160(self.tx_ref.hash + struct.pack('>I', self.position))
elif self.script.is_update_claim or self.script.is_support_claim:
return self.script.values['claim_id']
else:
raise ValueError('No claim_id associated.')
@property
def claim_id(self) -> str:
return hexlify(self.claim_hash[::-1]).decode()
@property
def claim_name(self) -> str:
if self.script.is_claim_involved:
return self.script.values['claim_name'].decode()
raise ValueError('No claim_name associated.')
@property
def normalized_name(self) -> str:
return normalize_name(self.claim_name)
@property
def claim(self) -> Claim:
if self.is_claim:
if not isinstance(self.script.values['claim'], Claim):
self.script.values['claim'] = Claim.from_bytes(self.script.values['claim'])
return self.script.values['claim']
raise ValueError('Only claim name and claim update have the claim payload.')
@property
def can_decode_claim(self):
try:
return self.claim
except Exception:
return False
@property
def support(self) -> Support:
if self.is_support_data:
if not isinstance(self.script.values['support'], Support):
self.script.values['support'] = Support.from_bytes(self.script.values['support'])
return self.script.values['support']
raise ValueError('Only supports with data can be represented as Supports.')
@property
def can_decode_support(self):
try:
return self.support
except Exception:
return False
@property
def signable(self) -> Signable:
if self._signable is None:
if self.is_claim:
self._signable = self.claim
elif self.is_support_data:
self._signable = self.support
return self._signable
@property
def can_decode_signable(self) -> Signable:
try:
return self.signable
except Exception:
return False
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
return f"lbry://{self.claim_name}#{self.claim_id}"
raise ValueError('No claim associated.')
@property
def has_private_key(self):
return self.private_key is not None
def get_signature_digest(self, ledger):
if self.signable.unsigned_payload:
pieces = [
Base58.decode(self.get_address(ledger)),
self.signable.unsigned_payload,
self.signable.signing_channel_hash[::-1]
]
else:
pieces = [
self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
]
return sha256(b''.join(pieces))
def get_encoded_signature(self):
signature = hexlify(self.signable.signature)
r = int(signature[:int(len(signature)/2)], 16)
s = int(signature[int(len(signature)/2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
@staticmethod
def is_signature_valid(encoded_signature, signature_digest, public_key_bytes):
try:
public_key = load_der_public_key(public_key_bytes, default_backend())
public_key.verify( # pylint: disable=no-value-for-parameter
encoded_signature, signature_digest,
ec.ECDSA(Prehashed(hashes.SHA256()))
)
return True
except (ValueError, InvalidSignature):
pass
return False
def is_signed_by(self, channel: 'Output', ledger=None):
return self.is_signature_valid(
self.get_encoded_signature(),
self.get_signature_digest(ledger),
channel.claim.channel.public_key_bytes
)
def sign(self, channel: 'Output', first_input_id=None):
self.channel = channel
self.signable.signing_channel_hash = channel.claim_hash
digest = sha256(b''.join([
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
self.signable.signing_channel_hash,
self.signable.to_message_bytes()
]))
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
self.script.generate()
def clear_signature(self):
self.channel = None
self.claim.clear_signature()
@staticmethod
def _sync_generate_channel_private_key():
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
public_key_bytes = private_key.get_verifying_key().to_der()
return private_key, public_key_bytes
async def generate_channel_private_key(self):
private_key, public_key_bytes = await asyncio.get_running_loop().run_in_executor(
None, Output._sync_generate_channel_private_key
)
self.private_key = private_key
self.claim.channel.public_key_bytes = public_key_bytes
self.script.generate()
return self.private_key
def is_channel_private_key(self, private_key):
return self.claim.channel.public_key_bytes == private_key.get_verifying_key().to_der()
@classmethod
def pay_claim_name_pubkey_hash(
cls, amount: int, claim_name: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_claim_name_pubkey_hash(
claim_name.encode(), claim, pubkey_hash)
return cls(amount, script)
@classmethod
def pay_update_claim_pubkey_hash(
cls, amount: int, claim_name: str, claim_id: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_update_claim_pubkey_hash(
claim_name.encode(), unhexlify(claim_id)[::-1], claim, pubkey_hash
)
return cls(amount, script)
@classmethod
def pay_support_pubkey_hash(cls, amount: int, claim_name: str, claim_id: str, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_support_pubkey_hash(
claim_name.encode(), unhexlify(claim_id)[::-1], pubkey_hash
)
return cls(amount, script)
@classmethod
def pay_support_data_pubkey_hash(
cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output':
script = OutputScript.pay_support_data_pubkey_hash(
claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash
)
return cls(amount, script)
@classmethod
def add_purchase_data(cls, purchase: Purchase) -> 'Output':
script = OutputScript.return_data(purchase)
return cls(0, script)
@property
def is_purchase_data(self) -> bool:
return self.script.is_return_data and (
isinstance(self.script.values['data'], Purchase) or
Purchase.has_start_byte(self.script.values['data'])
)
@property
def purchase_data(self) -> Purchase:
if self.is_purchase_data:
if not isinstance(self.script.values['data'], Purchase):
self.script.values['data'] = Purchase.from_bytes(self.script.values['data'])
return self.script.values['data']
raise ValueError('Output does not have purchase data.')
@property
def can_decode_purchase_data(self):
try:
return self.purchase_data
except: # pylint: disable=bare-except
return False
@property
def purchased_claim_id(self):
if self.purchase is not None:
return self.purchase.purchase_data.claim_id
if self.purchased_claim is not None:
return self.purchased_claim.claim_id
@property
def purchased_claim_hash(self):
if self.purchase is not None:
return self.purchase.purchase_data.claim_hash
if self.purchased_claim is not None:
return self.purchased_claim.claim_hash
@property
def has_price(self):
if self.can_decode_claim:
claim = self.claim
if claim.is_stream:
stream = claim.stream
return stream.has_fee and stream.fee.amount and stream.fee.amount > 0
return False
@property
def price(self):
return self.claim.stream.fee
class Transaction:
def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
height: int = -2, position: int = -1, timestamp: int = 0) -> None:
self._raw = raw
self._raw_sans_segwit = None
self.is_segwit_flag = 0
self.witnesses: List[bytes] = []
self.ref = TXRefMutable(self)
self.version = version
self.locktime = locktime
self._inputs: List[Input] = []
self._outputs: List[Output] = []
self.is_verified = is_verified
# Height Progression
# -2: not broadcast
# -1: in mempool but has unconfirmed inputs
# 0: in mempool and all inputs confirmed
# +num: confirmed in a specific block (height)
self.height = height
self.position = position
self.timestamp = timestamp
self._day: int = 0
if raw is not None:
self.deserialize()
def __repr__(self):
return f"TX({self.id[:10]}...{self.id[-10:]})"
@property
def is_broadcast(self):
return self.height > -2
@property
def is_mempool(self):
return self.height in (-1, 0)
@property
def is_confirmed(self):
return self.height > 0
@property
def id(self):
return self.ref.id
@property
def hash(self):
return self.ref.hash
@property
def day(self):
if self._day is None and self.timestamp > 0:
self._day = date.fromtimestamp(self.timestamp).toordinal()
return self._day
@property
def raw(self):
if self._raw is None:
self._raw = self._serialize()
return self._raw
@property
def raw_sans_segwit(self):
if self.is_segwit_flag:
if self._raw_sans_segwit is None:
self._raw_sans_segwit = self._serialize(sans_segwit=True)
return self._raw_sans_segwit
return self.raw
def _reset(self):
self._raw = None
self._raw_sans_segwit = None
self.ref.reset()
@property
def inputs(self) -> ReadOnlyList[Input]:
return ReadOnlyList(self._inputs)
@property
def outputs(self) -> ReadOnlyList[Output]:
return ReadOnlyList(self._outputs)
def _add(self, existing_ios: List, new_ios: Iterable[InputOutput], reset=False) -> 'Transaction':
for txio in new_ios:
txio.tx_ref = self.ref
txio.position = len(existing_ios)
existing_ios.append(txio)
if reset:
self._reset()
return self
def add_inputs(self, inputs: Iterable[Input]) -> 'Transaction':
return self._add(self._inputs, inputs, True)
def add_outputs(self, outputs: Iterable[Output]) -> 'Transaction':
return self._add(self._outputs, outputs, True)
@property
def size(self) -> int:
""" Size in bytes of the entire transaction. """
return len(self.raw)
@property
def base_size(self) -> int:
""" Size of transaction without inputs or outputs in bytes. """
return (
self.size
- sum(txi.size for txi in self._inputs)
- sum(txo.size for txo in self._outputs)
)
@property
def input_sum(self):
return sum(i.amount for i in self.inputs if i.txo_ref.txo is not None)
@property
def output_sum(self):
return sum(o.amount for o in self.outputs)
@property
def net_account_balance(self) -> int:
balance = 0
for txi in self.inputs:
if txi.txo_ref.txo is None:
continue
if txi.is_my_input is True:
balance -= txi.amount
elif txi.is_my_input is None:
raise ValueError(
"Cannot access net_account_balance if inputs do not "
"have is_my_input set properly."
)
for txo in self.outputs:
if txo.is_my_output is True:
balance += txo.amount
elif txo.is_my_output is None:
raise ValueError(
"Cannot access net_account_balance if outputs do not "
"have is_my_output set properly."
)
return balance
@property
def fee(self) -> int:
return self.input_sum - self.output_sum
def get_base_fee(self, ledger) -> int:
""" Fee for base tx excluding inputs and outputs. """
return self.base_size * ledger.fee_per_byte
def get_effective_input_sum(self, ledger) -> int:
""" Sum of input values *minus* the cost involved to spend them. """
return sum(txi.amount - txi.get_fee(ledger) for txi in self._inputs)
def get_total_output_sum(self, ledger) -> int:
""" Sum of output values *plus* the cost involved to spend them. """
return sum(txo.amount + txo.get_fee(ledger) for txo in self._outputs)
def _serialize(self, with_inputs: bool = True, sans_segwit: bool = False) -> bytes:
stream = BCDataStream()
stream.write_uint32(self.version)
if with_inputs:
stream.write_compact_size(len(self._inputs))
for txin in self._inputs:
txin.serialize_to(stream)
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
stream.write_uint32(self.locktime)
return stream.get_bytes()
def _serialize_for_signature(self, signing_input: int) -> bytes:
stream = BCDataStream()
stream.write_uint32(self.version)
stream.write_compact_size(len(self._inputs))
for i, txin in enumerate(self._inputs):
if signing_input == i:
assert txin.txo_ref.txo is not None
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
else:
txin.serialize_to(stream, b'')
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
stream.write_uint32(self.locktime)
stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL
return stream.get_bytes()
def deserialize(self, stream=None):
if self._raw is not None or stream is not None:
stream = stream or BCDataStream(self._raw)
start = stream.tell()
self.version = stream.read_uint32()
input_count = stream.read_compact_size()
if input_count == 0:
self.is_segwit_flag = stream.read_uint8()
input_count = stream.read_compact_size()
self._add(self._inputs, [
Input.deserialize_from(stream) for _ in range(input_count)
])
output_count = stream.read_compact_size()
self._add(self._outputs, [
Output.deserialize_from(stream, start) for _ in range(output_count)
])
if self.is_segwit_flag:
# drain witness portion of transaction
# too many witnesses for no crime
self.witnesses = []
for _ in range(input_count):
for _ in range(stream.read_compact_size()):
self.witnesses.append(stream.read(stream.read_compact_size()))
self.locktime = stream.read_uint32()
return self
@staticmethod
def signature_hash_type(hash_type):
return hash_type
@property
def my_inputs(self):
for txi in self.inputs:
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output:
yield txi
def _filter_my_outputs(self, f):
for txo in self.outputs:
if txo.is_my_output and f(txo.script):
yield txo
def _filter_other_outputs(self, f):
for txo in self.outputs:
if not txo.is_my_output and f(txo.script):
yield txo
def _filter_any_outputs(self, f):
for txo in self.outputs:
if f(txo):
yield txo
@property
def my_claim_outputs(self):
return self._filter_my_outputs(lambda s: s.is_claim_name)
@property
def my_update_outputs(self):
return self._filter_my_outputs(lambda s: s.is_update_claim)
@property
def my_support_outputs(self):
return self._filter_my_outputs(lambda s: s.is_support_claim)
@property
def any_purchase_outputs(self):
return self._filter_any_outputs(lambda o: o.purchase is not None)
@property
def other_support_outputs(self):
return self._filter_other_outputs(lambda s: s.is_support_claim)
@property
def my_abandon_outputs(self):
for txi in self.inputs:
abandon = txi.txo_ref.txo
if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved:
is_update = False
if abandon.script.is_claim_name or abandon.script.is_update_claim:
for update in self.my_update_outputs:
if abandon.claim_id == update.claim_id:
is_update = True
break
if not is_update:
yield abandon
@@ -1,25 +1,4 @@
import re
from binascii import unhexlify, hexlify
from typing import TypeVar, Sequence, Optional
from torba.client.constants import COIN
def coins_to_satoshis(coins):
if not isinstance(coins, str):
raise ValueError("{coins} must be a string")
result = re.search(r'^(\d{1,10})\.(\d{1,8})$', coins)
if result is not None:
whole, fractional = result.groups()
return int(whole+fractional.ljust(8, "0"))
raise ValueError("'{lbc}' is not a valid coin decimal")
def satoshis_to_coins(satoshis):
coins = '{:.8f}'.format(satoshis / COIN).rstrip('0')
if coins.endswith('.'):
return coins+'0'
else:
return coins
T = TypeVar('T')
@@ -37,34 +16,6 @@ class ReadOnlyList(Sequence[T]):
return len(self.lst)
def subclass_tuple(name, base):
return type(name, (base,), {'__slots__': ()})
class cachedproperty:
def __init__(self, f):
self.f = f
def __get__(self, obj, objtype):
obj = obj or objtype
value = self.f(obj)
setattr(obj, self.f.__name__, value)
return value
def bytes_to_int(be_bytes):
""" Interprets a big-endian sequence of bytes as an integer. """
return int(hexlify(be_bytes), 16)
def int_to_bytes(value):
""" Converts an integer to a big-endian sequence of bytes. """
length = (value.bit_length() + 7) // 8
s = '%x' % value
return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2))
class ArithUint256:
# https://github.com/bitcoin/bitcoin/blob/master/src/arith_uint256.cpp
@@ -1,3 +1,4 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
COMMIT_HASH = "none"
DOCKER_TAG = "none"
+113 -153
View File
@@ -1,78 +1,17 @@
import os
import sys
import signal
import pathlib
import json
import asyncio
import pathlib
import argparse
import logging
import logging.handlers
import textwrap
import subprocess
import aiohttp
from aiohttp.web import GracefulExit
from docopt import docopt
from lbry import __version__ as lbrynet_version
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry import __version__
from lbry.conf import Config, CLIConfig
from lbry.extras.daemon.Daemon import Daemon
log = logging.getLogger('lbry')
def display(data):
print(json.dumps(data, indent=2))
async def execute_command(conf, method, params, callback=display):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
async with session.get(conf.api_connection_url, json=message) as resp:
try:
data = await resp.json()
if 'result' in data:
return callback(data['result'])
elif 'error' in data:
return callback(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
print("Could not connect to daemon. Are you sure it's running?")
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
if x.lower() == 'false':
return False
if x.isdigit():
return int(x)
return x
def remove_brackets(key):
if key.startswith("<") and key.endswith(">"):
return str(key[1:-1])
return key
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
if arg is None:
continue
k = None
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
kwargs[k] = normalize_value(arg, k)
return kwargs
from lbry.service import Daemon, Client, FullNode, LightClient
from lbry.service.metadata import interface
def split_subparser_argument(parent, original, name, condition):
@@ -101,7 +40,7 @@ class ArgumentParser(argparse.ArgumentParser):
self._optionals.title = 'Options'
if group_name is None:
self.epilog = (
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
"Run 'lbrynet COMMAND --help' for more information on a command or group."
)
else:
self.epilog = (
@@ -153,66 +92,76 @@ class HelpFormatter(argparse.HelpFormatter):
)
def add_command_parser(parent, command):
subcommand = parent.add_parser(
command['name'],
help=command['doc'].strip().splitlines()[0]
)
subcommand.set_defaults(
api_method_name=command['api_method_name'],
command=command['name'],
doc=command['doc'],
replaced_by=command.get('replaced_by', None)
)
def add_command_parser(parent, method_name, command):
short = command['desc']['text'][0] if command['desc'] else ''
subcommand = parent.add_parser(command['name'], help=short)
subcommand.set_defaults(api_method_name=method_name, command=command['name'], doc=command['help'])
def get_argument_parser():
main = ArgumentParser(
root = ArgumentParser(
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
)
main.add_argument(
root.add_argument(
'-v', '--version', dest='cli_version', action="store_true",
help='Show lbrynet CLI version and exit.'
)
main.set_defaults(group=None, command=None)
CLIConfig.contribute_to_argparse(main)
sub = main.add_subparsers(metavar='COMMAND')
root.set_defaults(group=None, command=None)
CLIConfig.contribute_to_argparse(root)
sub = root.add_subparsers(metavar='COMMAND')
start = sub.add_parser(
'start',
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
help='Start LBRY Network interface.'
)
start.add_argument(
"service", choices=[LightClient.name, FullNode.name], default=LightClient.name, nargs="?"
)
start.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
start.add_argument(
'--no-logging', dest='no_logging', action="store_true",
help='Disable all logging of any kind.'
)
start.add_argument(
'--verbose', nargs="*",
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
start.add_argument(
'--initial-headers', dest='initial_headers',
help='Specify path to initial blockchain headers, faster than downloading them on first run.'
)
install = sub.add_parser("install", help="Install lbrynet with various system services.")
install.add_argument("system", choices=["systemd"])
install.add_argument(
"--global", dest="install_global", action="store_true",
help="Install system wide (requires running as root), default is for current user only."
)
Config.contribute_to_argparse(start)
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
install.set_defaults(command='install', install_parser=install, doc=install.format_help())
api = Daemon.get_api_definitions()
groups = {}
for group_name in sorted(api['groups']):
group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name])
for group_name in sorted(interface['groups']):
group_parser = sub.add_parser(group_name, group_name=group_name, help=interface['groups'][group_name])
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
nicer_order = ['stop', 'get', 'publish', 'resolve']
for command_name in sorted(api['commands']):
for command_name in sorted(interface['commands']):
if command_name not in nicer_order:
nicer_order.append(command_name)
for command_name in nicer_order:
command = api['commands'][command_name]
if command['group'] is None:
add_command_parser(sub, command)
command = interface['commands'][command_name]
if command.get('group') is None:
add_command_parser(sub, command_name, command)
else:
add_command_parser(groups[command['group']], command)
add_command_parser(groups[command['group']], command_name, command)
return main
return root
def ensure_directory_exists(path: str):
@@ -220,66 +169,64 @@ def ensure_directory_exists(path: str):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
LOG_MODULES = ('lbry', 'torba', 'aioupnp')
async def execute_command(conf, method, params):
client = Client(f"http://{conf.api}/ws")
await client.connect()
responses = await client.send(method, **params)
result = await responses.first
await client.disconnect()
print(result)
return result
def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config):
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
file_handler = logging.handlers.RotatingFileHandler(conf.log_file_path, maxBytes=2097152, backupCount=5)
file_handler.setFormatter(default_formatter)
for module_name in LOG_MODULES:
logger.getChild(module_name).addHandler(file_handler)
if not args.quiet:
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
for module_name in LOG_MODULES:
logger.getChild(module_name).addHandler(handler)
logger.getChild('lbry').setLevel(logging.INFO)
logger.getChild('torba').setLevel(logging.INFO)
logger.getChild('aioupnp').setLevel(logging.WARNING)
logger.getChild('aiohttp').setLevel(logging.CRITICAL)
if args.verbose is not None:
if len(args.verbose) > 0:
for module in args.verbose:
logger.getChild(module).setLevel(logging.DEBUG)
else:
logger.getChild('lbry').setLevel(logging.DEBUG)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
return x
if x.lower() == 'true':
return True
if x.lower() == 'false':
return False
if x.isdigit():
return int(x)
return x
def run_daemon(args: argparse.Namespace, conf: Config):
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
setup_logging(logging.getLogger(), args, conf)
daemon = Daemon(conf)
def remove_brackets(key):
if key.startswith("<") and key.endswith(">"):
return str(key[1:-1])
return key
def __exit():
raise GracefulExit()
try:
loop.add_signal_handler(signal.SIGINT, __exit)
loop.add_signal_handler(signal.SIGTERM, __exit)
except NotImplementedError:
pass # Not implemented on Windows
def set_kwargs(parsed_args):
kwargs = {}
for key, arg in parsed_args.items():
if arg is None:
continue
k = None
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
k = remove_brackets(key[2:])
elif remove_brackets(key) not in kwargs:
k = remove_brackets(key)
kwargs[k] = normalize_value(arg, k)
return kwargs
try:
loop.run_until_complete(daemon.start())
loop.run_forever()
except (GracefulExit, KeyboardInterrupt, asyncio.CancelledError):
pass
finally:
loop.run_until_complete(daemon.stop())
logging.shutdown()
if hasattr(loop, 'shutdown_asyncgens'):
loop.run_until_complete(loop.shutdown_asyncgens())
def install_systemd_service():
systemd_service = textwrap.dedent(f"""\
[Unit]
Description=LBRYnet
[Service]
Type=simple
ExecStart={sys.argv[0]} start --full-node
""")
subprocess.run(
["systemctl", "edit", "--user", "--full", "--force", "lbrynet.service"],
input=systemd_service, text=True, check=True,
env=dict(os.environ, SYSTEMD_EDITOR="cp /dev/stdin"),
)
def main(argv=None):
@@ -287,28 +234,41 @@ def main(argv=None):
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
conf = Config()
conf.set_arguments(args)
conf.set_environment()
conf.set_default_paths()
conf.set_persisted()
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
ensure_directory_exists(directory)
if args.cli_version:
print(f"lbrynet {lbrynet_version}")
print(f"lbrynet {__version__}")
elif args.command == 'start':
if args.help:
args.start_parser.print_help()
elif args.service == FullNode.name:
return Daemon.from_config(FullNode, conf).run()
elif args.service == LightClient.name:
return Daemon.from_config(LightClient, conf).run()
else:
run_daemon(args, conf)
print(f'Only `start {FullNode.name}` or `start {LightClient.name}` is currently supported.')
elif args.command == 'install':
if args.help:
args.install_parser.print_help()
elif args.system == 'systemd':
install_systemd_service()
elif args.command is not None:
doc = args.doc
api_method_name = args.api_method_name
if args.replaced_by:
print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.")
doc = args.replaced_by['doc']
api_method_name = args.replaced_by['api_method_name']
if args.help:
print(doc)
else:
parsed = docopt(doc, command_args)
parsed = docopt(
# TODO: ugly hack because docopt doesn't support commands with spaces in them
doc.replace(api_method_name.replace('_', ' '), api_method_name, 1),
command_args
)
params = set_kwargs(parsed)
asyncio.get_event_loop().run_until_complete(execute_command(conf, api_method_name, params))
elif args.group is not None:
+193 -110
View File
@@ -1,20 +1,22 @@
import os
import re
import sys
import typing
import logging
import yaml
from argparse import ArgumentParser
from contextlib import contextmanager
from appdirs import user_data_dir, user_config_dir
from typing import Tuple
import yaml
from lbry.utils.dirs import user_data_dir, user_download_dir
from lbry.error import InvalidCurrencyError
from lbry.dht import constants
from torba.client.coinselection import STRATEGIES
from lbry.wallet.coinselection import COIN_SELECTION_STRATEGIES
from lbry.event import EventRegistry
log = logging.getLogger(__name__)
NOT_SET = type('NOT_SET', (object,), {})
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
T = typing.TypeVar('T')
CURRENCIES = {
@@ -35,7 +37,7 @@ class Setting(typing.Generic[T]):
self.metavar = metavar
def __set_name__(self, owner, name):
self.name = name
self.name = name # pylint: disable=attribute-defined-outside-init
@property
def cli_name(self):
@@ -63,13 +65,13 @@ class Setting(typing.Generic[T]):
for location in obj.modify_order:
location[self.name] = val
def validate(self, val):
def validate(self, value):
raise NotImplementedError()
def deserialize(self, value):
def deserialize(self, value): # pylint: disable=no-self-use
return value
def serialize(self, value):
def serialize(self, value): # pylint: disable=no-self-use
return value
def contribute_to_argparse(self, parser: ArgumentParser):
@@ -82,14 +84,18 @@ class Setting(typing.Generic[T]):
class String(Setting[str]):
def validate(self, val):
assert isinstance(val, str), \
def validate(self, value):
assert isinstance(value, str), \
f"Setting '{self.name}' must be a string."
# TODO: removes this after pylint starts to understand generics
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
return super().__get__(obj, owner)
class Integer(Setting[int]):
def validate(self, val):
assert isinstance(val, int), \
def validate(self, value):
assert isinstance(value, int), \
f"Setting '{self.name}' must be an integer."
def deserialize(self, value):
@@ -97,8 +103,8 @@ class Integer(Setting[int]):
class Float(Setting[float]):
def validate(self, val):
assert isinstance(val, float), \
def validate(self, value):
assert isinstance(value, float), \
f"Setting '{self.name}' must be a decimal."
def deserialize(self, value):
@@ -106,8 +112,8 @@ class Float(Setting[float]):
class Toggle(Setting[bool]):
def validate(self, val):
assert isinstance(val, bool), \
def validate(self, value):
assert isinstance(value, bool), \
f"Setting '{self.name}' must be a true/false value."
def contribute_to_argparse(self, parser: ArgumentParser):
@@ -127,10 +133,10 @@ class Toggle(Setting[bool]):
class Path(String):
def __init__(self, doc: str, default: str = '', *args, **kwargs):
def __init__(self, doc: str, *args, default: str = '', **kwargs):
super().__init__(doc, default, *args, **kwargs)
def __get__(self, obj, owner):
def __get__(self, obj, owner) -> str:
value = super().__get__(obj, owner)
if isinstance(value, str):
return os.path.expanduser(os.path.expandvars(value))
@@ -187,7 +193,7 @@ class MaxKeyFee(Setting[dict]):
)
parser.add_argument(
self.no_cli_name,
help=f"Disable maximum key fee check.",
help="Disable maximum key fee check.",
dest=self.name,
const=None,
action="store_const",
@@ -204,16 +210,16 @@ class StringChoice(String):
raise ValueError(f"Default value must be one of: {', '.join(valid_values)}")
self.valid_values = valid_values
def validate(self, val):
super().validate(val)
if val not in self.valid_values:
def validate(self, value):
super().validate(value)
if value not in self.valid_values:
raise ValueError(f"Setting '{self.name}' value must be one of: {', '.join(self.valid_values)}")
class ListSetting(Setting[list]):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
def validate(self, value):
assert isinstance(value, (tuple, list)), \
f"Setting '{self.name}' must be a tuple or list."
def contribute_to_argparse(self, parser: ArgumentParser):
@@ -226,10 +232,10 @@ class ListSetting(Setting[list]):
class Servers(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
def validate(self, value):
assert isinstance(value, (tuple, list)), \
f"Setting '{self.name}' must be a tuple or list of servers."
for idx, server in enumerate(val):
for idx, server in enumerate(value):
assert isinstance(server, (tuple, list)) and len(server) == 2, \
f"Server defined '{server}' at index {idx} in setting " \
f"'{self.name}' must be a tuple or list of two items."
@@ -260,10 +266,10 @@ class Servers(ListSetting):
class Strings(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
def validate(self, value):
assert isinstance(value, (tuple, list)), \
f"Setting '{self.name}' must be a tuple or list of strings."
for idx, string in enumerate(val):
for idx, string in enumerate(value):
assert isinstance(string, str), \
f"Value of '{string}' at index {idx} in setting " \
f"'{self.name}' must be a string."
@@ -320,7 +326,7 @@ class ConfigFileAccess:
cls = type(self.configuration)
with open(self.path, 'r') as config_file:
raw = config_file.read()
serialized = yaml.load(raw) or {}
serialized = yaml.full_load(raw) or {}
for key, value in serialized.items():
attr = getattr(cls, key, None)
if attr is None:
@@ -377,8 +383,13 @@ class BaseConfig:
self.environment = {} # from environment variables
self.persisted = {} # from config file
self._updating_config = False
self.events = EventRegistry()
self.set(**kwargs)
def set(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
return self
@contextmanager
def update_config(self):
@@ -457,7 +468,27 @@ class BaseConfig:
self.persisted.save()
class CLIConfig(BaseConfig):
class TranscodeConfig(BaseConfig):
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
'', previous_names=['ffmpeg_folder'])
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
class CLIConfig(TranscodeConfig):
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
@@ -475,19 +506,30 @@ class CLIConfig(BaseConfig):
class Config(CLIConfig):
db_url = String("Database connection URL, uses a local file based SQLite by default.")
workers = Integer(
"Multiprocessing, specify number of worker processes lbrynet can start (including main process)."
" (-1: threads only, 0: equal to number of CPUs, >1: specific number of processes)", -1
)
console = StringChoice(
"Basic text console output or advanced colored output with progress bars.",
["basic", "advanced", "none"], "advanced"
)
# directories
data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path(
"Directory path to place assembled files downloaded from LBRY.",
previous_names=['download_directory'], metavar='DIR'
)
wallet_dir = Path(
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
download_dir = Path("Directory to store downloaded files.", metavar='DIR')
data_dir = Path("Main directory containing blobs, wallets and blockchain data.", metavar='DIR')
blob_dir = Path("Directory to store blobs (default: 'data_dir'/blobs).", metavar='DIR')
wallet_dir = Path("Directory to store wallets (default: 'data_dir'/wallets).", metavar='DIR')
wallet_storage = StringChoice("Wallet storage mode.", ["file", "database"], "file")
wallets = Strings(
"Wallet files in 'wallet_dir' to load at startup.",
['default_wallet']
"Wallet files in 'wallet_dir' to load at startup.", ['default_wallet']
)
create_default_wallet = Toggle(
"Create an initial wallet if it does not exist on startup.", True
)
create_default_account = Toggle(
"Create an initial account if it does not exist in the default wallet.", True
)
# network
@@ -497,6 +539,7 @@ class Config(CLIConfig):
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# routing table
@@ -511,7 +554,7 @@ class Config(CLIConfig):
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
@@ -539,6 +582,7 @@ class Config(CLIConfig):
"Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
)
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
# reflector settings
reflect_streams = Toggle(
@@ -553,16 +597,16 @@ class Config(CLIConfig):
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv1.lbry.com', 50001),
('spv2.lbry.com', 50001),
('spv3.lbry.com', 50001),
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
full_nodes = Servers("Full blockchain nodes", [
('sdk.lbry.tech', 5278),
('spv12.lbry.com', 50001),
('spv13.lbry.com', 50001),
('spv14.lbry.com', 50001),
('spv15.lbry.com', 50001),
('spv16.lbry.com', 50001),
('spv17.lbry.com', 50001),
('spv18.lbry.com', 50001),
('spv19.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
@@ -574,15 +618,22 @@ class Config(CLIConfig):
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
blockchain = StringChoice("Blockchain network type.", ["mainnet", "regtest", "testnet"], "mainnet")
lbrycrd_rpc_user = String("Username for connecting to lbrycrd.", "rpcuser")
lbrycrd_rpc_pass = String("Password for connecting to lbrycrd.", "rpcpassword")
lbrycrd_rpc_host = String("Hostname for connecting to lbrycrd.", "localhost")
lbrycrd_rpc_port = Integer("Port for connecting to lbrycrd.", 9245)
lbrycrd_peer_port = Integer("Peer port for lbrycrd.", 9246)
lbrycrd_zmq = String("ZMQ events address.")
lbrycrd_dir = Path("Directory containing lbrycrd data.", metavar='DIR')
search_censor_channel_ids = Strings("List of channel ids for filtering out search results.", [])
resolve_censor_channel_ids = Strings("List of channel ids for filtering out resolve results.", [])
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
"Whether to share usage stats and diagnostic info with LBRY.", False,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
@@ -595,7 +646,12 @@ class Config(CLIConfig):
coin_selection_strategy = StringChoice(
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
COIN_SELECTION_STRATEGIES, "standard")
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
)
@property
def streaming_host(self):
@@ -605,9 +661,18 @@ class Config(CLIConfig):
def streaming_port(self):
return int(self.streaming_server.split(':')[1])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.set_default_paths()
@classmethod
def with_null_dir(cls):
return cls.with_same_dir('/dev/null')
@classmethod
def with_same_dir(cls, same_dir):
return cls(
data_dir=same_dir,
download_dir=same_dir,
wallet_dir=same_dir,
lbrycrd_dir=same_dir,
)
def set_default_paths(self):
if 'darwin' in sys.platform.lower():
@@ -619,58 +684,76 @@ class Config(CLIConfig):
else:
return
cls = type(self)
cls.data_dir.default, cls.wallet_dir.default, cls.download_dir.default = get_directories()
cls.config.default = os.path.join(
self.data_dir, 'daemon_settings.yml'
)
cls.data_dir.default, cls.wallet_dir.default,\
cls.blob_dir.default, cls.download_dir.default = get_directories()
old_settings_file = os.path.join(self.data_dir, 'daemon_settings.yml')
if os.path.exists(old_settings_file):
cls.config.default = old_settings_file
else:
cls.config.default = os.path.join(self.data_dir, 'settings.yml')
if self.data_dir != cls.data_dir.default:
cls.blob_dir.default = os.path.join(self.data_dir, 'blobs')
cls.wallet_dir.default = os.path.join(self.data_dir, 'wallets')
@property
def log_file_path(self):
return os.path.join(self.data_dir, 'lbrynet.log')
return os.path.join(self.data_dir, 'daemon.log')
@property
def db_url_or_default(self):
if self.db_url:
return self.db_url
return 'sqlite:///'+os.path.join(self.data_dir, f'{self.blockchain}.db')
def get_windows_directories() -> typing.Tuple[str, str, str]:
from lbry.winpaths import get_path, FOLDERID, UserHandle # pylint: disable=import-outside-toplevel
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
def get_windows_directories() -> Tuple[str, str, str, str]:
# very old
data_dir = user_data_dir('lbrynet', roaming=True)
blob_dir = os.path.join(data_dir, 'blobfiles')
wallet_dir = os.path.join(user_data_dir('lbryum', roaming=True), 'wallets')
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
return data_dir, wallet_dir, blob_dir, user_download_dir()
# old
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
data_dir = os.path.join(appdata, 'lbrynet')
lbryum_dir = os.path.join(appdata, 'lbryum')
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
return data_dir, lbryum_dir, download_dir
# new
data_dir = user_data_dir('lbrynet', 'lbry')
lbryum_dir = user_data_dir('lbryum', 'lbry')
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
return data_dir, lbryum_dir, download_dir
def get_darwin_directories() -> typing.Tuple[str, str, str]:
data_dir = user_data_dir('LBRY')
lbryum_dir = os.path.expanduser('~/.lbryum')
download_dir = os.path.expanduser('~/Downloads')
return data_dir, lbryum_dir, download_dir
def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
download_dir = os.path.expanduser('~/Downloads')
# old
data_dir = os.path.expanduser('~/.lbrynet')
lbryum_dir = os.path.expanduser('~/.lbryum')
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
return data_dir, lbryum_dir, download_dir
blob_dir = os.path.join(data_dir, 'blobfiles')
wallet_dir = os.path.join(user_data_dir('lbryum', 'lbry'), 'wallets')
if os.path.isdir(blob_dir) and os.path.isdir(wallet_dir):
return data_dir, wallet_dir, blob_dir, user_download_dir()
# new
return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir
return get_universal_directories()
def get_darwin_directories() -> Tuple[str, str, str, str]:
data_dir = user_data_dir('LBRY')
blob_dir = os.path.join(data_dir, 'blobfiles')
wallet_dir = os.path.expanduser('~/.lbryum/wallets')
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
return data_dir, wallet_dir, blob_dir, user_download_dir()
return get_universal_directories()
def get_linux_directories() -> Tuple[str, str, str, str]:
# very old
data_dir = os.path.expanduser('~/.lbrynet')
blob_dir = os.path.join(data_dir, 'blobfiles')
wallet_dir = os.path.join(os.path.expanduser('~/.lbryum'), 'wallets')
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
return data_dir, wallet_dir, blob_dir, user_download_dir()
# old
data_dir = user_data_dir('lbry/lbrynet')
blob_dir = os.path.join(data_dir, 'blobfiles')
wallet_dir = user_data_dir('lbry/lbryum/wallets')
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
return data_dir, wallet_dir, blob_dir, user_download_dir()
# new
return get_universal_directories()
def get_universal_directories() -> Tuple[str, str, str, str]:
lbrynet_dir = user_data_dir('lbrynet', 'LBRY')
return (
lbrynet_dir,
os.path.join(lbrynet_dir, 'wallets'),
os.path.join(lbrynet_dir, 'blobs'),
user_download_dir()
)
@@ -72,13 +72,13 @@ class ConnectionManager:
self._status['outgoing_bps'].clear()
now = time.perf_counter()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['total_sent'] += v
self._status['outgoing_bps'][k] = v / (now - last)
k, sent = self.outgoing.popitem()
self._status['total_sent'] += sent
self._status['outgoing_bps'][k] = sent / (now - last)
while self.incoming:
k, v = self.incoming.popitem()
self._status['total_received'] += v
self._status['incoming_bps'][k] = v / (now - last)
k, received = self.incoming.popitem()
self._status['total_received'] += received
self._status['incoming_bps'][k] = received / (now - last)
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
)) / 1000000.0
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
+496
View File
@@ -0,0 +1,496 @@
import os
import sys
import time
import itertools
import logging
from typing import Dict, Any, Type
from tempfile import TemporaryFile
from tqdm.std import tqdm, Bar
from tqdm.utils import FormatReplace, _unicode, disp_len, disp_trim, _is_ascii
from lbry import __version__
from lbry.service.base import Service
from lbry.service.full_node import FullNode
from lbry.service.light_client import LightClient
log = logging.getLogger(__name__)
class RedirectOutput:
silence_lines = [
b'libprotobuf ERROR google/protobuf/wire_format_lite.cc:626',
]
def __init__(self, stream_type: str):
assert stream_type in ('stderr', 'stdout')
self.stream_type = stream_type
self.stream_no = getattr(sys, stream_type).fileno()
self.last_flush = time.time()
self.last_read = 0
self.backup = None
self.file = None
def __enter__(self):
self.backup = os.dup(self.stream_no)
setattr(sys, self.stream_type, os.fdopen(self.backup, 'w'))
self.file = TemporaryFile()
self.backup = os.dup2(self.file.fileno(), self.stream_no)
def __exit__(self, exc_type, exc_val, exc_tb):
self.file.close()
os.dup2(self.backup, self.stream_no)
os.close(self.backup)
setattr(sys, self.stream_type, os.fdopen(self.stream_no, 'w'))
def capture(self):
self.__enter__()
def release(self):
self.__exit__(None, None, None)
def flush(self, writer, force=False):
if not force and (time.time() - self.last_flush) < 5:
return
self.file.seek(self.last_read)
for line in self.file.readlines():
silence = False
for bad_line in self.silence_lines:
if bad_line in line:
silence = True
break
if not silence:
writer(line.decode().rstrip())
self.last_read = self.file.tell()
self.last_flush = time.time()
class Console:
def __init__(self, service: Service):
self.service = service
def starting(self):
pass
def stopping(self):
pass
class Basic(Console):
def __init__(self, service: Service):
super().__init__(service)
self.service.sync.on_progress.listen(self.on_sync_progress)
self.tasks = {}
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
def starting(self):
conf = self.service.conf
s = [f'LBRY v{__version__}']
if isinstance(self.service, FullNode):
s.append('Full Node')
elif isinstance(self.service, LightClient):
s.append('Light Client')
if conf.workers == -1:
s.append('Threads Only')
else:
workers = os.cpu_count() if conf.workers == 0 else conf.workers
s.append(f'{workers} Worker' if workers == 1 else f'{workers} Workers')
s.append(f'({os.cpu_count()} CPUs available)')
log.info(' '.join(s))
def stopping(self):
log.info('exiting')
@staticmethod
def maybe_log_progress(event, done, total, last):
if done == 0:
log.info("%s 0%%", event)
return 0
elif done == total:
log.info("%s 100%%", event)
return 1
else:
percent = done/total
if percent >= 0.25 > last:
log.info("%s 25%%", event)
return 0.25
elif percent >= 0.50 > last:
log.info("%s 50%%", event)
return 0.50
elif percent >= 0.75 > last:
log.info("%s 75%%", event)
return 0.75
return last
def on_sync_progress(self, event):
e, data = event["event"], event["data"]
name, current, total, last = e, data['done'][0], 0, 0
if not e.endswith("init") and not e.endswith("main") and not e.endswith("indexes"):
name = f"{e}#{data['id']}"
if "total" in data:
total, last = self.tasks[name] = (data["total"][0], last)
elif name in self.tasks:
total, last = self.tasks[name]
elif total == 0:
return
progress_status = (total, self.maybe_log_progress(name, current, total, last))
if progress_status[1] == 1:
del self.tasks[name]
else:
self.tasks[name] = progress_status
class Bar2(Bar):
def __init__(self, frac, default_len=10, charset=None):
super().__init__(frac[0], default_len, charset)
self.frac2 = frac[1]
def __format__(self, format_spec):
width = self.default_len
row1 = (1,)*int(self.frac2 * width * 2)
row2 = (2,)*int(self.frac * width * 2)
fill = []
for one, two, _ in itertools.zip_longest(row1, row2, range(width*2)):
fill.append((one or 0)+(two or 0))
bar = []
for i in range(0, width*2, 2):
if fill[i] == 1:
if fill[i+1] == 1:
bar.append('▀')
else:
bar.append('▘')
elif fill[i] == 2:
if fill[i+1] == 2:
bar.append('▄')
else:
bar.append('▖')
elif fill[i] == 3:
if fill[i+1] == 1:
bar.append('▛')
elif fill[i+1] == 2:
bar.append('▙')
elif fill[i+1] == 3:
bar.append('█')
else:
bar.append('▌')
else:
bar.append(' ')
return ''.join(bar)
class tqdm2(tqdm): # pylint: disable=invalid-name
def __init__(self, initial=(0, 0), unit=('it', 'it'), total=(None, None), **kwargs):
self.n2 = self.last_print_n2 = initial[1] # pylint: disable=invalid-name
self.unit2 = unit[1]
self.total2 = total[1]
super().__init__(initial=initial[0], unit=unit[0], total=total[0], **kwargs)
@property
def format_dict(self):
d = super().format_dict
d.update({
'n2': self.n2,
'unit2': self.unit2,
'total2': self.total2,
})
return d
def update(self, n=(1, 1)):
if self.disable:
return
last_last_print_t = self.last_print_t
self.n2 += n[1]
super().update(n[0])
if last_last_print_t != self.last_print_t:
self.last_print_n2 = self.n2
@staticmethod
def format_meter(
n, total, elapsed, ncols=None, prefix='', ascii=False, # pylint: disable=redefined-builtin
unit='it', unit_scale=False, rate=None, bar_format=None,
postfix=None, unit_divisor=1000, initial=0, **extra_kwargs
):
# sanity check: total
if total and n >= (total + 0.5): # allow float imprecision (#849)
total = None
# apply custom scale if necessary
if unit_scale and unit_scale not in (True, 1):
if total:
total *= unit_scale
n *= unit_scale
if rate:
rate *= unit_scale # by default rate = 1 / self.avg_time
unit_scale = False
elapsed_str = tqdm.format_interval(elapsed)
# if unspecified, attempt to use rate = average speed
# (we allow manual override since predicting time is an arcane art)
if rate is None and elapsed:
rate = n / elapsed
inv_rate = 1 / rate if rate else None
format_sizeof = tqdm.format_sizeof
rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else
'{0:5.2f}'.format(rate))
if rate else '?') + unit + '/s'
rate_inv_fmt = ((format_sizeof(inv_rate) if unit_scale else
'{0:5.2f}'.format(inv_rate))
if inv_rate else '?') + 's/' + unit
rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt
if unit_scale:
n_fmt = format_sizeof(n, divisor=unit_divisor)
total_fmt = format_sizeof(total, divisor=unit_divisor) \
if total is not None else '?'
else:
n_fmt = str(n)
total_fmt = str(total) if total is not None else '?'
try:
postfix = ', ' + postfix if postfix else ''
except TypeError:
pass
remaining = (total - n) / rate if rate and total else 0
remaining_str = tqdm.format_interval(remaining) if rate else '?'
# format the stats displayed to the left and right sides of the bar
if prefix:
# old prefix setup work around
bool_prefix_colon_already = (prefix[-2:] == ": ")
l_bar = prefix if bool_prefix_colon_already else prefix + ": "
else:
l_bar = ''
r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)
# Custom bar formatting
# Populate a dict with all available progress indicators
format_dict = dict(
# slight extension of self.format_dict
n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
elapsed=elapsed_str, elapsed_s=elapsed,
ncols=ncols, desc=prefix or '', unit=unit,
rate=inv_rate if inv_rate and inv_rate > 1 else rate,
rate_fmt=rate_fmt, rate_noinv=rate,
rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
rate_inv_fmt=rate_inv_fmt,
postfix=postfix, unit_divisor=unit_divisor,
# plus more useful definitions
remaining=remaining_str, remaining_s=remaining,
l_bar=l_bar, r_bar=r_bar,
**extra_kwargs)
# total is known: we can predict some stats
if total:
n2, total2 = extra_kwargs['n2'], extra_kwargs['total2'] # pylint: disable=invalid-name
# fractional and percentage progress
frac = n / total
frac2 = n2 / total2
percentage = frac * 100
l_bar += '{0:3.0f}%|'.format(percentage)
if ncols == 0:
return l_bar[:-1] + r_bar[1:]
format_dict.update(l_bar=l_bar)
if bar_format:
format_dict.update(percentage=percentage)
# auto-remove colon for empty `desc`
if not prefix:
bar_format = bar_format.replace("{desc}: ", '')
else:
bar_format = "{l_bar}{bar}{r_bar}"
full_bar = FormatReplace()
try:
nobar = bar_format.format(bar=full_bar, **format_dict)
except UnicodeEncodeError:
bar_format = _unicode(bar_format)
nobar = bar_format.format(bar=full_bar, **format_dict)
if not full_bar.format_called:
# no {bar}, we can just format and return
return nobar
# Formatting progress bar space available for bar's display
full_bar = Bar2(
(frac, frac2),
max(1, ncols - disp_len(nobar))
if ncols else 10,
charset=Bar2.ASCII if ascii is True else ascii or Bar2.UTF)
if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):
bar_format = _unicode(bar_format)
res = bar_format.format(bar=full_bar, **format_dict)
return disp_trim(res, ncols) if ncols else res
elif bar_format:
# user-specified bar_format but no total
l_bar += '|'
format_dict.update(l_bar=l_bar, percentage=0)
full_bar = FormatReplace()
nobar = bar_format.format(bar=full_bar, **format_dict)
if not full_bar.format_called:
return nobar
full_bar = Bar2(
(0, 0),
max(1, ncols - disp_len(nobar))
if ncols else 10,
charset=Bar2.BLANK)
res = bar_format.format(bar=full_bar, **format_dict)
return disp_trim(res, ncols) if ncols else res
else:
# no total: no progressbar, ETA, just progress stats
return ((prefix + ": ") if prefix else '') + \
'{0}{1} [{2}, {3}{4}]'.format(
n_fmt, unit, elapsed_str, rate_fmt, postfix)
class Advanced(Basic):
FORMAT = '{l_bar}{bar}| {n_fmt:>8}/{total_fmt:>8} [{elapsed:>7}<{remaining:>8}, {rate_fmt:>17}]'
def __init__(self, service: Service):
super().__init__(service)
self.bars: Dict[Any, tqdm] = {}
self.stderr = RedirectOutput('stderr')
def starting(self):
self.stderr.capture()
super().starting()
def stopping(self):
for bar in self.bars.values():
bar.close()
super().stopping()
#self.stderr.flush(self.bars['read'].write, True)
#self.stderr.release()
def get_or_create_bar(self, name, desc, units, totals, leave=False, bar_format=None, postfix=None, position=None):
bar = self.bars.get(name)
if bar is None:
if len(units) == 2:
bar = self.bars[name] = tqdm2(
desc=desc, unit=units, total=totals,
bar_format=bar_format or self.FORMAT, leave=leave,
postfix=postfix, position=position
)
else:
bar = self.bars[name] = tqdm(
desc=desc, unit=units[0], total=totals[0],
bar_format=bar_format or self.FORMAT, leave=leave,
postfix=postfix, position=position
)
return bar
def sync_init(self, name, d):
bar_name = f"{name}#{d['id']}"
bar = self.bars.get(bar_name)
if bar is None:
label = d.get('label', name[-11:])
self.get_or_create_bar(bar_name, label, d['units'], d['total'], True)
else:
if d['done'][0] != -1:
bar.update(d['done'][0] - bar.n)
if d['done'][0] == -1 or d['done'][0] == bar.total:
bar.close()
self.bars.pop(bar_name)
def sync_main(self, name, d):
bar = self.bars.get(name)
if bar is None:
label = d.get('label', name[-11:])
self.get_or_create_bar(name, label, d['units'], d['total'], True)
#self.last_stats = f"{d['txs']:,d} txs, {d['claims']:,d} claims and {d['supports']:,d} supports"
#self.get_or_create_bar("read", "├─ blocks read", "blocks", d['blocks'], True)
#self.get_or_create_bar("save", "└─┬ txs saved", "txs", d['txs'], True)
else:
if d['done'] == (-1,)*len(d['done']):
base_name = name[:name.rindex('.')]
for child_name, child_bar in self.bars.items():
if child_name.startswith(base_name):
child_bar.close()
bar.close()
self.bars.pop(name)
else:
if len(d['done']) == 2:
bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
else:
bar.update(d['done'][0]-bar.n)
def sync_task(self, name, d):
bar_name = f"{name}#{d['id']}"
bar = self.bars.get(bar_name)
if bar is None:
#assert d['done'][0] == 0
label = d.get('label', name[-11:])
self.get_or_create_bar(
f"{name}#{d['id']}", label, d['units'], d['total'],
name.split('.')[-1] not in ('insert', 'update', 'file')
)
else:
if d['done'][0] != -1:
main_bar_name = f"{name[:name.rindex('.')]}.main"
if len(d['done']) > 1:
diff = tuple(a-b for a, b in zip(d['done'], (bar.n, bar.n2)))
else:
diff = d['done'][0] - bar.n
if main_bar_name != name:
main_bar = self.bars.get(main_bar_name)
if main_bar and main_bar.unit == bar.unit:
main_bar.update(diff)
bar.update(diff)
if d['done'][0] == -1 or d['done'][0] == bar.total:
bar.close()
self.bars.pop(bar_name)
def update_other_bars(self, e, d):
if d['total'] == 0:
return
bar = self.bars.get(e)
if not bar:
name = (
' '.join(e.split('.')[-2:])
.replace('support', 'suprt')
.replace('channels', 'chanls')
.replace('signatures', 'sigs')
)
bar = self.get_or_create_bar(e, f"├─ {name:>12}", d['unit'], d['total'], True)
diff = d['step']-bar.n
bar.update(diff)
#if d['step'] == d['total']:
#bar.close()
def on_sync_progress(self, event):
e, d = event['event'], event.get('data', {})
if e.endswith(".init"):
self.sync_init(e, d)
elif e.endswith(".main"):
self.sync_main(e, d)
else:
self.sync_task(e, d)
# if e.endswith("sync.start"):
# self.sync_start(d)
# self.stderr.flush(self.bars['read'].write)
# elif e.endswith("sync.complete"):
# self.stderr.flush(self.bars['read'].write, True)
# self.sync_complete()
# else:
# self.stderr.flush(self.bars['read'].write)
# self.update_progress(e, d)
def console_class_from_name(name) -> Type[Console]:
return {'basic': Basic, 'advanced': Advanced}.get(name, Console)
+8
View File
@@ -0,0 +1,8 @@
DEFAULT_PAGE_SIZE = 20
NULL_HASH32 = b'\x00'*32
CENT = 1000000
COIN = 100*CENT
INVALIDATED_SIGNATURE_GRACE_PERIOD = 50
File renamed without changes.
+86
View File
@@ -0,0 +1,86 @@
from lbry.crypto.hash import double_sha256
from lbry.crypto.util import bytes_to_int, int_to_bytes
class Base58Error(Exception):
""" Exception used for Base58 errors. """
class Base58:
""" Class providing base 58 functionality. """
chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
assert len(chars) == 58
char_map = {c: n for n, c in enumerate(chars)}
@classmethod
def char_value(cls, c):
val = cls.char_map.get(c)
if val is None:
raise Base58Error(f'invalid base 58 character "{c}"')
return val
@classmethod
def decode(cls, txt):
""" Decodes txt into a big-endian bytearray. """
if isinstance(txt, memoryview):
txt = str(txt)
if isinstance(txt, bytes):
txt = txt.decode()
if not isinstance(txt, str):
raise TypeError('a string is required')
if not txt:
raise Base58Error('string cannot be empty')
value = 0
for c in txt:
value = value * 58 + cls.char_value(c)
result = int_to_bytes(value)
# Prepend leading zero bytes if necessary
count = 0
for c in txt:
if c != '1':
break
count += 1
if count:
result = bytes((0,)) * count + result
return result
@classmethod
def encode(cls, be_bytes):
"""Converts a big-endian bytearray into a base58 string."""
value = bytes_to_int(be_bytes)
txt = ''
while value:
value, mod = divmod(value, 58)
txt += cls.chars[mod]
for byte in be_bytes:
if byte != 0:
break
txt += '1'
return txt[::-1]
@classmethod
def decode_check(cls, txt, hash_fn=double_sha256):
""" Decodes a Base58Check-encoded string to a payload. The version prefixes it. """
be_bytes = cls.decode(txt)
result, check = be_bytes[:-4], be_bytes[-4:]
if check != hash_fn(result)[:4]:
raise Base58Error(f'invalid base 58 checksum for {txt}')
return result
@classmethod
def encode_check(cls, payload, hash_fn=double_sha256):
""" Encodes a payload bytearray (which includes the version byte(s))
into a Base58Check string."""
be_bytes = payload + hash_fn(payload)[:4]
return cls.encode(be_bytes)
@@ -1,16 +1,8 @@
# Copyright (c) 2017, Neil Booth
# Copyright (c) 2018, LBRY Inc.
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
""" Logic for BIP32 Hierarchical Key Derivation. """
from coincurve import PublicKey, PrivateKey as _PrivateKey
from torba.client.hash import Base58, hmac_sha512, hash160, double_sha256
from torba.client.util import cachedproperty
from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
from lbry.crypto.base58 import Base58
from lbry.utils import cachedproperty
class DerivationError(Exception):
+71
View File
@@ -0,0 +1,71 @@
import os
import base64
import typing
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbry.error import InvalidPasswordError
from lbry.crypto.hash import double_sha256
def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str:
if init_vector is not None:
assert len(init_vector) == 16
else:
init_vector = os.urandom(16)
key = double_sha256(secret.encode())
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
padder = PKCS7(AES.block_size).padder()
padded_data = padder.update(value.encode()) + padder.finalize()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
return base64.b64encode(init_vector + encrypted_data).decode()
def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
try:
data = base64.b64decode(value.encode())
key = double_sha256(secret.encode())
init_vector, data = data[:16], data[16:]
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
unpadder = PKCS7(AES.block_size).unpadder()
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
return result.decode(), init_vector
except UnicodeDecodeError:
raise InvalidPasswordError()
except ValueError as e:
if e.args[0] == 'Invalid padding bytes.':
raise InvalidPasswordError()
raise
def better_aes_encrypt(secret: str, value: bytes) -> bytes:
init_vector = os.urandom(16)
key = scrypt(secret.encode(), salt=init_vector)
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
padder = PKCS7(AES.block_size).padder()
padded_data = padder.update(value) + padder.finalize()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data)
def better_aes_decrypt(secret: str, value: bytes) -> bytes:
try:
data = base64.b64decode(value)
_, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4)
init_vector, data = data[:16], data[16:]
key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p))
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
unpadder = PKCS7(AES.block_size).unpadder()
return unpadder.update(decryptor.update(data)) + unpadder.finalize()
except ValueError as e:
if e.args[0] == 'Invalid padding bytes.':
raise InvalidPasswordError()
raise
def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1):
kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend())
return kdf.derive(passphrase)
+47
View File
@@ -0,0 +1,47 @@
import hashlib
import hmac
from binascii import hexlify, unhexlify
def sha256(x):
""" Simple wrapper of hashlib sha256. """
return hashlib.sha256(x).digest()
def sha512(x):
""" Simple wrapper of hashlib sha512. """
return hashlib.sha512(x).digest()
def ripemd160(x):
""" Simple wrapper of hashlib ripemd160. """
h = hashlib.new('ripemd160')
h.update(x)
return h.digest()
def double_sha256(x):
""" SHA-256 of SHA-256, as used extensively in bitcoin. """
return sha256(sha256(x))
def hmac_sha512(key, msg):
""" Use SHA-512 to provide an HMAC. """
return hmac.new(key, msg, hashlib.sha512).digest()
def hash160(x):
""" RIPEMD-160 of SHA-256.
Used to make bitcoin addresses from pubkeys. """
return ripemd160(sha256(x))
def hash_to_hex_str(x: bytes) -> str:
""" Convert a big-endian binary hash to displayed hex string.
Display form of a binary hash is reversed and converted to hex. """
return hexlify(x[::-1])
def hex_str_to_hash(x: str) -> bytes:
""" Convert a displayed hex string to a binary hash. """
return unhexlify(x)[::-1]
+13
View File
@@ -0,0 +1,13 @@
from binascii import unhexlify, hexlify
def bytes_to_int(be_bytes):
""" Interprets a big-endian sequence of bytes as an integer. """
return int(hexlify(be_bytes), 16)
def int_to_bytes(value):
""" Converts an integer to a big-endian sequence of bytes. """
length = (value.bit_length() + 7) // 8
s = '%x' % value
return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2))
+5
View File
@@ -0,0 +1,5 @@
from .database import Database, Result
from .constants import (
TXO_TYPES, SPENDABLE_TYPE_CODES,
CLAIM_TYPE_CODES, CLAIM_TYPE_NAMES
)
+73
View File
@@ -0,0 +1,73 @@
MAX_QUERY_VARIABLES = 900
TXO_TYPES = {
"other": 0,
"stream": 1,
"channel": 2,
"support": 3,
"purchase": 4,
"collection": 5,
"repost": 6,
}
CLAIM_TYPE_NAMES = [
'stream',
'channel',
'collection',
'repost',
]
CONTENT_TYPE_NAMES = [
name for name in CLAIM_TYPE_NAMES if name != "channel"
]
CLAIM_TYPE_CODES = [
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
]
CONTENT_TYPE_CODES = [
TXO_TYPES[name] for name in CONTENT_TYPE_NAMES
]
SPENDABLE_TYPE_CODES = [
TXO_TYPES['other'],
TXO_TYPES['purchase']
]
STREAM_TYPES = {
'video': 1,
'audio': 2,
'image': 3,
'document': 4,
'binary': 5,
'model': 6
}
MATURE_TAGS = (
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
)
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
SEARCH_INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted',
'amount', 'staked_amount', 'support_amount',
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
}
SEARCH_PARAMS = {
'name', 'text', 'claim_id', 'claim_ids', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
'public_key_id', 'claim_type', 'stream_types', 'media_types', 'fee_currency',
'has_channel_signature', 'signature_valid',
'any_tags', 'all_tags', 'not_tags', 'reposted_claim_id',
'any_locations', 'all_locations', 'not_locations',
'any_languages', 'all_languages', 'not_languages',
'is_controlling', 'limit', 'offset', 'order_by',
'no_totals',
} | SEARCH_INTEGER_PARAMS
SEARCH_ORDER_FIELDS = {
'name', 'claim_hash', 'claim_id'
} | SEARCH_INTEGER_PARAMS
+402
View File
@@ -0,0 +1,402 @@
import os
import asyncio
import tempfile
import multiprocessing as mp
from binascii import hexlify
from typing import List, Optional, Iterable, Iterator, TypeVar, Generic, TYPE_CHECKING, Dict, Tuple
from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
from functools import partial
from sqlalchemy import create_engine, text
from lbry.event import EventController
from lbry.crypto.bip32 import PubKey
from lbry.blockchain.transaction import Transaction, Output
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
from .query_context import initialize, uninitialize, ProgressPublisher
from . import queries as q
from . import sync
if TYPE_CHECKING:
from lbry.blockchain.ledger import Ledger
def clean_wallet_account_ids(constraints):
wallet = constraints.pop('wallet', None)
account = constraints.pop('account', None)
accounts = constraints.pop('accounts', [])
if account and not accounts:
accounts = [account]
if wallet:
constraints['wallet_account_ids'] = [account.id for account in wallet.accounts]
if not accounts:
accounts = wallet.accounts
if accounts:
constraints['account_ids'] = [account.id for account in accounts]
async def add_channel_keys_to_txo_results(accounts: List, txos: Iterable[Output]):
sub_channels = set()
for txo in txos:
if txo.is_claim and txo.claim.is_channel:
for account in accounts:
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
txo.private_key = private_key
break
if txo.channel is not None:
sub_channels.add(txo.channel)
if sub_channels:
await add_channel_keys_to_txo_results(accounts, sub_channels)
ResultType = TypeVar('ResultType')
class Result(Generic[ResultType]):
__slots__ = 'rows', 'total', 'censor'
def __init__(self, rows: List[ResultType], total, censor=None):
self.rows = rows
self.total = total
self.censor = censor
def __getitem__(self, item: int) -> ResultType:
return self.rows[item]
def __iter__(self) -> Iterator[ResultType]:
return iter(self.rows)
def __len__(self):
return len(self.rows)
def __repr__(self):
return repr(self.rows)
class Database:
def __init__(self, ledger: 'Ledger'):
self.url = ledger.conf.db_url_or_default
self.ledger = ledger
self.workers = self._normalize_worker_processes(ledger.conf.workers)
self.executor: Optional[Executor] = None
self.message_queue = mp.Queue()
self.stop_event = mp.Event()
self._on_progress_controller = EventController()
self.on_progress = self._on_progress_controller.stream
self.progress_publisher = ProgressPublisher(
self.message_queue, self._on_progress_controller
)
@staticmethod
def _normalize_worker_processes(workers):
if workers == 0:
return os.cpu_count()
elif workers > 0:
return workers
return 1
@classmethod
def temp_from_url_regtest(cls, db_url, lbrycrd_config=None):
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
directory = tempfile.mkdtemp()
if lbrycrd_config:
conf = lbrycrd_config
conf.data_dir = directory
conf.download_dir = directory
conf.wallet_dir = directory
else:
conf = Config.with_same_dir(directory)
conf.set(blockchain="regtest", db_url=db_url)
ledger = RegTestLedger(conf)
return cls(ledger)
@classmethod
def temp_sqlite_regtest(cls, lbrycrd_config=None):
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
directory = tempfile.mkdtemp()
if lbrycrd_config:
conf = lbrycrd_config
conf.data_dir = directory
conf.download_dir = directory
conf.wallet_dir = directory
else:
conf = Config.with_same_dir(directory).set(blockchain="regtest")
ledger = RegTestLedger(conf)
return cls(ledger)
@classmethod
def temp_sqlite(cls):
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
conf = Config.with_same_dir(tempfile.mkdtemp())
return cls(Ledger(conf))
@classmethod
def from_url(cls, db_url):
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
return cls(Ledger(Config.with_null_dir().set(db_url=db_url)))
@classmethod
def in_memory(cls):
return cls.from_url('sqlite:///:memory:')
def sync_create(self, name):
engine = create_engine(self.url)
db = engine.connect()
db.execute(text("COMMIT"))
db.execute(text(f"CREATE DATABASE {name}"))
async def create(self, name):
return await asyncio.get_running_loop().run_in_executor(None, self.sync_create, name)
def sync_drop(self, name):
engine = create_engine(self.url)
db = engine.connect()
db.execute(text("COMMIT"))
db.execute(text(f"DROP DATABASE IF EXISTS {name}"))
async def drop(self, name):
return await asyncio.get_running_loop().run_in_executor(None, self.sync_drop, name)
async def open(self):
assert self.executor is None, "Database already open."
self.progress_publisher.start()
kwargs = {
"initializer": initialize,
"initargs": (
self.ledger,
self.message_queue, self.stop_event
)
}
if self.workers > 1:
self.executor = ProcessPoolExecutor(max_workers=self.workers, **kwargs)
else:
self.executor = ThreadPoolExecutor(max_workers=1, **kwargs)
return await self.run(q.check_version_and_create_tables)
async def close(self):
self.progress_publisher.stop()
if self.executor is not None:
if isinstance(self.executor, ThreadPoolExecutor):
await self.run(uninitialize)
self.executor.shutdown()
self.executor = None
# fixes "OSError: handle is closed"
# seems to only happen when running in PyCharm
# https://github.com/python/cpython/pull/6084#issuecomment-564585446
# TODO: delete this in Python 3.8/3.9?
from concurrent.futures.process import _threads_wakeups # pylint: disable=import-outside-toplevel
_threads_wakeups.clear()
async def run(self, func, *args, **kwargs):
if kwargs:
clean_wallet_account_ids(kwargs)
return await asyncio.get_running_loop().run_in_executor(
self.executor, partial(func, *args, **kwargs)
)
async def fetch_result(self, func, *args, **kwargs) -> Result:
rows, total = await self.run(func, *args, **kwargs)
return Result(rows, total)
async def execute(self, sql):
return await self.run(q.execute, sql)
async def execute_sql_object(self, sql):
return await self.run(q.execute_sql_object, sql)
async def execute_fetchall(self, sql):
return await self.run(q.execute_fetchall, sql)
async def has_filters(self):
return await self.run(q.has_filters)
async def has_claims(self):
return await self.run(q.has_claims)
async def has_supports(self):
return await self.run(q.has_supports)
async def has_wallet(self, wallet_id):
return await self.run(q.has_wallet, wallet_id)
async def get_wallet(self, wallet_id: str):
return await self.run(q.get_wallet, wallet_id)
async def add_wallet(self, wallet_id: str, data: str):
return await self.run(q.add_wallet, wallet_id, data)
async def get_best_block_height(self) -> int:
return await self.run(q.get_best_block_height)
async def process_all_things_after_sync(self):
return await self.run(sync.process_all_things_after_sync)
async def get_block_headers(self, start_height: int, end_height: int = None):
return await self.run(q.get_block_headers, start_height, end_height)
async def get_filters(self, start_height: int, end_height: int = None, granularity: int = 0):
filters = []
for row in await self.run(q.get_filters, start_height, end_height, granularity):
record = {
"height": row["height"],
"filter": hexlify(row["address_filter"]).decode(),
}
if granularity == 0:
record["txid"] = hexlify(row["tx_hash"][::-1]).decode()
filters.append(record)
return filters
async def get_missing_required_filters(self, height) -> Dict[int, Tuple[int, int]]:
return await self.run(q.get_missing_required_filters, height)
async def get_missing_sub_filters_for_addresses(self, granularity, address_manager):
return await self.run(q.get_missing_sub_filters_for_addresses, granularity, address_manager)
async def get_missing_tx_for_addresses(self, address_manager):
return await self.run(q.get_missing_tx_for_addresses, address_manager)
async def insert_blocks(self, blocks):
return await self.run(q.insert_blocks, blocks)
async def insert_block_filters(self, filters):
return await self.run(q.insert_block_filters, filters)
async def insert_tx_filters(self, filters):
return await self.run(q.insert_tx_filters, filters)
async def insert_transactions(self, txs):
return await self.run(q.insert_transactions, txs)
async def update_address_used_times(self, addresses):
return await self.run(q.update_address_used_times, addresses)
async def reserve_outputs(self, txos, is_reserved=True):
txo_hashes = [txo.hash for txo in txos]
if txo_hashes:
return await self.run(
q.reserve_outputs, txo_hashes, is_reserved
)
async def release_outputs(self, txos):
return await self.reserve_outputs(txos, is_reserved=False)
async def release_tx(self, tx):
return await self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
async def release_all_outputs(self, account):
return await self.run(q.release_all_outputs, account.id)
async def get_balance(self, **constraints):
return await self.run(q.get_balance, **constraints)
async def get_report(self, accounts):
return await self.run(q.get_report, accounts=accounts)
async def get_addresses(self, **constraints) -> Result[dict]:
addresses = await self.fetch_result(q.get_addresses, **constraints)
if addresses and 'pubkey' in addresses[0]:
for address in addresses:
address['pubkey'] = PubKey(
self.ledger, bytes(address.pop('pubkey')), bytes(address.pop('chain_code')),
address.pop('n'), address.pop('depth')
)
return addresses
async def get_all_addresses(self):
return await self.run(q.get_all_addresses)
async def get_address(self, **constraints):
for address in await self.get_addresses(limit=1, **constraints):
return address
async def add_keys(self, account, chain, pubkeys):
return await self.run(q.add_keys, [{
'account': account.id,
'address': k.address,
'chain': chain,
'pubkey': k.pubkey_bytes,
'chain_code': k.chain_code,
'n': k.n,
'depth': k.depth
} for k in pubkeys])
async def generate_addresses_using_filters(self, best_height, allowed_gap, address_manager):
return await self.run(
q.generate_addresses_using_filters, best_height, allowed_gap, address_manager
)
async def get_raw_transactions(self, **constraints):
return await self.run(q.get_raw_transactions, **constraints)
async def get_transactions(self, **constraints) -> Result[Transaction]:
return await self.fetch_result(q.get_transactions, **constraints)
async def get_transaction(self, **constraints) -> Optional[Transaction]:
txs = await self.get_transactions(limit=1, **constraints)
if txs:
return txs[0]
async def get_purchases(self, **constraints) -> Result[Output]:
return await self.fetch_result(q.get_purchases, **constraints)
async def search_claims(self, **constraints) -> Result[Output]:
#assert set(constraints).issubset(SEARCH_PARAMS), \
# f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
claims, total, censor = await self.run(q.search_claims, **constraints)
return Result(claims, total, censor)
async def protobuf_search_claims(self, **constraints) -> str:
return await self.run(q.protobuf_search_claims, **constraints)
async def search_supports(self, **constraints) -> Result[Output]:
return await self.fetch_result(q.search_supports, **constraints)
async def sum_supports(self, claim_hash, include_channel_content=False, exclude_own_supports=False) \
-> Tuple[List[Dict], int]:
return await self.run(q.sum_supports, claim_hash, include_channel_content, exclude_own_supports)
async def resolve(self, urls, **kwargs) -> Dict[str, Output]:
return await self.run(q.resolve, urls, **kwargs)
async def protobuf_resolve(self, urls, **kwargs) -> str:
return await self.run(q.protobuf_resolve, urls, **kwargs)
async def get_txo_sum(self, **constraints) -> int:
return await self.run(q.get_txo_sum, **constraints)
async def get_txo_plot(self, **constraints) -> List[dict]:
return await self.run(q.get_txo_plot, **constraints)
async def get_txos(self, **constraints) -> Result[Output]:
txos = await self.fetch_result(q.get_txos, **constraints)
if 'wallet' in constraints:
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
return txos
async def get_utxos(self, **constraints) -> Result[Output]:
return await self.get_txos(spent_height=0, **constraints)
async def get_supports(self, **constraints) -> Result[Output]:
return await self.get_utxos(txo_type=TXO_TYPES['support'], **constraints)
async def get_claims(self, **constraints) -> Result[Output]:
if 'txo_type' not in constraints:
constraints['txo_type__in'] = CLAIM_TYPE_CODES
txos = await self.fetch_result(q.get_txos, **constraints)
if 'wallet' in constraints:
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
return txos
async def get_streams(self, **constraints) -> Result[Output]:
return await self.get_claims(txo_type=TXO_TYPES['stream'], **constraints)
async def get_channels(self, **constraints) -> Result[Output]:
return await self.get_claims(txo_type=TXO_TYPES['channel'], **constraints)
async def get_collections(self, **constraints) -> Result[Output]:
return await self.get_claims(txo_type=TXO_TYPES['collection'], **constraints)
+7
View File
@@ -0,0 +1,7 @@
from .base import *
from .txio import *
from .search import *
from .resolve import *
from .address import *
from .wallet import *
from .filters import *
+205
View File
@@ -0,0 +1,205 @@
import logging
from typing import Tuple, List, Set, Iterator, Optional
from sqlalchemy import func
from sqlalchemy.future import select
from lbry.crypto.hash import hash160
from lbry.crypto.bip32 import PubKey
from ..utils import query
from ..query_context import context
from ..tables import TXO, PubkeyAddress, AccountAddress
from .filters import (
get_filter_matchers, get_filter_matchers_at_granularity, has_filter_range,
get_tx_matchers_for_missing_txs,
)
log = logging.getLogger(__name__)
class DatabaseAddressIterator:
def __init__(self, account_id, chain):
self.account_id = account_id
self.chain = chain
self.n = -1
@staticmethod
def get_sql(account_id, chain):
return (
select(
AccountAddress.c.pubkey,
AccountAddress.c.n
).where(
(AccountAddress.c.account == account_id) &
(AccountAddress.c.chain == chain)
).order_by(AccountAddress.c.n)
)
@staticmethod
def get_address_hash_bytes(account_id, chain):
return [
bytearray(hash160(row['pubkey'])) for row in context().fetchall(
DatabaseAddressIterator.get_sql(account_id, chain)
)
]
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
with context().connect_streaming() as c:
sql = self.get_sql(self.account_id, self.chain)
for row in c.execute(sql):
self.n = row['n']
yield hash160(row['pubkey']), self.n, False
class PersistingAddressIterator(DatabaseAddressIterator):
def __init__(self, account_id, chain, pubkey_bytes, chain_code, depth):
super().__init__(account_id, chain)
self.pubkey_bytes = pubkey_bytes
self.chain_code = chain_code
self.depth = depth
self.pubkey_buffer = []
def flush(self):
if self.pubkey_buffer:
add_keys([{
'account': self.account_id,
'address': k.address,
'chain': self.chain,
'pubkey': k.pubkey_bytes,
'chain_code': k.chain_code,
'n': k.n,
'depth': k.depth
} for k in self.pubkey_buffer])
self.pubkey_buffer.clear()
def __enter__(self) -> 'PersistingAddressIterator':
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.flush()
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
yield from super().__iter__()
pubkey = PubKey(context().ledger, self.pubkey_bytes, self.chain_code, 0, self.depth)
while True:
self.n += 1
pubkey_child = pubkey.child(self.n)
self.pubkey_buffer.append(pubkey_child)
if len(self.pubkey_buffer) >= 900:
self.flush()
yield hash160(pubkey_child.pubkey_bytes), self.n, True
class BatchAddressIterator:
def __init__(self, iterator: PersistingAddressIterator, size):
self.iterator = iterator
self.size = size
def __iter__(self) -> Iterator[bytearray]:
i = iter(self.iterator)
while True:
yield [bytearray(next(i)[0]) for _ in range(self.size)]
def generate_addresses_using_filters(best_height, allowed_gap, address_manager) -> Set:
need, have = set(), set()
matchers = get_filter_matchers(best_height)
with PersistingAddressIterator(*address_manager) as address_iterator:
for addresses in BatchAddressIterator(address_iterator, allowed_gap):
has_match = False
for matcher, filter_range in matchers:
if matcher.MatchAny(addresses):
has_match = True
if filter_range not in need and filter_range not in have:
if has_filter_range(*filter_range):
have.add(filter_range)
else:
need.add(filter_range)
if not has_match:
break
return need
def get_missing_sub_filters_for_addresses(granularity, address_manager):
need = set()
filters = get_filter_matchers_at_granularity(granularity)
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
for matcher, filter_range in filters:
if matcher.MatchAny(addresses) and not has_filter_range(*filter_range):
need.add(filter_range)
return need
def get_missing_tx_for_addresses(address_manager):
need = set()
filters = get_tx_matchers_for_missing_txs()
print(f' loaded tx filters ({len(filters)})')
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
print(f' loaded addresses ({len(addresses)})')
print(' matching...')
for i, (tx_hash, matcher) in enumerate(filters):
if i > 0 and i % 1000 == 0:
print(f' {i} of {len(filters)} processed')
if matcher.MatchAny(addresses):
need.add(tx_hash)
return need
def update_address_used_times(addresses):
context().execute(
PubkeyAddress.update()
.values(used_times=(
select(func.count(TXO.c.address))
.where((TXO.c.address == PubkeyAddress.c.address)),
))
.where(PubkeyAddress.c.address._in(addresses))
)
def select_addresses(cols, **constraints):
return context().fetchall(query(
[AccountAddress, PubkeyAddress],
select(*cols).select_from(PubkeyAddress.join(AccountAddress)),
**constraints
))
def get_addresses(cols=None, include_total=False, **constraints) -> Tuple[List[dict], Optional[int]]:
if cols is None:
cols = (
PubkeyAddress.c.address,
PubkeyAddress.c.used_times,
AccountAddress.c.account,
AccountAddress.c.chain,
AccountAddress.c.pubkey,
AccountAddress.c.chain_code,
AccountAddress.c.n,
AccountAddress.c.depth
)
return (
select_addresses(cols, **constraints),
get_address_count(**constraints) if include_total else None
)
def get_address_count(**constraints):
count = select_addresses([func.count().label("total")], **constraints)
return count[0]["total"] or 0
def get_all_addresses():
return [r["address"] for r in context().fetchall(select(PubkeyAddress.c.address))]
def add_keys(pubkeys):
c = context()
current_limit = c.variable_limit // len(pubkeys[0]) # (overall limit) // (maximum on a query)
for start in range(0, len(pubkeys), current_limit - 1):
batch = pubkeys[start:(start + current_limit - 1)]
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
c.execute(c.insert_or_ignore(AccountAddress).values(batch))
+99
View File
@@ -0,0 +1,99 @@
from sqlalchemy import text, between
from sqlalchemy.future import select
from ..query_context import context
from ..tables import (
SCHEMA_VERSION, metadata, Version,
Claim, Support, Block, TX,
pg_add_account_address_constraints_and_indexes
)
def execute(sql):
return context().execute(text(sql))
def execute_sql_object(sql):
return context().execute(sql)
def execute_fetchall(sql):
return context().fetchall(text(sql))
def has_claims():
return context().has_records(Claim)
def has_supports():
return context().has_records(Support)
def get_best_block_height():
return context().fetchmax(Block.c.height, -1)
def insert_blocks(blocks):
loader = context().get_bulk_loader()
for block in blocks:
loader.add_block(block)
loader.flush(return_row_count_for_table=None)
def get_block_headers(first, last=None):
q = (
select(
Block.c.height,
Block.c.block_hash,
Block.c.previous_hash,
Block.c.timestamp,
)
.select_from(Block)
)
if last is not None:
query = (
q.where(between(Block.c.height, first, last))
.order_by(Block.c.height)
)
else:
query = q.where(Block.c.height == first)
rows = context().fetchall(query)
for row in rows:
row['block_hash'] = bytes(row['block_hash'])
row['previous_hash'] = bytes(row['previous_hash'])
return rows
def insert_transactions(txs):
loader = context().get_bulk_loader()
for block_hash, tx in txs:
loader.add_transaction(block_hash, tx)
loader.flush(return_row_count_for_table=None)
def check_version_and_create_tables():
with context("db.connecting") as ctx:
if ctx.has_table('version'):
version = ctx.fetchone(select(Version.c.version).limit(1))
if version and version['version'] == SCHEMA_VERSION:
return
metadata.drop_all(ctx.engine)
metadata.create_all(ctx.engine)
ctx.execute(Version.insert().values(version=SCHEMA_VERSION))
for table in metadata.sorted_tables:
disable_trigger_and_constraints(table.name)
if ctx.is_postgres:
for statement in pg_add_account_address_constraints_and_indexes:
ctx.execute(text(statement))
def disable_trigger_and_constraints(table_name):
ctx = context()
if ctx.is_postgres:
ctx.execute(text(f"ALTER TABLE {table_name} DISABLE TRIGGER ALL;"))
if table_name in ('tag', 'stake', 'block_filter', 'mempool_filter'):
return
if ctx.is_postgres:
ctx.execute(text(
f"ALTER TABLE {table_name} DROP CONSTRAINT {table_name}_pkey CASCADE;"
))
+175
View File
@@ -0,0 +1,175 @@
from math import log10
from typing import Dict, List, Tuple, Set, Optional
from sqlalchemy import between, func, or_
from sqlalchemy.future import select
from lbry.blockchain.block import PyBIP158, get_address_filter
from ..query_context import context
from ..tables import BlockFilter, TXFilter, TX
def has_filters():
return context().has_records(BlockFilter)
def get_sub_filter_range(granularity: int, height: int):
end = height
if granularity >= 3:
end = height + 10**(granularity-1) * 9
elif granularity == 2:
end = height + 99
return granularity - 1, height, end
def has_filter_range(factor: int, start: int, end: int):
if factor >= 1:
filters = context().fetchtotal(
(BlockFilter.c.factor == factor) &
between(BlockFilter.c.height, start, end)
)
expected = 10 if factor >= 2 else 100
return filters == expected
elif factor == 0:
tx_filters_count = context().fetchtotal(TXFilter.c.height == start)
return tx_filters_count > 0
def get_filters(start_height, end_height=None, granularity=0):
assert granularity >= 0, "filter granularity must be 0 or positive number"
if granularity == 0:
query = (
select(TXFilter.c.height, TXFilter.c.address_filter, TXFilter.c.tx_hash)
.select_from(TXFilter)
.where(between(TXFilter.c.height, start_height, end_height))
.order_by(TXFilter.c.height)
)
else:
factor = granularity if granularity < 100 else log10(granularity)
if end_height is None:
height_condition = (BlockFilter.c.height == start_height)
elif end_height == -1:
height_condition = (BlockFilter.c.height >= start_height)
else:
height_condition = between(BlockFilter.c.height, start_height, end_height)
query = (
select(BlockFilter.c.height, BlockFilter.c.address_filter)
.select_from(BlockFilter)
.where(height_condition & (BlockFilter.c.factor == factor))
.order_by(BlockFilter.c.height)
)
return context().fetchall(query)
def get_minimal_required_filter_ranges(height) -> Dict[int, Tuple[int, int]]:
minimal = {}
if height >= 100_000:
minimal[5] = (0, ((height // 100_000)-1) * 100_000)
if height >= 10_000:
start = height - height % 100_000
minimal[4] = (start, start + (((height - start) // 10_000) - 1) * 10_000)
if height >= 1_000:
start = height - height % 10_000
minimal[3] = (start, start+(((height-start) // 1_000)-1) * 1_000)
if height >= 100:
start = height - height % 1_000
minimal[2] = (start, start+(((height-start) // 100)-1) * 100)
start = height - height % 100
if start < height:
minimal[1] = (start, height)
return minimal
def get_maximum_known_filters() -> Dict[str, Optional[int]]:
query = select(
select(func.max(BlockFilter.c.height))
.where(BlockFilter.c.factor == 1)
.scalar_subquery().label('1'),
select(func.max(BlockFilter.c.height))
.where(BlockFilter.c.factor == 2)
.scalar_subquery().label('2'),
select(func.max(BlockFilter.c.height))
.where(BlockFilter.c.factor == 3)
.scalar_subquery().label('3'),
select(func.max(BlockFilter.c.height))
.where(BlockFilter.c.factor == 4)
.scalar_subquery().label('4'),
select(func.max(BlockFilter.c.height))
.where(BlockFilter.c.factor == 5)
.scalar_subquery().label('5'),
)
return context().fetchone(query)
def get_missing_required_filters(height) -> Set[Tuple[int, int, int]]:
known_filters = get_maximum_known_filters()
missing_filters = set()
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
known_height = known_filters.get(str(granularity))
if known_height is not None and known_height >= start:
if granularity == 1:
adjusted_height = known_height + 1
else:
adjusted_height = known_height + 10**granularity
if adjusted_height <= end:
missing_filters.add((granularity, adjusted_height, end))
else:
missing_filters.add((granularity, start, end))
return missing_filters
def get_filter_matchers(height) -> List[Tuple[PyBIP158, Tuple[int, int, int]]]:
conditions = []
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
conditions.append(
(BlockFilter.c.factor == granularity) &
between(BlockFilter.c.height, start, end)
)
query = (
select(BlockFilter.c.factor, BlockFilter.c.height, BlockFilter.c.address_filter)
.select_from(BlockFilter)
.where(or_(*conditions))
.order_by(BlockFilter.c.height.desc())
)
return [(
get_address_filter(bf["address_filter"]),
get_sub_filter_range(bf["factor"], bf["height"])
) for bf in context().fetchall(query)]
def get_filter_matchers_at_granularity(granularity) -> List[Tuple[PyBIP158, Tuple]]:
query = (
select(BlockFilter.c.height, BlockFilter.c.address_filter)
.where(BlockFilter.c.factor == granularity)
.order_by(BlockFilter.c.height.desc())
)
return [(
get_address_filter(bf["address_filter"]),
get_sub_filter_range(granularity, bf["height"])
) for bf in context().fetchall(query)]
def get_tx_matchers_for_missing_txs() -> List[Tuple[int, PyBIP158]]:
query = (
select(TXFilter.c.tx_hash, TXFilter.c.address_filter)
.where(TXFilter.c.tx_hash.notin_(select(TX.c.tx_hash)))
)
return [
(bf["tx_hash"], get_address_filter(bf["address_filter"]))
for bf in context().fetchall(query)
]
def insert_block_filters(filters):
loader = context().get_bulk_loader()
for height, factor, address_filter in filters:
loader.add_block_filter(height, factor, address_filter)
loader.flush(return_row_count_for_table=None)
def insert_tx_filters(filters):
loader = context().get_bulk_loader()
for tx_hash, height, address_filter in filters:
loader.add_transaction_filter(tx_hash, height, address_filter)
loader.flush(return_row_count_for_table=None)
+101
View File
@@ -0,0 +1,101 @@
import logging
import itertools
from typing import List, Dict
from lbry.schema.url import URL
from lbry.schema.result import Outputs as ResultOutput
from lbry.error import ResolveCensoredError
from lbry.blockchain.transaction import Output
from . import rows_to_txos
from ..query_context import context
from .search import select_claims
log = logging.getLogger(__name__)
def resolve_claims(**constraints):
censor = context().get_resolve_censor()
rows = context().fetchall(select_claims(**constraints))
rows = censor.apply(rows)
return rows_to_txos(rows), censor
def _get_referenced_rows(txo_rows: List[Output], censor_channels: List[bytes]):
repost_hashes = set(txo.reposted_claim.claim_hash for txo in txo_rows if txo.reposted_claim)
channel_hashes = set(itertools.chain(
(txo.channel.claim_hash for txo in txo_rows if txo.channel),
censor_channels
))
reposted_txos = []
if repost_hashes:
reposted_txos = resolve_claims(**{'claim.claim_hash__in': repost_hashes})
if reposted_txos:
reposted_txos = reposted_txos[0]
channel_hashes |= set(txo.channel.claim_hash for txo in reposted_txos if txo.channel)
channel_txos = []
if channel_hashes:
channel_txos = resolve_claims(**{'claim.claim_hash__in': channel_hashes})
channel_txos = channel_txos[0] if channel_txos else []
# channels must come first for client side inflation to work properly
return channel_txos + reposted_txos
def protobuf_resolve(urls, **kwargs) -> str:
txo_rows = [resolve_url(raw_url) for raw_url in urls]
extra_txo_rows = _get_referenced_rows(
[txo_row for txo_row in txo_rows if isinstance(txo_row, Output)],
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
)
return ResultOutput.to_base64(txo_rows, extra_txo_rows)
def resolve(urls, **kwargs) -> Dict[str, Output]:
return {url: resolve_url(url) for url in urls}
def resolve_url(raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
return e
channel = None
if url.has_channel:
q = url.channel.to_dict()
if set(q) == {'name'}:
q['is_controlling'] = True
else:
q['order_by'] = ['^creation_height']
matches, censor = resolve_claims(**q, limit=1)
if matches:
channel = matches[0]
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
elif not channel:
return LookupError(f'Could not find channel in "{raw_url}".')
if url.has_stream:
q = url.stream.to_dict()
if channel is not None:
q['order_by'] = ['^creation_height']
q['channel_hash'] = channel.claim_hash
q['is_signature_valid'] = True
elif set(q) == {'name'}:
q['is_controlling'] = True
matches, censor = resolve_claims(**q, limit=1)
if matches:
stream = matches[0]
stream.channel = channel
return stream
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
else:
return LookupError(f'Could not find claim at "{raw_url}".')
return channel
+472
View File
@@ -0,0 +1,472 @@
import struct
import logging
from decimal import Decimal
from binascii import unhexlify
from typing import Tuple, List, Optional, Dict
from sqlalchemy import func, case, text
from sqlalchemy.future import select, Select
from lbry.schema.tags import clean_tags
from lbry.schema.result import Censor, Outputs as ResultOutput
from lbry.schema.url import normalize_name
from lbry.blockchain.transaction import Output
from ..utils import query
from ..query_context import context
from ..tables import TX, TXO, Claim, Support, Trend, CensoredClaim
from ..constants import (
TXO_TYPES, STREAM_TYPES, ATTRIBUTE_ARRAY_MAX_LENGTH,
SEARCH_INTEGER_PARAMS, SEARCH_ORDER_FIELDS
)
from .txio import BASE_SELECT_TXO_COLUMNS, rows_to_txos
log = logging.getLogger(__name__)
BASE_SELECT_SUPPORT_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
Support.c.channel_hash,
Support.c.is_signature_valid,
]
def compat_layer(**constraints):
# for old sdk, to be removed later
replacements = {
"effective_amount": "staked_amount",
"trending_mixed": "trend_mixed",
"trending_group": "trend_group",
"trending_local": "trend_local"
}
for old_key, new_key in replacements.items():
if old_key in constraints:
constraints[new_key] = constraints.pop(old_key)
order_by = constraints.get("order_by", [])
if old_key in order_by:
constraints["order_by"] = [order_key if order_key != old_key else new_key for order_key in order_by]
return constraints
def select_supports(cols: List = None, **constraints) -> Select:
if cols is None:
cols = BASE_SELECT_SUPPORT_COLUMNS
joins = Support.join(TXO, ).join(TX)
return query([Support], select(*cols).select_from(joins), **constraints)
def search_supports(**constraints) -> Tuple[List[Output], Optional[int]]:
total = None
if constraints.pop('include_total', False):
total = search_support_count(**constraints)
if 'claim_id' in constraints:
constraints['claim_hash'] = unhexlify(constraints.pop('claim_id'))[::-1]
rows = context().fetchall(select_supports(**constraints))
txos = rows_to_txos(rows, include_tx=False)
return txos, total
def sum_supports(claim_hash, include_channel_content=False, exclude_own_supports=False) -> Tuple[List[Dict], int]:
supporter = Claim.alias("supporter")
content = Claim.alias("content")
where_condition = (content.c.claim_hash == claim_hash)
if include_channel_content:
where_condition |= (content.c.channel_hash == claim_hash)
support_join_condition = TXO.c.channel_hash == supporter.c.claim_hash
if exclude_own_supports:
support_join_condition &= TXO.c.channel_hash != claim_hash
q = select(
supporter.c.short_url.label("supporter"),
func.sum(TXO.c.amount).label("staked"),
).select_from(
TXO
.join(content, TXO.c.claim_hash == content.c.claim_hash)
.join(supporter, support_join_condition)
).where(
where_condition &
(TXO.c.txo_type == TXO_TYPES["support"]) &
((TXO.c.address == content.c.address) | ((TXO.c.address != content.c.address) & (TXO.c.spent_height == 0)))
).group_by(
supporter.c.short_url
).order_by(
text("staked DESC, supporter ASC")
)
result = context().fetchall(q)
total = sum([row['staked'] for row in result])
return result, total
def search_support_count(**constraints) -> int:
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = context().fetchall(select_supports([func.count().label('total')], **constraints))
return count[0]['total'] or 0
channel_claim = Claim.alias('channel')
BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
Claim.c.activation_height,
Claim.c.takeover_height,
Claim.c.creation_height,
Claim.c.expiration_height,
Claim.c.is_controlling,
Claim.c.channel_hash,
Claim.c.reposted_count,
Claim.c.reposted_claim_hash,
Claim.c.short_url,
Claim.c.signed_claim_count,
Claim.c.signed_support_count,
(Claim.c.amount + Claim.c.staked_support_amount).label('staked_amount'),
Claim.c.staked_support_amount,
Claim.c.staked_support_count,
Claim.c.is_signature_valid,
case([(
channel_claim.c.short_url.isnot(None),
channel_claim.c.short_url + '/' + Claim.c.short_url
)]).label('canonical_url'),
func.coalesce(Trend.c.trend_local, 0).label('trend_local'),
func.coalesce(Trend.c.trend_mixed, 0).label('trend_mixed'),
func.coalesce(Trend.c.trend_global, 0).label('trend_global'),
func.coalesce(Trend.c.trend_group, 0).label('trend_group'),
CensoredClaim.c.censor_type,
CensoredClaim.c.censoring_channel_hash
]
def select_claims(cols: List = None, for_count=False, **constraints) -> Select:
constraints = compat_layer(**constraints)
if cols is None:
cols = BASE_SELECT_CLAIM_COLUMNS
if 'order_by' in constraints:
order_by_parts = constraints['order_by']
if isinstance(order_by_parts, str):
order_by_parts = [order_by_parts]
sql_order_by = []
for order_by in order_by_parts:
is_asc = order_by.startswith('^')
column = order_by[1:] if is_asc else order_by
if column not in SEARCH_ORDER_FIELDS:
raise NameError(f'{column} is not a valid order_by field')
if column == 'name':
column = 'claim_name'
table = "trend" if column.startswith('trend') else "claim"
column = f"{table}.{column}"
if column in ('trend.trend_group', 'trend.trend_mixed', 'claim.release_time'):
column = f"COALESCE({column}, {-1 * (1<<32)})"
sql_order_by.append(
f"{column} {'ASC' if is_asc else 'DESC'}"
)
constraints['order_by'] = sql_order_by
ops = {'<=': '__lte', '>=': '__gte', '<': '__lt', '>': '__gt'}
for constraint in SEARCH_INTEGER_PARAMS:
if constraint in constraints:
value = constraints.pop(constraint)
postfix = ''
if isinstance(value, str):
if len(value) >= 2 and value[:2] in ops:
postfix, value = ops[value[:2]], value[2:]
elif len(value) >= 1 and value[0] in ops:
postfix, value = ops[value[0]], value[1:]
if constraint == 'fee_amount':
value = Decimal(value)*1000
constraints[f'{constraint}{postfix}'] = int(value)
if 'sequence' in constraints:
constraints['order_by'] = 'activation_height ASC'
constraints['offset'] = int(constraints.pop('sequence')) - 1
constraints['limit'] = 1
if 'amount_order' in constraints:
constraints['order_by'] = 'staked_amount DESC'
constraints['offset'] = int(constraints.pop('amount_order')) - 1
constraints['limit'] = 1
if 'claim_id' in constraints:
claim_id = constraints.pop('claim_id')
if len(claim_id) == 40:
constraints['claim_id'] = claim_id
else:
constraints['claim_id__like'] = f'{claim_id[:40]}%'
elif 'claim_ids' in constraints:
constraints['claim_id__in'] = set(constraints.pop('claim_ids'))
if 'reposted_claim_id' in constraints:
constraints['reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
if 'name' in constraints:
constraints['normalized'] = normalize_name(constraints.pop('name'))
if 'public_key_id' in constraints:
constraints['public_key_hash'] = (
context().ledger.address_to_hash160(constraints.pop('public_key_id')))
if 'channel_id' in constraints:
channel_id = constraints.pop('channel_id')
if channel_id:
if isinstance(channel_id, str):
channel_id = [channel_id]
constraints['channel_hash__in'] = {
unhexlify(cid)[::-1] for cid in channel_id
}
if 'not_channel_id' in constraints:
not_channel_ids = constraints.pop('not_channel_id')
if not_channel_ids:
not_channel_ids_binary = {
unhexlify(ncid)[::-1] for ncid in not_channel_ids
}
constraints['claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
if constraints.get('has_channel_signature', False):
constraints['channel_hash__not_in'] = not_channel_ids_binary
else:
constraints['null_or_not_channel__or'] = {
'signature_valid__is_null': True,
'channel_hash__not_in': not_channel_ids_binary
}
if 'is_signature_valid' in constraints:
has_channel_signature = constraints.pop('has_channel_signature', False)
is_signature_valid = constraints.pop('is_signature_valid')
if has_channel_signature:
constraints['is_signature_valid'] = is_signature_valid
else:
constraints['null_or_signature__or'] = {
'is_signature_valid__is_null': True,
'is_signature_valid': is_signature_valid
}
elif constraints.pop('has_channel_signature', False):
constraints['is_signature_valid__is_not_null'] = True
if 'txid' in constraints:
tx_hash = unhexlify(constraints.pop('txid'))[::-1]
nout = constraints.pop('nout', 0)
constraints['txo_hash'] = tx_hash + struct.pack('<I', nout)
if 'claim_type' in constraints:
claim_types = constraints.pop('claim_type')
if isinstance(claim_types, str):
claim_types = {claim_types}
if claim_types:
constraints['claim_type__in'] = {
TXO_TYPES[claim_type] for claim_type in claim_types
}
if 'stream_type' in constraints:
stream_types = constraints.pop('stream_type')
if isinstance(stream_types, str):
stream_types = {stream_types}
if stream_types:
constraints['stream_type__in'] = {
STREAM_TYPES[stream_type] for stream_type in stream_types
}
if 'media_type' in constraints:
media_types = constraints.pop('media_type')
if isinstance(media_types, str):
media_types = {media_types}
if media_types:
constraints['media_type__in'] = set(media_types)
if 'fee_currency' in constraints:
constraints['fee_currency'] = constraints.pop('fee_currency').lower()
_apply_constraints_for_array_attributes(constraints, 'tag', clean_tags, for_count)
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
if 'text' in constraints:
# TODO: fix
constraints["search"] = constraints.pop("text")
return query(
[Claim, TXO],
select(*cols)
.select_from(
Claim.join(TXO).join(TX)
.join(Trend, Trend.c.claim_hash == Claim.c.claim_hash, isouter=True)
.join(channel_claim, Claim.c.channel_hash == channel_claim.c.claim_hash, isouter=True)
.join(
CensoredClaim,
(CensoredClaim.c.claim_hash == Claim.c.claim_hash) |
(CensoredClaim.c.claim_hash == Claim.c.reposted_claim_hash) |
(CensoredClaim.c.claim_hash == Claim.c.channel_hash),
isouter=True
)
), **constraints
)
def protobuf_search_claims(**constraints) -> str:
txos, _, censor = search_claims(**constraints)
return ResultOutput.to_base64(txos, [], blocked=censor)
def search_claims(**constraints) -> Tuple[List[Output], Optional[int], Optional[Censor]]:
ctx = context()
search_censor = ctx.get_search_censor()
total = None
if constraints.pop('include_total', False):
total = search_claim_count(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
channel_url = constraints.pop('channel', None)
if channel_url:
from .resolve import resolve_url # pylint: disable=import-outside-toplevel
channel = resolve_url(channel_url)
if isinstance(channel, Output):
constraints['channel_hash'] = channel.claim_hash
else:
return [], total, search_censor
rows = ctx.fetchall(select_claims(**constraints))
rows = search_censor.apply(rows)
txos = rows_to_txos(rows, include_tx=False)
annotate_with_channels(txos)
return txos, total, search_censor
def annotate_with_channels(txos):
channel_hashes = set()
for txo in txos:
if txo.can_decode_claim and txo.claim.is_signed:
channel_hashes.add(txo.claim.signing_channel_hash)
if channel_hashes:
rows = context().fetchall(select_claims(claim_hash__in=channel_hashes))
channels = {
txo.claim_hash: txo for txo in
rows_to_txos(rows, include_tx=False)
}
for txo in txos:
if txo.can_decode_claim and txo.claim.is_signed:
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
def search_claim_count(**constraints) -> int:
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = context().fetchall(select_claims([func.count().label('total')], **constraints))
return count[0]['total'] or 0
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
CASE WHEN claim.claim_type = {TXO_TYPES['repost']}
THEN claim.reposted_claim_hash
ELSE claim.claim_hash
END
"""
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
not_items = set(cleaner(constraints.pop(f'not_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
# if attr == 'tag':
# common_tags = any_items & COMMON_TAGS.keys()
# if common_tags:
# any_items -= common_tags
# if len(common_tags) < 5:
# for item in common_tags:
# index_name = COMMON_TAGS[item]
# any_queries[f'#_common_tag_{index_name}'] = f"""
# EXISTS(
# SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
# WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
# AND tag = '{item}'
# )
# """
# elif len(common_tags) >= 5:
# constraints.update({
# f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
# })
# values = ', '.join(
# f':$any_common_tag{i}' for i in range(len(common_tags))
# )
# any_queries[f'#_any_common_tags'] = f"""
# EXISTS(
# SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
# AND tag IN ({values})
# )
# """
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
f'$all_{attr}{i}': item for i, item in enumerate(all_items)
})
values = ', '.join(
f':$all_{attr}{i}' for i in range(len(all_items))
)
if for_count:
constraints[f'#_all_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
)
"""
else:
constraints[f'#_all_{attr}'] = f"""
{len(all_items)}=(
SELECT count(*) FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if not_items:
constraints.update({
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
})
values = ', '.join(
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
SELECT 1 FROM {attr} WHERE
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
AND {attr} IN ({values})
)
"""
+676
View File
@@ -0,0 +1,676 @@
import logging
from datetime import date
from typing import Tuple, List, Optional, Union
from sqlalchemy import union, func, text, between, distinct, case, false, not_, exists
from sqlalchemy.future import select, Select
from lbry.constants import INVALIDATED_SIGNATURE_GRACE_PERIOD
from ...blockchain.transaction import (
Transaction, Output, OutputScript, TXRefImmutable
)
from ..tables import (
TX, TXO, TXI, txi_join_account, txo_join_account,
Claim, Support, AccountAddress
)
from ..utils import query, in_account_ids
from ..query_context import context
from ..constants import TXO_TYPES, CLAIM_TYPE_CODES, MAX_QUERY_VARIABLES
log = logging.getLogger(__name__)
minimum_txo_columns = (
TXO.c.amount, TXO.c.position.label('txo_position'),
TX.c.tx_hash, TX.c.height, TX.c.timestamp,
func.substr(TX.c.raw, TXO.c.script_offset + 1, TXO.c.script_length).label('src'),
)
def row_to_txo(row):
return Output(
amount=row.amount,
script=OutputScript(row.src),
tx_ref=TXRefImmutable.from_hash(row.tx_hash, row.height, row.timestamp),
position=row.txo_position,
)
def where_txo_type_in(txo_type: Optional[Union[tuple, int]] = None):
if txo_type is not None:
if isinstance(txo_type, int):
return TXO.c.txo_type == txo_type
assert len(txo_type) > 0
if len(txo_type) == 1:
return TXO.c.txo_type == txo_type[0]
else:
return TXO.c.txo_type.in_(txo_type)
return TXO.c.txo_type.in_(CLAIM_TYPE_CODES)
def where_unspent_txos(
txo_types: Tuple[int, ...],
blocks: Tuple[int, int] = None,
missing_in_supports_table: bool = False,
missing_in_claims_table: bool = False,
missing_or_stale_in_claims_table: bool = False,
):
condition = where_txo_type_in(txo_types) & (TXO.c.spent_height == 0)
if blocks is not None:
condition &= between(TXO.c.height, *blocks)
if missing_in_supports_table:
condition &= not_(
exists(select(1).where(Support.c.txo_hash == TXO.c.txo_hash))
)
elif missing_or_stale_in_claims_table:
condition &= not_(
exists(select(1).where(Claim.c.txo_hash == TXO.c.txo_hash))
)
elif missing_in_claims_table:
condition &= not_(
exists(select(1).where(Claim.c.claim_hash == TXO.c.claim_hash))
)
return condition
def where_abandoned_claims():
return Claim.c.claim_hash.notin_(
select(TXO.c.claim_hash).where(where_unspent_txos(CLAIM_TYPE_CODES))
)
def count_abandoned_claims():
return context().fetchtotal(where_abandoned_claims())
def where_abandoned_supports():
return Support.c.txo_hash.notin_(
select(TXO.c.txo_hash).where(where_unspent_txos(TXO_TYPES['support']))
)
def count_abandoned_supports():
return context().fetchtotal(where_abandoned_supports())
def count_unspent_txos(
txo_types: Tuple[int, ...],
blocks: Tuple[int, int] = None,
missing_in_supports_table: bool = False,
missing_in_claims_table: bool = False,
missing_or_stale_in_claims_table: bool = False,
):
return context().fetchtotal(
where_unspent_txos(
txo_types, blocks,
missing_in_supports_table,
missing_in_claims_table,
missing_or_stale_in_claims_table,
)
)
def distribute_unspent_txos(
txo_types: Tuple[int, ...],
blocks: Tuple[int, int] = None,
missing_in_supports_table: bool = False,
missing_in_claims_table: bool = False,
missing_or_stale_in_claims_table: bool = False,
number_of_buckets: int = 10
) -> Tuple[int, List[Tuple[int, int]]]:
chunks = (
select(func.ntile(number_of_buckets).over(order_by=TXO.c.height).label('chunk'), TXO.c.height)
.where(
where_unspent_txos(
txo_types, blocks,
missing_in_supports_table,
missing_in_claims_table,
missing_or_stale_in_claims_table,
)
).cte('chunks')
)
sql = (
select(
func.count('*').label('items'),
func.min(chunks.c.height).label('start_height'),
func.max(chunks.c.height).label('end_height'),
).group_by(chunks.c.chunk).order_by(chunks.c.chunk)
)
total = 0
buckets = []
for bucket in context().fetchall(sql):
total += bucket['items']
if len(buckets) > 0:
if buckets[-1][-1] == bucket['start_height']:
if bucket['start_height'] == bucket['end_height']:
continue
bucket['start_height'] += 1
buckets.append((bucket['start_height'], bucket['end_height']))
return total, buckets
def where_changed_support_txos(blocks: Optional[Tuple[int, int]]):
return (
(TXO.c.txo_type == TXO_TYPES['support']) & (
between(TXO.c.height, blocks[0], blocks[-1]) |
between(TXO.c.spent_height, blocks[0], blocks[-1])
)
)
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]):
return Claim.c.claim_hash.in_(
select(TXO.c.claim_hash).where(
where_changed_support_txos(blocks)
)
)
def count_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> int:
sql = (
select(func.count(distinct(TXO.c.claim_hash)).label('total'))
.where(where_changed_support_txos(blocks))
)
return context().fetchone(sql)['total']
def where_channels_changed(blocks: Optional[Tuple[int, int]]):
channel = TXO.alias('channel')
return TXO.c.channel_hash.in_(
select(channel.c.claim_hash).where(
(channel.c.txo_type == TXO_TYPES['channel']) & (
between(channel.c.height, blocks[0], blocks[-1]) |
between(channel.c.spent_height, blocks[0], blocks[-1])
)
)
)
def where_changed_content_txos(blocks: Optional[Tuple[int, int]]):
return (
(TXO.c.channel_hash.isnot(None)) & (
between(TXO.c.height, blocks[0], blocks[-1]) |
between(TXO.c.spent_height, blocks[0], blocks[-1])
)
)
def where_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
content = Claim.alias("content")
return Claim.c.claim_hash.in_(
union(
select(TXO.c.channel_hash).where(where_changed_content_txos(blocks)),
select(content.c.channel_hash).where(
content.c.channel_hash.isnot(None) &
# content.c.public_key_height is updated when
# channel signature is revalidated
between(content.c.public_key_height, blocks[0], blocks[-1])
)
)
)
def count_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
return context().fetchtotal(where_channels_with_changed_content(blocks))
def where_changed_repost_txos(blocks: Optional[Tuple[int, int]]):
return (
(TXO.c.txo_type == TXO_TYPES['repost']) & (
between(TXO.c.height, blocks[0], blocks[-1]) |
between(TXO.c.spent_height, blocks[0], blocks[-1])
)
)
def where_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
return Claim.c.claim_hash.in_(
select(TXO.c.reposted_claim_hash).where(
where_changed_repost_txos(blocks)
)
)
def count_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
sql = (
select(func.count(distinct(TXO.c.reposted_claim_hash)).label('total'))
.where(where_changed_repost_txos(blocks))
)
return context().fetchone(sql)['total']
def where_claims_with_stale_signatures(s, blocks: Optional[Tuple[int, int]], stream=None):
stream = Claim.alias('stream') if stream is None else stream
channel = Claim.alias('channel')
return (
s.select_from(stream.join(channel, stream.c.channel_hash == channel.c.claim_hash))
.where(
(stream.c.public_key_height < channel.c.public_key_height) &
(stream.c.public_key_height <= blocks[1]-INVALIDATED_SIGNATURE_GRACE_PERIOD)
)
)
def count_claims_with_stale_signatures(blocks: Optional[Tuple[int, int]]):
return context().fetchone(
where_claims_with_stale_signatures(select(func.count('*').label('total')), blocks)
)['total']
def select_transactions(cols, account_ids=None, **constraints):
s: Select = select(*cols).select_from(TX)
if not {'tx_hash', 'tx_hash__in'}.intersection(constraints):
assert account_ids, (
"'accounts' argument required when "
"no 'tx_hash' constraint is present"
)
where = in_account_ids(account_ids)
tx_hashes = union(
select(TXO.c.tx_hash).select_from(txo_join_account).where(where),
select(TXI.c.tx_hash).select_from(txi_join_account).where(where)
)
s = s.where(TX.c.tx_hash.in_(tx_hashes))
return context().fetchall(query([TX], s, **constraints))
TXO_NOT_MINE = Output(None, None, is_my_output=False)
def get_raw_transactions(tx_hash__in):
return context().fetchall(
select(TX.c.tx_hash, TX.c.raw).where(TX.c.tx_hash.in_(tx_hash__in))
)
def get_transactions(include_total=False, **constraints) -> Tuple[List[Transaction], Optional[int]]:
account_ids = constraints.pop('account_ids', None)
include_is_my_input = constraints.pop('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
tx_rows = select_transactions(
[TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position, TX.c.timestamp, TX.c.is_verified],
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
account_ids=account_ids,
**constraints
)
txids, txs, txi_txoids = [], [], []
for row in tx_rows:
txids.append(row['tx_hash'])
txs.append(Transaction(
raw=row['raw'], height=row['height'], position=row['position'],
timestamp=row['timestamp'], is_verified=bool(row['is_verified'])
))
for txi in txs[-1].inputs:
txi_txoids.append(txi.txo_ref.hash)
annotated_txos = {}
for offset in range(0, len(txids), MAX_QUERY_VARIABLES):
annotated_txos.update({
txo.id: txo for txo in
get_txos(
wallet_account_ids=account_ids,
tx_hash__in=txids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.tx_hash',
include_is_my_input=include_is_my_input,
include_is_my_output=include_is_my_output,
)[0]
})
referenced_txos = {}
for offset in range(0, len(txi_txoids), MAX_QUERY_VARIABLES):
referenced_txos.update({
txo.id: txo for txo in
get_txos(
wallet_account_ids=account_ids,
txo_hash__in=txi_txoids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.txo_hash',
include_is_my_output=include_is_my_output,
)[0]
})
for tx in txs:
for txi in tx.inputs:
txo = referenced_txos.get(txi.txo_ref.id)
if txo:
txi.txo_ref = txo.ref
for txo in tx.outputs:
_txo = annotated_txos.get(txo.id)
if _txo:
txo.update_annotations(_txo)
else:
txo.update_annotations(TXO_NOT_MINE)
for tx in txs:
txos = tx.outputs
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
txos[0].purchase = txos[1]
return txs, get_transaction_count(**constraints) if include_total else None
def get_transaction_count(**constraints):
constraints.pop('wallet', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = select_transactions([func.count().label('total')], **constraints)
return count[0]['total'] or 0
BASE_SELECT_TXO_COLUMNS = [
TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position.label('tx_position'),
TX.c.is_verified, TX.c.timestamp,
TXO.c.txo_type, TXO.c.position.label('txo_position'), TXO.c.amount, TXO.c.spent_height,
TXO.c.script_offset, TXO.c.script_length,
]
def select_txos(
cols=None, account_ids=None, is_my_input=None,
is_my_output=True, is_my_input_or_output=None, exclude_internal_transfers=False,
include_is_my_input=False, claim_id_not_in_claim_table=None,
txo_id_not_in_claim_table=None, txo_id_not_in_support_table=None,
**constraints
) -> Select:
if cols is None:
cols = BASE_SELECT_TXO_COLUMNS
s: Select = select(*cols)
if account_ids:
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
if is_my_input_or_output:
include_is_my_input = True
s = s.where(
TXO.c.address.in_(my_addresses) | (
(TXI.c.address.isnot(None)) &
(TXI.c.address.in_(my_addresses))
)
)
else:
if is_my_output:
s = s.where(TXO.c.address.in_(my_addresses))
elif is_my_output is False:
s = s.where(TXO.c.address.notin_(my_addresses))
if is_my_input:
include_is_my_input = True
s = s.where(
(TXI.c.address.isnot(None)) &
(TXI.c.address.in_(my_addresses))
)
elif is_my_input is False:
include_is_my_input = True
s = s.where(
(TXI.c.address.is_(None)) |
(TXI.c.address.notin_(my_addresses))
)
if exclude_internal_transfers:
include_is_my_input = True
s = s.where(
(TXO.c.txo_type != TXO_TYPES['other']) |
(TXO.c.address.notin_(my_addresses))
(TXI.c.address.is_(None)) |
(TXI.c.address.notin_(my_addresses))
)
joins = TXO.join(TX)
if constraints.pop('is_spent', None) is False:
s = s.where((TXO.c.spent_height == 0) & (TXO.c.is_reserved == false()))
if include_is_my_input:
joins = joins.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
if claim_id_not_in_claim_table:
s = s.where(TXO.c.claim_hash.notin_(select(Claim.c.claim_hash)))
elif txo_id_not_in_claim_table:
s = s.where(TXO.c.txo_hash.notin_(select(Claim.c.txo_hash)))
elif txo_id_not_in_support_table:
s = s.where(TXO.c.txo_hash.notin_(select(Support.c.txo_hash)))
return query([TXO, TX], s.select_from(joins), **constraints)
META_ATTRS = (
'activation_height', 'takeover_height', 'creation_height', 'staked_amount',
'short_url', 'canonical_url', 'staked_support_amount', 'staked_support_count',
'signed_claim_count', 'signed_support_count', 'is_signature_valid',
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
'reposted_count', 'expiration_height',
)
def rows_to_txos(rows: List[dict], include_tx=True) -> List[Output]:
txos = []
tx_cache = {}
for row in rows:
if include_tx:
if row['tx_hash'] not in tx_cache:
tx_cache[row['tx_hash']] = Transaction(
row['raw'], height=row['height'], position=row['tx_position'],
timestamp=row['timestamp'],
is_verified=bool(row['is_verified']),
)
txo = tx_cache[row['tx_hash']].outputs[row['txo_position']]
else:
source = row['raw'][row['script_offset']:row['script_offset']+row['script_length']]
txo = Output(
amount=row['amount'],
script=OutputScript(source),
tx_ref=TXRefImmutable.from_hash(row['tx_hash'], row['height'], row['timestamp']),
position=row['txo_position'],
)
txo.spent_height = row['spent_height']
if 'is_my_input' in row:
txo.is_my_input = bool(row['is_my_input'])
if 'is_my_output' in row:
txo.is_my_output = bool(row['is_my_output'])
if 'is_my_input' in row and 'is_my_output' in row:
if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']:
txo.is_internal_transfer = True
else:
txo.is_internal_transfer = False
if 'received_tips' in row:
txo.received_tips = row['received_tips']
for attr in META_ATTRS:
if attr in row:
txo.meta[attr] = row[attr]
txos.append(txo)
return txos
def get_txos(no_tx=False, include_total=False, **constraints) -> Tuple[List[Output], Optional[int]]:
wallet_account_ids = constraints.pop('wallet_account_ids', [])
include_is_my_input = constraints.get('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
include_received_tips = constraints.pop('include_received_tips', False)
select_columns = BASE_SELECT_TXO_COLUMNS + [
TXO.c.claim_name
]
my_accounts = None
if wallet_account_ids:
my_accounts = select(AccountAddress.c.address).where(in_account_ids(wallet_account_ids))
if include_is_my_output and my_accounts is not None:
if constraints.get('is_my_output', None) in (True, False):
select_columns.append(text(f"{1 if constraints['is_my_output'] else 0} AS is_my_output"))
else:
select_columns.append(TXO.c.address.in_(my_accounts).label('is_my_output'))
if include_is_my_input and my_accounts is not None:
if constraints.get('is_my_input', None) in (True, False):
select_columns.append(text(f"{1 if constraints['is_my_input'] else 0} AS is_my_input"))
else:
select_columns.append((
(TXI.c.address.isnot(None)) &
(TXI.c.address.in_(my_accounts))
).label('is_my_input'))
if include_received_tips:
support = TXO.alias('support')
select_columns.append(
select(func.coalesce(func.sum(support.c.amount), 0))
.select_from(support).where(
(support.c.claim_hash == TXO.c.claim_hash) &
(support.c.txo_type == TXO_TYPES['support']) &
(support.c.address.in_(my_accounts)) &
(support.c.txo_hash.notin_(select(TXI.c.txo_hash)))
).label('received_tips')
)
if 'order_by' not in constraints or constraints['order_by'] == 'height':
constraints['order_by'] = [
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
]
elif constraints.get('order_by', None) == 'none':
del constraints['order_by']
rows = context().fetchall(select_txos(select_columns, **constraints))
txos = rows_to_txos(rows, not no_tx)
channel_hashes = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_hashes.add(txo.claim.signing_channel_hash)
if channel_hashes:
channels = {
txo.claim_hash: txo for txo in
get_txos(
txo_type=TXO_TYPES['channel'], spent_height=0,
wallet_account_ids=wallet_account_ids, claim_hash__in=channel_hashes
)[0]
}
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
return txos, get_txo_count(**constraints) if include_total else None
def _clean_txo_constraints_for_aggregation(constraints):
constraints.pop('include_is_my_input', None)
constraints.pop('include_is_my_output', None)
constraints.pop('include_received_tips', None)
constraints.pop('wallet_account_ids', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
def get_txo_count(**constraints):
_clean_txo_constraints_for_aggregation(constraints)
count = context().fetchall(select_txos([func.count().label('total')], **constraints))
return count[0]['total'] or 0
def get_txo_sum(**constraints):
_clean_txo_constraints_for_aggregation(constraints)
result = context().fetchall(select_txos([func.sum(TXO.c.amount).label('total')], **constraints))
return result[0]['total'] or 0
def get_balance(account_ids):
ctx = context()
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
if ctx.is_postgres:
txo_address_check = TXO.c.address == func.any(func.array(my_addresses))
txi_address_check = TXI.c.address == func.any(func.array(my_addresses))
else:
txo_address_check = TXO.c.address.in_(my_addresses)
txi_address_check = TXI.c.address.in_(my_addresses)
s: Select = (
select(
func.coalesce(func.sum(TXO.c.amount), 0).label("total"),
func.coalesce(func.sum(case(
[(TXO.c.txo_type != TXO_TYPES["other"], TXO.c.amount)],
)), 0).label("reserved"),
func.coalesce(func.sum(case(
[(where_txo_type_in(CLAIM_TYPE_CODES), TXO.c.amount)],
)), 0).label("claims"),
func.coalesce(func.sum(case(
[(where_txo_type_in(TXO_TYPES["support"]), TXO.c.amount)],
)), 0).label("supports"),
func.coalesce(func.sum(case(
[(where_txo_type_in(TXO_TYPES["support"]) & (
(TXI.c.address.isnot(None)) & txi_address_check
), TXO.c.amount)],
)), 0).label("my_supports"),
)
.where((TXO.c.spent_height == 0) & txo_address_check)
.select_from(
TXO.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
)
)
result = ctx.fetchone(s)
return {
"total": result["total"],
"available": result["total"] - result["reserved"],
"reserved": result["reserved"],
"reserved_subtotals": {
"claims": result["claims"],
"supports": result["my_supports"],
"tips": result["supports"] - result["my_supports"]
}
}
def get_report(account_ids):
return
def get_txo_plot(start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
_clean_txo_constraints_for_aggregation(constraints)
if start_day is None:
# TODO: Fix
current_ordinal = 0 # self.ledger.headers.estimated_date(self.ledger.headers.height).toordinal()
constraints['day__gte'] = current_ordinal - days_back
else:
constraints['day__gte'] = date.fromisoformat(start_day).toordinal()
if end_day is not None:
constraints['day__lte'] = date.fromisoformat(end_day).toordinal()
elif days_after is not None:
constraints['day__lte'] = constraints['day__gte'] + days_after
plot = context().fetchall(select_txos(
[TX.c.day, func.sum(TXO.c.amount).label('total')],
group_by='day', order_by='day', **constraints
))
for row in plot:
row['day'] = date.fromordinal(row['day'])
return plot
def get_purchases(**constraints) -> Tuple[List[Output], Optional[int]]:
accounts = constraints.pop('accounts', None)
assert accounts, "'accounts' argument required to find purchases"
if not {'purchased_claim_hash', 'purchased_claim_hash__in'}.intersection(constraints):
constraints['purchased_claim_hash__is_not_null'] = True
constraints['tx_hash__in'] = (
select(TXI.c.tx_hash).select_from(txi_join_account).where(in_account_ids(accounts))
)
txs, count = get_transactions(**constraints)
return [tx.outputs[0] for tx in txs], count
def get_supports_summary(self, **constraints):
return get_txos(
txo_type=TXO_TYPES['support'],
spent_height=0, is_my_output=True,
include_is_my_input=True,
no_tx=True,
**constraints
)
def reserve_outputs(txo_hashes, is_reserved=True):
context().execute(
TXO.update()
.values(is_reserved=is_reserved)
.where(TXO.c.txo_hash.in_(txo_hashes))
)
def release_all_outputs(account_id):
context().execute(
TXO.update().values(is_reserved=False).where(
TXO.c.is_reserved & TXO.c.address.in_(
select(AccountAddress.c.address).where(in_account_ids(account_id))
)
)
)
+24
View File
@@ -0,0 +1,24 @@
from sqlalchemy import exists
from sqlalchemy.future import select
from ..query_context import context
from ..tables import Wallet
def has_wallet(wallet_id: str) -> bool:
sql = select(exists(select(Wallet.c.wallet_id).where(Wallet.c.wallet_id == wallet_id)))
return context().execute(sql).fetchone()[0]
def get_wallet(wallet_id: str):
return context().fetchone(
select(Wallet.c.data).where(Wallet.c.wallet_id == wallet_id)
)
def add_wallet(wallet_id: str, data: str):
c = context()
c.execute(
c.insert_or_replace(Wallet, ["data"])
.values(wallet_id=wallet_id, data=data)
)
+766
View File
@@ -0,0 +1,766 @@
import os
import time
import traceback
import functools
from io import BytesIO
import multiprocessing as mp
from decimal import Decimal
from typing import Dict, List, Optional, Tuple
from dataclasses import dataclass, field
from contextvars import ContextVar
from sqlalchemy import create_engine, inspect, bindparam, func, case, exists, event as sqlalchemy_event
from sqlalchemy.future import select
from sqlalchemy.engine import Engine
from sqlalchemy.sql import Insert, text
try:
from pgcopy import CopyManager
except ImportError:
CopyManager = None
from lbry.event import EventQueuePublisher
from lbry.blockchain.ledger import Ledger
from lbry.blockchain.transaction import Transaction, Output, Input
from lbry.schema.tags import clean_tags
from lbry.schema.result import Censor
from lbry.schema.mime_types import guess_stream_type
from .utils import pg_insert
from .tables import (
Block, BlockFilter,
TX, TXFilter, TXO, TXI, Claim, Tag, Support
)
from .constants import TXO_TYPES, STREAM_TYPES
_context: ContextVar['QueryContext'] = ContextVar('_context')
@dataclass
class QueryContext:
engine: Engine
ledger: Ledger
message_queue: mp.Queue
stop_event: mp.Event
stack: List[List]
metrics: Dict
is_tracking_metrics: bool
blocked_streams: Dict
blocked_channels: Dict
filtered_streams: Dict
filtered_channels: Dict
pid: int
# QueryContext __enter__/__exit__ state
current_timer_name: Optional[str] = None
current_timer_time: float = 0
current_progress: Optional['ProgressContext'] = None
copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
_variable_limit: Optional[int] = None
@property
def is_postgres(self):
return self.engine.dialect.name == 'postgresql'
@property
def is_sqlite(self):
return self.engine.dialect.name == 'sqlite'
@property
def variable_limit(self):
if self._variable_limit is not None:
return self._variable_limit
if self.is_sqlite:
for result in self.fetchall(text('PRAGMA COMPILE_OPTIONS;')):
for _, value in result.items():
if value.startswith('MAX_VARIABLE_NUMBER'):
self._variable_limit = int(value.split('=')[1])
return self._variable_limit
self._variable_limit = 999 # todo: default for 3.32.0 is 32766, but we are still hitting 999 somehow
else:
self._variable_limit = 32766
return self._variable_limit
def raise_unsupported_dialect(self):
raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')
@classmethod
def get_resolve_censor(cls) -> Censor:
return Censor(Censor.RESOLVE)
@classmethod
def get_search_censor(cls) -> Censor:
return Censor(Censor.SEARCH)
def pg_copy(self, table, rows):
with self.engine.begin() as c:
copy_manager = self.copy_managers.get(table.name)
if copy_manager is None:
self.copy_managers[table.name] = copy_manager = CopyManager(
c.connection, table.name, rows[0].keys()
)
copy_manager.conn = c.connection
copy_manager.copy(map(dict.values, rows), BytesIO)
copy_manager.conn = None
def pg_has_pk_constraint(self, table_name):
claim_constraints = self.fetchall(text(
f"select * from information_schema.table_constraints as tc "
f"where tc.table_name='{table_name}' and constraint_type='PRIMARY KEY'"
))
return len(claim_constraints) > 0
def connect_without_transaction(self):
return self.engine.connect().execution_options(isolation_level="AUTOCOMMIT")
def connect_streaming(self):
return self.engine.connect().execution_options(stream_results=True)
def execute_notx(self, sql, *args):
with self.connect_without_transaction() as c:
return c.execute(sql, *args)
def execute(self, sql, *args):
with self.engine.begin() as c:
return c.execute(sql, *args)
def fetchone(self, sql, *args):
with self.engine.begin() as c:
row = c.execute(sql, *args).fetchone()
return dict(row._mapping) if row else row
def fetchall(self, sql, *args):
with self.engine.begin() as c:
rows = c.execute(sql, *args).fetchall()
return [dict(row._mapping) for row in rows]
def fetchtotal(self, condition) -> int:
sql = select(func.count('*').label('total')).where(condition)
return self.fetchone(sql)['total']
def fetchmax(self, column, default: int) -> int:
sql = select(func.coalesce(func.max(column), default).label('max_result'))
return self.fetchone(sql)['max_result']
def has_records(self, table) -> bool:
sql = select(exists([1], from_obj=table).label('result'))
return bool(self.fetchone(sql)['result'])
def insert_or_ignore(self, table):
if self.is_sqlite:
return table.insert().prefix_with("OR IGNORE")
elif self.is_postgres:
return pg_insert(table).on_conflict_do_nothing()
else:
self.raise_unsupported_dialect()
def insert_or_replace(self, table, replace):
if self.is_sqlite:
return table.insert().prefix_with("OR REPLACE")
elif self.is_postgres:
insert = pg_insert(table)
return insert.on_conflict_do_update(
table.primary_key, set_={col: getattr(insert.excluded, col) for col in replace}
)
else:
self.raise_unsupported_dialect()
def has_table(self, table):
return inspect(self.engine).has_table(table)
def get_bulk_loader(self) -> 'BulkLoader':
return BulkLoader(self)
def reset_metrics(self):
self.stack = []
self.metrics = {}
def with_timer(self, timer_name: str) -> 'QueryContext':
self.current_timer_name = timer_name
return self
@property
def elapsed(self):
return time.perf_counter() - self.current_timer_time
def __enter__(self) -> 'QueryContext':
self.current_timer_time = time.perf_counter()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.current_timer_name = None
self.current_timer_time = 0
self.current_progress = None
def context(with_timer: str = None) -> 'QueryContext':
if isinstance(with_timer, str):
return _context.get().with_timer(with_timer)
return _context.get()
def set_postgres_settings(connection, _):
cursor = connection.cursor()
cursor.execute('SET work_mem="500MB";')
cursor.execute('COMMIT;')
cursor.close()
def set_sqlite_settings(connection, _):
connection.isolation_level = None
cursor = connection.cursor()
cursor.execute('PRAGMA journal_mode=WAL;')
cursor.close()
def do_sqlite_begin(connection):
# see: https://bit.ly/3j4vvXm
connection.exec_driver_sql("BEGIN")
def initialize(
ledger: Ledger, message_queue: mp.Queue, stop_event: mp.Event,
track_metrics=False, block_and_filter=None):
url = ledger.conf.db_url_or_default
engine = create_engine(url)
if engine.name == "postgresql":
sqlalchemy_event.listen(engine, "connect", set_postgres_settings)
elif engine.name == "sqlite":
sqlalchemy_event.listen(engine, "connect", set_sqlite_settings)
sqlalchemy_event.listen(engine, "begin", do_sqlite_begin)
if block_and_filter is not None:
blocked_streams, blocked_channels, filtered_streams, filtered_channels = block_and_filter
else:
blocked_streams = blocked_channels = filtered_streams = filtered_channels = {}
_context.set(
QueryContext(
pid=os.getpid(), engine=engine,
ledger=ledger, message_queue=message_queue, stop_event=stop_event,
stack=[], metrics={}, is_tracking_metrics=track_metrics,
blocked_streams=blocked_streams, blocked_channels=blocked_channels,
filtered_streams=filtered_streams, filtered_channels=filtered_channels,
)
)
def uninitialize():
ctx = _context.get(None)
if ctx is not None:
ctx.engine.dispose()
_context.set(None)
class Event:
_events: List['Event'] = []
__slots__ = 'id', 'name', 'units'
def __init__(self, name: str, units: Tuple[str]):
self.id = None
self.name = name
self.units = units
@classmethod
def get_by_id(cls, event_id) -> 'Event':
return cls._events[event_id]
@classmethod
def get_by_name(cls, name) -> 'Event':
for event in cls._events:
if event.name == name:
return event
@classmethod
def add(cls, name: str, *units: str) -> 'Event':
assert cls.get_by_name(name) is None, f"Event {name} already exists."
assert name.count('.') == 3, f"Event {name} does not follow pattern of: [module].sync.[phase].[task]"
event = cls(name, units)
cls._events.append(event)
event.id = cls._events.index(event)
return event
def event_emitter(name: str, *units: str, throttle=1):
event = Event.add(name, *units)
def wrapper(f):
@functools.wraps(f)
def with_progress(*args, **kwargs):
with progress(event, throttle=throttle) as p:
try:
return f(*args, **kwargs, p=p)
except BreakProgress:
raise
except:
traceback.print_exc()
raise
return with_progress
return wrapper
class ProgressPublisher(EventQueuePublisher):
def message_to_event(self, message):
total, extra = None, None
if len(message) == 3:
event_id, progress_id, done = message
elif len(message) == 5:
event_id, progress_id, done, total, extra = message
else:
raise TypeError("progress message must be tuple of 3 or 5 values.")
event = Event.get_by_id(event_id)
d = {
"event": event.name,
"data": {"id": progress_id, "done": done}
}
if total is not None:
d['data']['total'] = total
d['data']['units'] = event.units
if isinstance(extra, dict):
d['data'].update(extra)
return d
class BreakProgress(Exception):
"""Break out of progress when total is 0."""
class Progress:
def __init__(self, message_queue: mp.Queue, event: Event, throttle=1):
self.message_queue = message_queue
self.event = event
self.progress_id = 0
self.throttle = throttle
self.last_done = (0,)*len(event.units)
self.last_done_queued = (0,)*len(event.units)
self.totals = (0,)*len(event.units)
def __enter__(self) -> 'Progress':
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.last_done != self.last_done_queued:
self.message_queue.put((self.event.id, self.progress_id, self.last_done))
self.last_done_queued = self.last_done
if exc_type == BreakProgress:
return True
if self.last_done != self.totals: # or exc_type is not None:
# TODO: add exception info into closing message if there is any
self.message_queue.put((
self.event.id, self.progress_id, (-1,)*len(self.event.units)
))
def start(self, *totals: int, progress_id=0, label=None, extra=None):
assert len(totals) == len(self.event.units), \
f"Totals {totals} do not match up with units {self.event.units}."
if not any(totals):
raise BreakProgress
self.totals = totals
self.progress_id = progress_id
extra = {} if extra is None else extra.copy()
if label is not None:
extra['label'] = label
self.step(*((0,)*len(totals)), force=True, extra=extra)
def step(self, *done: int, force=False, extra=None):
if done == ():
assert len(self.totals) == 1, "Incrementing step() only works with one unit progress."
done = (self.last_done[0]+1,)
assert len(done) == len(self.totals), \
f"Done elements {done} don't match total elements {self.totals}."
self.last_done = done
send_condition = force or extra is not None or (
# throttle rate of events being generated (only throttles first unit value)
(self.throttle == 1 or done[0] % self.throttle == 0) and
# deduplicate finish event by not sending a step where done == total
any(i < j for i, j in zip(done, self.totals)) and
# deduplicate same event
done != self.last_done_queued
)
if send_condition:
if extra is not None:
self.message_queue.put_nowait(
(self.event.id, self.progress_id, done, self.totals, extra)
)
else:
self.message_queue.put_nowait(
(self.event.id, self.progress_id, done)
)
self.last_done_queued = done
def add(self, *done: int, force=False, extra=None):
assert len(done) == len(self.last_done), \
f"Done elements {done} don't match total elements {self.last_done}."
self.step(
*(i+j for i, j in zip(self.last_done, done)),
force=force, extra=extra
)
def iter(self, items: List):
self.start(len(items))
for item in items:
yield item
self.step()
class ProgressContext(Progress):
def __init__(self, ctx: QueryContext, event: Event, throttle=1):
super().__init__(ctx.message_queue, event, throttle)
self.ctx = ctx
def __enter__(self) -> 'ProgressContext':
self.ctx.__enter__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
return any((
self.ctx.__exit__(exc_type, exc_val, exc_tb),
super().__exit__(exc_type, exc_val, exc_tb)
))
def progress(e: Event, throttle=1) -> ProgressContext:
ctx = context(e.name)
ctx.current_progress = ProgressContext(ctx, e, throttle=throttle)
return ctx.current_progress
class BulkLoader:
def __init__(self, ctx: QueryContext):
self.ctx = ctx
self.ledger = ctx.ledger
self.blocks = []
self.txs = []
self.txos = []
self.txis = []
self.supports = []
self.claims = []
self.tags = []
self.update_claims = []
self.delete_tags = []
self.tx_filters = []
self.block_filters = []
@staticmethod
def block_to_row(block: Block) -> dict:
return {
'block_hash': block.block_hash,
'previous_hash': block.prev_block_hash,
'file_number': block.file_number,
'height': 0 if block.is_first_block else block.height,
'timestamp': block.timestamp,
}
@staticmethod
def tx_to_row(block_hash: bytes, tx: Transaction) -> dict:
row = {
'tx_hash': tx.hash,
#'block_hash': block_hash,
'raw': tx.raw,
'height': tx.height,
'position': tx.position,
'is_verified': tx.is_verified,
'timestamp': tx.timestamp,
'day': tx.day,
'purchased_claim_hash': None,
}
txos = tx.outputs
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
txos[0].purchase = txos[1]
row['purchased_claim_hash'] = txos[1].purchase_data.claim_hash
return row
@staticmethod
def txi_to_row(tx: Transaction, txi: Input) -> dict:
return {
'tx_hash': tx.hash,
'txo_hash': txi.txo_ref.hash,
'position': txi.position,
'height': tx.height,
}
def txo_to_row(self, tx: Transaction, txo: Output) -> dict:
row = {
'tx_hash': tx.hash,
'txo_hash': txo.hash,
'address': txo.get_address(self.ledger) if txo.has_address else None,
'position': txo.position,
'amount': txo.amount,
'height': tx.height,
'script_offset': txo.script.offset,
'script_length': txo.script.length,
'txo_type': 0,
'claim_id': None,
'claim_hash': None,
'claim_name': None,
'channel_hash': None,
'signature': None,
'signature_digest': None,
'reposted_claim_hash': None,
'public_key': None,
'public_key_hash': None
}
if txo.is_claim:
if txo.can_decode_claim:
claim = txo.claim
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
if claim.is_channel:
row['public_key'] = claim.channel.public_key_bytes
row['public_key_hash'] = self.ledger.address_to_hash160(
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
)
elif claim.is_repost:
row['reposted_claim_hash'] = claim.repost.reference.claim_hash
else:
row['txo_type'] = TXO_TYPES['stream']
elif txo.is_support:
row['txo_type'] = TXO_TYPES['support']
elif txo.purchase is not None:
row['txo_type'] = TXO_TYPES['purchase']
row['claim_id'] = txo.purchased_claim_id
row['claim_hash'] = txo.purchased_claim_hash
if txo.script.is_claim_involved:
signable = txo.can_decode_signable
if signable and signable.is_signed:
row['channel_hash'] = signable.signing_channel_hash
row['signature'] = txo.get_encoded_signature()
row['signature_digest'] = txo.get_signature_digest(self.ledger)
row['claim_id'] = txo.claim_id
row['claim_hash'] = txo.claim_hash
try:
row['claim_name'] = txo.claim_name.replace('\x00', '')
except UnicodeDecodeError:
pass
return row
def claim_to_rows(
self, txo: Output, staked_support_amount: int, staked_support_count: int, reposted_count: int,
signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
) -> Tuple[dict, List]:
tx = txo.tx_ref
d = {
'claim_type': None,
'address': txo.get_address(self.ledger),
'txo_hash': txo.hash,
'amount': txo.amount,
'height': tx.height,
'timestamp': tx.timestamp,
# support
'staked_amount': txo.amount + staked_support_amount,
'staked_support_amount': staked_support_amount,
'staked_support_count': staked_support_count,
# basic metadata
'title': None,
'description': None,
'author': None,
# streams
'stream_type': None,
'media_type': None,
'duration': None,
'release_time': None,
'fee_amount': 0,
'fee_currency': None,
# reposts
'reposted_claim_hash': None,
'reposted_count': reposted_count,
# signed claims
'channel_hash': None,
'is_signature_valid': None,
# channels (on last change) and streams (on last re-validation)
'public_key_hash': None,
'public_key_height': None,
}
claim = txo.can_decode_claim
if not claim:
return d, []
if claim.is_stream:
d['claim_type'] = TXO_TYPES['stream']
d['media_type'] = claim.stream.source.media_type
d['stream_type'] = STREAM_TYPES[guess_stream_type(d['media_type'])]
d['title'] = claim.stream.title.replace('\x00', '')
d['description'] = claim.stream.description.replace('\x00', '')
d['author'] = claim.stream.author.replace('\x00', '')
if claim.stream.video and claim.stream.video.duration:
d['duration'] = claim.stream.video.duration
if claim.stream.audio and claim.stream.audio.duration:
d['duration'] = claim.stream.audio.duration
if claim.stream.release_time:
d['release_time'] = claim.stream.release_time
if claim.stream.has_fee:
fee = claim.stream.fee
if isinstance(fee.amount, Decimal):
d['fee_amount'] = int(fee.amount*1000)
if isinstance(fee.currency, str):
d['fee_currency'] = fee.currency.lower()
elif claim.is_repost:
d['claim_type'] = TXO_TYPES['repost']
d['reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
d['claim_type'] = TXO_TYPES['channel']
d['public_key_hash'] = self.ledger.address_to_hash160(
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
)
if claim.is_signed:
d['channel_hash'] = claim.signing_channel_hash
d['is_signature_valid'] = (
all((signature, signature_digest, channel_public_key)) and
Output.is_signature_valid(
signature, signature_digest, channel_public_key
)
)
if channel_public_key:
d['public_key_hash'] = self.ledger.address_to_hash160(
self.ledger.public_key_to_address(channel_public_key)
)
tags = []
if claim.message.tags:
claim_hash = txo.claim_hash
tags = [
{'claim_hash': claim_hash, 'tag': tag}
for tag in clean_tags(claim.message.tags)
]
return d, tags
def support_to_row(
self, txo: Output, channel_public_key: bytes = None,
signature: bytes = None, signature_digest: bytes = None
):
tx = txo.tx_ref
d = {
'txo_hash': txo.ref.hash,
'claim_hash': txo.claim_hash,
'address': txo.get_address(self.ledger),
'amount': txo.amount,
'height': tx.height,
'timestamp': tx.timestamp,
'emoji': None,
'channel_hash': None,
'is_signature_valid': None,
}
support = txo.can_decode_support
if support:
d['emoji'] = support.emoji
if support.is_signed:
d['channel_hash'] = support.signing_channel_hash
d['is_signature_valid'] = (
all((signature, signature_digest, channel_public_key)) and
Output.is_signature_valid(
signature, signature_digest, channel_public_key
)
)
return d
def add_block(self, block: Block):
self.blocks.append(self.block_to_row(block))
for tx in block.txs:
self.add_transaction(block.block_hash, tx)
return self
def add_block_filter(self, height: int, factor: int, address_filter: bytes):
self.block_filters.append({
'height': height,
'factor': factor,
'address_filter': address_filter
})
def add_transaction(self, block_hash: bytes, tx: Transaction):
self.txs.append(self.tx_to_row(block_hash, tx))
for txi in tx.inputs:
if txi.coinbase is None:
self.txis.append(self.txi_to_row(tx, txi))
for txo in tx.outputs:
self.txos.append(self.txo_to_row(tx, txo))
return self
def add_transaction_filter(self, tx_hash: bytes, height: int, address_filter: bytes):
self.tx_filters.append({
'tx_hash': tx_hash,
'height': height,
'address_filter': address_filter
})
def add_support(self, txo: Output, **extra):
self.supports.append(self.support_to_row(txo, **extra))
def add_claim(
self, txo: Output, short_url: str,
creation_height: int, activation_height: int, expiration_height: int,
takeover_height: int = None, **extra
):
try:
claim_name = txo.claim_name.replace('\x00', '')
normalized_name = txo.normalized_name
except UnicodeDecodeError:
claim_name = normalized_name = ''
d, tags = self.claim_to_rows(txo, **extra)
d['claim_hash'] = txo.claim_hash
d['claim_id'] = txo.claim_id
d['claim_name'] = claim_name
d['normalized'] = normalized_name
d['short_url'] = short_url
d['creation_height'] = creation_height
d['activation_height'] = activation_height
d['expiration_height'] = expiration_height
d['takeover_height'] = takeover_height
d['is_controlling'] = takeover_height is not None
if d['public_key_hash'] is not None:
d['public_key_height'] = d['height']
self.claims.append(d)
self.tags.extend(tags)
return self
def update_claim(self, txo: Output, public_key_height=None, **extra):
d, tags = self.claim_to_rows(txo, **extra)
d['pk'] = txo.claim_hash
d['_public_key_height'] = public_key_height or d['height']
d['_public_key_hash'] = d['public_key_hash']
self.update_claims.append(d)
self.delete_tags.append({'pk': txo.claim_hash})
self.tags.extend(tags)
return self
def get_queries(self):
return (
(Block.insert(), self.blocks),
(BlockFilter.insert(), self.block_filters),
(TX.insert(), self.txs),
(TXFilter.insert(), self.tx_filters),
(TXO.insert(), self.txos),
(TXI.insert(), self.txis),
(Claim.insert(), self.claims),
(Tag.delete().where(Tag.c.claim_hash == bindparam('pk')), self.delete_tags),
(Claim.update()
.values(public_key_height=case([
(bindparam('_public_key_hash').is_(None), None),
(Claim.c.public_key_hash.is_(None) |
(Claim.c.public_key_hash != bindparam('_public_key_hash')),
bindparam('_public_key_height')),
], else_=Claim.c.public_key_height))
.where(Claim.c.claim_hash == bindparam('pk')),
self.update_claims),
(Tag.insert(), self.tags),
(Support.insert(), self.supports),
)
def flush(self, return_row_count_for_table) -> int:
done = 0
for sql, rows in self.get_queries():
if not rows:
continue
if self.ctx.is_postgres and isinstance(sql, Insert):
self.ctx.pg_copy(sql.table, rows)
else:
self.ctx.execute(sql, rows)
if sql.table == return_row_count_for_table:
done += len(rows)
rows.clear()
return done
+103
View File
@@ -0,0 +1,103 @@
from sqlalchemy.future import select
from lbry.db.query_context import progress, Event
from lbry.db.tables import TX, TXI, TXO, Claim, Support
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
from .queries import (
BASE_SELECT_TXO_COLUMNS,
rows_to_txos, where_unspent_txos,
where_abandoned_supports,
where_abandoned_claims
)
SPENDS_UPDATE_EVENT = Event.add("client.sync.spends.update", "steps")
CLAIMS_INSERT_EVENT = Event.add("client.sync.claims.insert", "claims")
CLAIMS_UPDATE_EVENT = Event.add("client.sync.claims.update", "claims")
CLAIMS_DELETE_EVENT = Event.add("client.sync.claims.delete", "claims")
SUPPORT_INSERT_EVENT = Event.add("client.sync.supports.insert", "supports")
SUPPORT_UPDATE_EVENT = Event.add("client.sync.supports.update", "supports")
SUPPORT_DELETE_EVENT = Event.add("client.sync.supports.delete", "supports")
def process_all_things_after_sync():
with progress(SPENDS_UPDATE_EVENT) as p:
p.start(2)
update_spent_outputs(p.ctx)
p.step(1)
set_input_addresses(p.ctx)
p.step(2)
with progress(SUPPORT_DELETE_EVENT) as p:
p.start(1)
sql = Support.delete().where(where_abandoned_supports())
p.ctx.execute(sql)
with progress(SUPPORT_INSERT_EVENT) as p:
loader = p.ctx.get_bulk_loader()
sql = (
select(*BASE_SELECT_TXO_COLUMNS)
.where(where_unspent_txos(TXO_TYPES['support'], missing_in_supports_table=True))
.select_from(TXO.join(TX))
)
for support in rows_to_txos(p.ctx.fetchall(sql)):
loader.add_support(support)
loader.flush(Support)
with progress(CLAIMS_DELETE_EVENT) as p:
p.start(1)
sql = Claim.delete().where(where_abandoned_claims())
p.ctx.execute(sql)
with progress(CLAIMS_INSERT_EVENT) as p:
loader = p.ctx.get_bulk_loader()
sql = (
select(*BASE_SELECT_TXO_COLUMNS)
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_in_claims_table=True))
.select_from(TXO.join(TX))
)
for claim in rows_to_txos(p.ctx.fetchall(sql)):
loader.add_claim(claim, '', 0, 0, 0, 0, staked_support_amount=0, staked_support_count=0)
loader.flush(Claim)
with progress(CLAIMS_UPDATE_EVENT) as p:
loader = p.ctx.get_bulk_loader()
sql = (
select(*BASE_SELECT_TXO_COLUMNS)
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_or_stale_in_claims_table=True))
.select_from(TXO.join(TX))
)
for claim in rows_to_txos(p.ctx.fetchall(sql)):
loader.update_claim(claim)
loader.flush(Claim)
def set_input_addresses(ctx):
# Update TXIs to have the address of TXO they are spending.
if ctx.is_sqlite:
address_query = select(TXO.c.address).where(TXI.c.txo_hash == TXO.c.txo_hash)
set_addresses = (
TXI.update()
.values(address=address_query.scalar_subquery())
.where(TXI.c.address.is_(None))
)
else:
set_addresses = (
TXI.update()
.values({TXI.c.address: TXO.c.address})
.where((TXI.c.address.is_(None)) & (TXI.c.txo_hash == TXO.c.txo_hash))
)
ctx.execute(set_addresses)
def update_spent_outputs(ctx):
# Update spent TXOs setting spent_height
set_spent_height = (
TXO.update()
.values({
TXO.c.spent_height: (
select(TXI.c.height)
.where(TXI.c.txo_hash == TXO.c.txo_hash)
.scalar_subquery()
)
}).where(
(TXO.c.spent_height == 0) &
(TXO.c.txo_hash.in_(select(TXI.c.txo_hash).where(TXI.c.address.is_(None))))
)
)
ctx.execute(set_spent_height)
+331
View File
@@ -0,0 +1,331 @@
# pylint: skip-file
from sqlalchemy import (
MetaData, Table, Column, ForeignKey,
LargeBinary, Text, SmallInteger, Integer, BigInteger, Boolean,
)
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
SCHEMA_VERSION = '1.4'
metadata = MetaData()
Version = Table(
'version', metadata,
Column('version', Text, primary_key=True),
)
Wallet = Table(
'wallet', metadata,
Column('wallet_id', Text, primary_key=True),
Column('data', Text),
)
PubkeyAddress = Table(
'pubkey_address', metadata,
Column('address', Text, primary_key=True),
Column('used_times', Integer, server_default='0'),
)
AccountAddress = Table(
'account_address', metadata,
Column('account', Text, primary_key=True),
Column('address', Text, ForeignKey(PubkeyAddress.columns.address), primary_key=True),
Column('chain', SmallInteger),
Column('pubkey', LargeBinary),
Column('chain_code', LargeBinary),
Column('n', Integer),
Column('depth', SmallInteger),
)
pg_add_account_address_constraints_and_indexes = [
"CREATE UNIQUE INDEX account_address_idx ON account_address (account, address);"
]
Block = Table(
'block', metadata,
Column('height', Integer, primary_key=True),
Column('block_hash', LargeBinary),
Column('previous_hash', LargeBinary),
Column('file_number', SmallInteger),
Column('timestamp', Integer),
)
pg_add_block_constraints_and_indexes = [
"ALTER TABLE block ADD PRIMARY KEY (height);",
]
BlockFilter = Table(
'block_filter', metadata,
Column('height', Integer),
Column('factor', SmallInteger),
Column('address_filter', LargeBinary),
)
pg_add_block_filter_constraints_and_indexes = [
"ALTER TABLE block_filter ADD PRIMARY KEY (height, factor);",
]
TX = Table(
'tx', metadata,
Column('tx_hash', LargeBinary, primary_key=True),
Column('raw', LargeBinary),
Column('height', Integer),
Column('position', SmallInteger),
Column('timestamp', Integer, nullable=True),
Column('day', Integer, nullable=True),
Column('is_verified', Boolean, server_default='FALSE'),
Column('purchased_claim_hash', LargeBinary, nullable=True),
)
pg_add_tx_constraints_and_indexes = [
"ALTER TABLE tx ADD PRIMARY KEY (tx_hash);",
]
TXFilter = Table(
'tx_filter', metadata,
Column('tx_hash', LargeBinary, primary_key=True),
Column('height', Integer),
Column('address_filter', LargeBinary),
)
pg_add_tx_filter_constraints_and_indexes = [
"ALTER TABLE tx_filter ADD PRIMARY KEY (tx_hash);",
"ALTER TABLE tx_filter ADD CONSTRAINT fk_tx_filter"
" FOREIGN KEY (tx_hash) REFERENCES tx (tx_hash) ON DELETE CASCADE;"
]
TXO = Table(
'txo', metadata,
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
Column('txo_hash', LargeBinary, primary_key=True),
Column('address', Text),
Column('position', SmallInteger),
Column('amount', BigInteger),
Column('height', Integer),
Column('spent_height', Integer, server_default='0'),
Column('script_offset', Integer),
Column('script_length', Integer),
Column('is_reserved', Boolean, server_default='0'),
# claims
Column('txo_type', SmallInteger, server_default='0'),
Column('claim_id', Text, nullable=True),
Column('claim_hash', LargeBinary, nullable=True),
Column('claim_name', Text, nullable=True),
Column('channel_hash', LargeBinary, nullable=True), # claims in channel
Column('signature', LargeBinary, nullable=True),
Column('signature_digest', LargeBinary, nullable=True),
# reposts
Column('reposted_claim_hash', LargeBinary, nullable=True),
# channels
Column('public_key', LargeBinary, nullable=True),
Column('public_key_hash', LargeBinary, nullable=True),
)
txo_join_account = TXO.join(AccountAddress, TXO.columns.address == AccountAddress.columns.address)
pg_add_txo_constraints_and_indexes = [
"ALTER TABLE txo ADD PRIMARY KEY (txo_hash);",
# find appropriate channel public key for signing a content claim
f"CREATE INDEX txo_channel_hash_by_height_desc_w_pub_key "
f"ON txo (claim_hash, height desc) INCLUDE (public_key) "
f"WHERE txo_type={TXO_TYPES['channel']};",
# for calculating supports on a claim
f"CREATE INDEX txo_unspent_supports ON txo (claim_hash) INCLUDE (amount) "
f"WHERE spent_height = 0 AND txo_type={TXO_TYPES['support']};",
# for calculating balance
f"CREATE INDEX txo_unspent_by_address ON txo (address) INCLUDE (amount, txo_type, tx_hash) "
f"WHERE spent_height = 0;",
# for finding modified claims in a block range
f"CREATE INDEX txo_claim_changes "
f"ON txo (height DESC) INCLUDE (claim_hash, txo_hash) "
f"WHERE spent_height = 0 AND txo_type IN {tuple(CLAIM_TYPE_CODES)};",
# for finding claims which need support totals re-calculated in a block range
f"CREATE INDEX txo_added_supports_by_height ON txo (height DESC) "
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
f"CREATE INDEX txo_spent_supports_by_height ON txo (spent_height DESC) "
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
# for finding claims which need repost totals re-calculated in a block range
f"CREATE INDEX txo_added_reposts_by_height ON txo (height DESC) "
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
f"CREATE INDEX txo_spent_reposts_by_height ON txo (spent_height DESC) "
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
"CREATE INDEX txo_reposted_claim_hash ON txo (reposted_claim_hash)"
"WHERE reposted_claim_hash IS NOT NULL AND spent_height = 0;",
"CREATE INDEX txo_height ON txo (height);",
# used by sum_supports query (at least)
"CREATE INDEX txo_claim_hash ON txo (claim_hash)",
]
TXI = Table(
'txi', metadata,
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
Column('address', Text, nullable=True),
Column('position', SmallInteger),
Column('height', Integer),
)
txi_join_account = TXI.join(AccountAddress, TXI.columns.address == AccountAddress.columns.address)
pg_add_txi_constraints_and_indexes = [
"ALTER TABLE txi ADD PRIMARY KEY (txo_hash);",
"CREATE INDEX txi_height ON txi (height);",
"CREATE INDEX txi_first_address ON txi (tx_hash) INCLUDE (address) WHERE position = 0;",
]
Claim = Table(
'claim', metadata,
Column('claim_hash', LargeBinary, primary_key=True),
Column('claim_id', Text),
Column('claim_name', Text),
Column('normalized', Text),
Column('address', Text),
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash)),
Column('amount', BigInteger),
Column('staked_amount', BigInteger),
Column('timestamp', Integer), # last updated timestamp
Column('creation_timestamp', Integer),
Column('release_time', BigInteger, nullable=True),
Column('height', Integer), # last updated height
Column('creation_height', Integer),
Column('activation_height', Integer),
Column('expiration_height', Integer),
Column('takeover_height', Integer, nullable=True),
Column('is_controlling', Boolean),
# short_url: normalized#shortest-unique-claim_id
Column('short_url', Text),
# canonical_url: channel's-short_url/normalized#shortest-unique-claim_id-within-channel
# canonical_url is computed dynamically
Column('title', Text, nullable=True),
Column('author', Text, nullable=True),
Column('description', Text, nullable=True),
Column('claim_type', SmallInteger),
Column('staked_support_count', Integer, server_default='0'),
Column('staked_support_amount', BigInteger, server_default='0'),
# streams
Column('stream_type', SmallInteger, nullable=True),
Column('media_type', Text, nullable=True),
Column('fee_amount', BigInteger, server_default='0'),
Column('fee_currency', Text, nullable=True),
Column('duration', Integer, nullable=True),
# reposts
Column('reposted_claim_hash', LargeBinary, nullable=True), # on claim doing the repost
Column('reposted_count', Integer, server_default='0'), # on claim being reposted
# claims which are channels
Column('signed_claim_count', Integer, server_default='0'),
Column('signed_support_count', Integer, server_default='0'),
Column('public_key_hash', LargeBinary, nullable=True), # included for claims in channel as well
Column('public_key_height', Integer, nullable=True), # last updated height
# claims which are inside channels
Column('channel_hash', LargeBinary, nullable=True),
Column('is_signature_valid', Boolean, nullable=True),
)
Tag = Table(
'tag', metadata,
Column('claim_hash', LargeBinary),
Column('tag', Text),
)
pg_add_claim_and_tag_constraints_and_indexes = [
"ALTER TABLE claim ADD PRIMARY KEY (claim_hash);",
# for checking if claim is up-to-date
"CREATE UNIQUE INDEX claim_txo_hash ON claim (txo_hash);",
# used by takeover process to reset winning claims
"CREATE INDEX claim_normalized ON claim (normalized);",
# ordering and search by release_time
"CREATE INDEX claim_release_time ON claim (release_time DESC NULLs LAST);",
# used to count()/sum() claims signed by channel
"CREATE INDEX signed_content ON claim (channel_hash) "
"INCLUDE (amount) WHERE is_signature_valid;",
# used to count()/sum() reposted claims
"CREATE INDEX reposted_content ON claim (reposted_claim_hash);",
# basic tag indexes
"ALTER TABLE tag ADD PRIMARY KEY (claim_hash, tag);",
"CREATE INDEX tags ON tag (tag) INCLUDE (claim_hash);",
# used by sum_supports query (at least)
"CREATE INDEX claim_channel_hash ON claim (channel_hash)",
]
Support = Table(
'support', metadata,
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
Column('claim_hash', LargeBinary),
Column('address', Text),
Column('amount', BigInteger),
Column('height', Integer),
Column('timestamp', Integer),
# support metadata
Column('emoji', Text),
# signed supports
Column('channel_hash', LargeBinary, nullable=True),
Column('signature', LargeBinary, nullable=True),
Column('signature_digest', LargeBinary, nullable=True),
Column('is_signature_valid', Boolean, nullable=True),
)
pg_add_support_constraints_and_indexes = [
"ALTER TABLE support ADD PRIMARY KEY (txo_hash);",
# used to count()/sum() supports signed by channel
"CREATE INDEX signed_support ON support (channel_hash) "
"INCLUDE (amount) WHERE is_signature_valid;",
]
Stake = Table(
'stake', metadata,
Column('claim_hash', LargeBinary),
Column('height', Integer),
Column('stake_min', BigInteger),
Column('stake_max', BigInteger),
Column('stake_sum', BigInteger),
Column('stake_count', Integer),
Column('stake_unique', Integer),
)
Trend = Table(
'trend', metadata,
Column('claim_hash', LargeBinary, primary_key=True),
Column('trend_group', BigInteger, server_default='0'),
Column('trend_mixed', BigInteger, server_default='0'),
Column('trend_local', BigInteger, server_default='0'),
Column('trend_global', BigInteger, server_default='0'),
)
CensoredClaim = Table(
'censored_claim', metadata,
Column('claim_hash', LargeBinary, primary_key=True),
Column('censor_type', SmallInteger),
Column('censoring_channel_hash', LargeBinary),
)
+25
View File
@@ -0,0 +1,25 @@
from sqlalchemy import select
from sqlalchemy.sql import func
from lbry.db.query_context import event_emitter, ProgressContext
from lbry.db.tables import Trend, Support, Claim
WINDOW = 576 # a day
@event_emitter("blockchain.sync.trending.update", "steps")
def calculate_trending(height, p: ProgressContext):
with p.ctx.engine.begin() as ctx:
ctx.execute(Trend.delete())
start = height - WINDOW
trending = func.sum(Support.c.amount * (WINDOW - (height - Support.c.height)))
sql = (
select([Claim.c.claim_hash, trending, trending, trending, 4])
.where(
(Support.c.claim_hash == Claim.c.claim_hash) &
(Support.c.height <= height) &
(Support.c.height >= start)
).group_by(Claim.c.claim_hash)
)
ctx.execute(Trend.insert().from_select(
['claim_hash', 'trend_global', 'trend_local', 'trend_mixed', 'trend_group'], sql
))
+174
View File
@@ -0,0 +1,174 @@
from itertools import islice
from typing import List, Union
from sqlalchemy import text, and_, or_
from sqlalchemy.sql.expression import Select, FunctionElement
from sqlalchemy.types import Numeric
from sqlalchemy.ext.compiler import compiles
try:
from sqlalchemy.dialects.postgresql import insert as pg_insert # pylint: disable=unused-import
except ImportError:
pg_insert = None
from .tables import AccountAddress
class greatest(FunctionElement): # pylint: disable=invalid-name
type = Numeric()
name = 'greatest'
@compiles(greatest)
def default_greatest(element, compiler, **kw):
return "greatest(%s)" % compiler.process(element.clauses, **kw)
@compiles(greatest, 'sqlite')
def sqlite_greatest(element, compiler, **kw):
return "max(%s)" % compiler.process(element.clauses, **kw)
class least(FunctionElement): # pylint: disable=invalid-name
type = Numeric()
name = 'least'
@compiles(least)
def default_least(element, compiler, **kw):
return "least(%s)" % compiler.process(element.clauses, **kw)
@compiles(least, 'sqlite')
def sqlite_least(element, compiler, **kw):
return "min(%s)" % compiler.process(element.clauses, **kw)
def chunk(rows, step):
it, total = iter(rows), len(rows)
for _ in range(0, total, step):
yield list(islice(it, step))
total -= step
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
if value is not None:
if isinstance(value, list):
value = [convert(v) for v in value]
if len(value) == 1:
constraints[column] = value[0]
elif len(value) > 1:
constraints[f"{column}__in"] = value
else:
constraints[column] = convert(value)
return constraints
def in_account_ids(account_ids: Union[List[str], str]):
if isinstance(account_ids, list):
if len(account_ids) > 1:
return AccountAddress.c.account.in_(account_ids)
account_ids = account_ids[0]
return AccountAddress.c.account == account_ids
def query(table, s: Select, **constraints) -> Select:
limit = constraints.pop('limit', None)
if limit is not None:
s = s.limit(limit)
offset = constraints.pop('offset', None)
if offset is not None:
s = s.offset(offset)
order_by = constraints.pop('order_by', None)
if order_by:
if isinstance(order_by, str):
s = s.order_by(text(order_by))
elif isinstance(order_by, list):
s = s.order_by(text(', '.join(order_by)))
else:
raise ValueError("order_by must be string or list")
group_by = constraints.pop('group_by', None)
if group_by is not None:
s = s.group_by(text(group_by))
account_ids = constraints.pop('account_ids', [])
if account_ids:
s = s.where(in_account_ids(account_ids))
if constraints:
s = s.where(and_(*constraints_to_clause(table, constraints)))
return s
def constraints_to_clause(tables, constraints):
clause = []
for key, constraint in constraints.items():
if key.endswith('__not'):
col, op = key[:-len('__not')], '__ne__'
elif key.endswith('__is_null'):
col = key[:-len('__is_null')]
op = '__eq__'
constraint = None
elif key.endswith('__is_not_null'):
col = key[:-len('__is_not_null')]
op = '__ne__'
constraint = None
elif key.endswith('__lt'):
col, op = key[:-len('__lt')], '__lt__'
elif key.endswith('__lte'):
col, op = key[:-len('__lte')], '__le__'
elif key.endswith('__gt'):
col, op = key[:-len('__gt')], '__gt__'
elif key.endswith('__gte'):
col, op = key[:-len('__gte')], '__ge__'
elif key.endswith('__like'):
col, op = key[:-len('__like')], 'like'
elif key.endswith('__not_like'):
col, op = key[:-len('__not_like')], 'notlike'
elif key.endswith('__in') or key.endswith('__not_in'):
if key.endswith('__in'):
col, op, one_val_op = key[:-len('__in')], 'in_', '__eq__'
else:
col, op, one_val_op = key[:-len('__not_in')], 'notin_', '__ne__'
if isinstance(constraint, Select):
pass
elif constraint:
if isinstance(constraint, (list, set, tuple)):
if len(constraint) == 1:
op = one_val_op
constraint = next(iter(constraint))
elif isinstance(constraint, str):
constraint = text(constraint)
else:
raise ValueError(f"{col} requires a list, set or string as constraint value.")
else:
continue
elif key.endswith('__or'):
clause.append(or_(*constraints_to_clause(tables, constraint)))
continue
else:
col, op = key, '__eq__'
attr = None
if '.' in col:
table_name, col = col.split('.')
_table = None
for table in tables:
if table.name == table_name.lower():
_table = table
break
if _table is not None:
attr = getattr(_table.c, col)
else:
raise ValueError(f"Table '{table_name}' not available: {', '.join([t.name for t in tables])}.")
else:
for table in tables:
attr = getattr(table.c, col, None)
if attr is not None:
break
if attr is None:
raise ValueError(f"Attribute '{col}' not found on tables: {', '.join([t.name for t in tables])}.")
clause.append(getattr(attr, op)(constraint))
return clause
File renamed without changes.
@@ -3,11 +3,15 @@ import typing
import logging
if typing.TYPE_CHECKING:
from lbry.dht.node import Node
from lbry.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class SQLiteStorage:
pass
class BlobAnnouncer:
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
self.loop = loop
@@ -24,7 +28,7 @@ class BlobAnnouncer:
else:
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
@@ -38,7 +42,7 @@ class BlobAnnouncer:
continue
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
while len(self.announce_queue):
while len(self.announce_queue) > 0:
log.info("%i blobs to announce", len(self.announce_queue))
announced = await asyncio.gather(*[
self._submit_announcement(
+41
View File
@@ -0,0 +1,41 @@
import hashlib
import os
HASH_CLASS = hashlib.sha384 # pylint: disable=invalid-name
HASH_LENGTH = HASH_CLASS().digest_size
HASH_BITS = HASH_LENGTH * 8
ALPHA = 5
K = 8
SPLIT_BUCKETS_UNDER_INDEX = 1
REPLACEMENT_CACHE_SIZE = 8
RPC_TIMEOUT = 5.0
RPC_ATTEMPTS = 5
RPC_ATTEMPTS_PRUNING_WINDOW = 600
ITERATIVE_LOOKUP_DELAY = RPC_TIMEOUT / 2.0 # TODO: use config val / 2 if rpc timeout is provided
REFRESH_INTERVAL = 3600 # 1 hour
REPLICATE_INTERVAL = REFRESH_INTERVAL
DATA_EXPIRATION = 86400 # 24 hours
TOKEN_SECRET_REFRESH_INTERVAL = 300 # 5 minutes
MAYBE_PING_DELAY = 300 # 5 minutes
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
RPC_ID_LENGTH = 20
PROTOCOL_VERSION = 1
BOTTOM_OUT_LIMIT = 3
MSG_SIZE_LIMIT = 1400
def digest(data: bytes) -> bytes:
h = HASH_CLASS()
h.update(data)
return h.digest()
def generate_id(num=None) -> bytes:
if num is not None:
return digest(str(num).encode())
else:
return digest(os.urandom(32))
def generate_rpc_id(num=None) -> bytes:
return generate_id(num)[:RPC_ID_LENGTH]
File renamed without changes.
+83 -70
View File
@@ -2,6 +2,7 @@ import logging
import asyncio
import typing
import binascii
import socket
from lbry.utils import resolve_host
from lbry.dht import constants
from lbry.dht.peer import make_kademlia_peer
@@ -18,8 +19,9 @@ log = logging.getLogger(__name__)
class Node:
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
storage: typing.Optional['SQLiteStorage'] = None):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
@@ -28,6 +30,7 @@ class Node:
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
self._storage = storage
async def refresh_node(self, force_once=False):
while True:
@@ -59,7 +62,7 @@ class Node:
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
await fut
continue
@@ -67,16 +70,18 @@ class Node:
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
if self._storage:
await self._storage.save_kademlia_peers(self.protocol.routing_table.get_peers())
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
await fut
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
assert len(hash_value) == constants.HASH_LENGTH
peers = await self.peer_search(hash_value)
if not self.protocol.external_ip:
@@ -89,8 +94,10 @@ class Node:
)
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
if stored_to:
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(stored_to), len(peers))
log.debug(
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(stored_to), len(peers)
)
else:
log.debug("Failed announcing %s, stored to 0 peers", blob_hash[:8])
return stored_to
@@ -111,7 +118,7 @@ class Node:
self.listening_port = None
log.info("Stopped DHT node")
async def start_listening(self, interface: str = '') -> None:
async def start_listening(self, interface: str = '0.0.0.0') -> None:
if not self.listening_port:
self.listening_port, _ = await self.loop.create_datagram_endpoint(
lambda: self.protocol, (interface, self.internal_udp_port)
@@ -121,60 +128,57 @@ class Node:
else:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: typing.Optional[str] = '',
async def join_network(self, interface: str = '0.0.0.0',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
peer_addresses = []
for node_id, address, udp_port, tcp_port in urls:
if (node_id, address, udp_port, tcp_port) not in peer_addresses and \
(address, udp_port) != (self.protocol.external_ip, self.protocol.udp_port):
peer_addresses.append((node_id, address, udp_port, tcp_port))
return [make_kademlia_peer(*peer_address) for peer_address in peer_addresses]
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and\
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
known_node_addresses.append((address, port))
url_to_addr[address] = host
if known_node_addresses:
peers = [
make_kademlia_peer(None, address, port)
for (address, port) in known_node_addresses
]
while True:
if not self.protocol.routing_table.get_peers():
if self.joined.is_set():
self.joined.clear()
self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
if self.protocol.routing_table.get_peers():
self.joined.set()
log.info(
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
else:
while True:
if self.protocol.routing_table.get_peers():
if not self.joined.is_set():
self.joined.set()
log.info(
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts()
)
else:
if self.joined.is_set():
self.joined.clear()
seed_peers = peers_from_urls(
await self._storage.get_persisted_kademlia_peers()
) if self._storage else []
if not seed_peers:
try:
seed_peers.extend(peers_from_urls([
(None, await resolve_host(address, udp_port, 'udp'), udp_port, None)
for address, udp_port in known_node_urls or []
]))
except socket.gaierror:
await asyncio.sleep(30, loop=self.loop)
continue
await asyncio.sleep(1, loop=self.loop)
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
self.join_network(
interface=interface, known_node_urls=known_node_urls
)
)
await asyncio.sleep(1, loop=self.loop)
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.bottom_out_limit,
max_results: int = constants.k) -> IterativeNodeFinder:
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
max_results: int = constants.K) -> IterativeNodeFinder:
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
@@ -186,7 +190,7 @@ class Node:
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
) -> typing.List['KademliaPeer']:
peers = []
@@ -197,25 +201,28 @@ class Node:
peers.sort(key=lambda peer: distance(peer.node_id))
return peers[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
async def _accumulate_peers_for_value(self, search_queue: asyncio.Queue, result_queue: asyncio.Queue):
tasks = []
try:
while True:
blob_hash = await search_queue.get()
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
tasks.append(self.loop.create_task(self._peers_for_value_producer(blob_hash, result_queue)))
finally:
for task in tasks:
task.cancel()
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
async def ping(_peer):
async def _peers_for_value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
async def put_into_result_queue_after_pong(_peer):
try:
await self.protocol.get_rpc_peer(_peer).ping()
result_queue.put_nowait([_peer])
log.debug("pong from %s:%i for %s", _peer.address, _peer.udp_port, blob_hash)
except asyncio.TimeoutError:
pass
# prioritize peers who reply to a dht ping first
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
to_put = []
for peer in results:
@@ -223,25 +230,31 @@ class Node:
continue
is_good = self.protocol.peer_manager.peer_is_good(peer)
if is_good:
# the peer has replied recently over UDP, it can probably be reached on the TCP port
to_put.append(peer)
elif is_good is None:
udp_port_to_try = 4444
if 3400 > peer.tcp_port > 3332:
if not peer.udp_port:
udp_port_to_try = (peer.tcp_port - 3333) + 4444
elif 4500 > peer.tcp_port > 4443:
if not peer.udp_port:
udp_port_to_try = peer.tcp_port
if not peer.udp_port:
peer = make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
self.loop.create_task(ping(peer))
# TODO: use the same port for TCP and UDP
# the udp port must be guessed
# default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
# including on a network with several nodes, then assume the udp port is proportionately
# based on a starting port of 4444
udp_port_to_try = peer.tcp_port
if 3400 > peer.tcp_port > 3332:
udp_port_to_try = (peer.tcp_port - 3333) + 4444
self.loop.create_task(put_into_result_queue_after_pong(
make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
))
else:
self.loop.create_task(put_into_result_queue_after_pong(peer))
else:
# the peer is known to be bad/unreachable, skip trying to connect to it over TCP
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
if to_put:
result_queue.put_nowait(to_put)
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue(loop=self.loop)
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
peer_queue: typing.Optional[asyncio.Queue] = None
) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
queue = peer_queue or asyncio.Queue(loop=self.loop)
return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
+30 -14
View File
@@ -15,14 +15,29 @@ log = logging.getLogger(__name__)
@lru_cache(1024)
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
udp_port: typing.Optional[int] = None,
tcp_port: typing.Optional[int] = None) -> 'KademliaPeer':
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port)
tcp_port: typing.Optional[int] = None,
allow_localhost: bool = False) -> 'KademliaPeer':
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
def is_valid_ipv4(address):
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
ALLOW_LOCALHOST = False
def is_valid_public_ipv4(address, allow_localhost: bool = False):
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
try:
ip = ipaddress.ip_address(address)
return ip.version == 4
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
parsed_ip.is_reserved,
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except ipaddress.AddressValueError:
return False
@@ -70,7 +85,7 @@ class PeerManager:
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
ts, token = self._node_tokens.get(node_id, (0, None))
if ts and ts > self._loop.time() - constants.token_secret_refresh_interval:
if ts and ts > self._loop.time() - constants.TOKEN_SECRET_REFRESH_INTERVAL:
return token
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
@@ -93,23 +108,23 @@ class PeerManager:
now = self._loop.time()
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
if last_failure and last_failure < now - constants.RPC_ATTEMPTS_PRUNING_WINDOW:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
for node_id, (age, token) in self._node_tokens.items(): # pylint: disable=unused-variable
if age < now - constants.TOKEN_SECRET_REFRESH_INTERVAL:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int): # pylint: disable=too-many-return-statements
"""
:return: False if peer is bad, None if peer is unknown, or True if peer is good
"""
delay = self._loop.time() - constants.check_refresh_interval
delay = self._loop.time() - constants.CHECK_REFRESH_INTERVAL
# fixme: find a way to re-enable that without breaking other parts
# if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
@@ -139,7 +154,7 @@ class PeerManager:
def peer_is_good(self, peer: 'KademliaPeer'):
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
node_id, address, tcp_port = decode_compact_address(compact_address)
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
@@ -151,16 +166,17 @@ class KademliaPeer:
udp_port: typing.Optional[int] = field(hash=True)
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
allow_localhost: bool = field(default=False, compare=False, hash=False)
def __post_init__(self):
if self._node_id is not None:
if not len(self._node_id) == constants.hash_length:
if not len(self._node_id) == constants.HASH_LENGTH:
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
raise ValueError("invalid udp port")
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
raise ValueError("invalid tcp port")
if not is_valid_ipv4(self.address):
if not is_valid_public_ipv4(self.address, self.allow_localhost):
raise ValueError(f"invalid ip address: '{self.address}'")
def update_tcp_port(self, tcp_port: int):
File renamed without changes.
Loaded 100 of 480 files, more files were not shown because too many files have changed in this diff. Show more