Compare commits

..
Author SHA1 Message Date
Victor Shyba 92258a2764 add concurrent_hub_requests conf 2021-05-04 22:38:59 -03:00
Victor Shyba ac88ada50e move request semaphore to session and apply to all requests 2021-05-04 22:38:46 -03:00
Victor Shyba e7fdf2aeb3 add hub_timeout and propagate it to network code 2021-05-04 22:38:39 -03:00
Lex Berezhny 99f56f5d22 v0.96.0 2021-04-28 15:26:58 -04:00
Jack Robison ad6281090d Merge pull request #3275 from lbryio/search_caching_issues
add caching to "search ahead" code and invalidate short_url cache on every block
2021-04-28 14:14:29 -04:00
Victor Shyba f0d334d3e2 refactor from review 2021-04-28 13:28:38 -03:00
Victor Shyba 5f829b048f use separator to avoid cache key conflicts 2021-04-27 22:57:04 -03:00
Victor Shyba 1a961e66ff invalidate short_id cache on new block 2021-04-27 22:57:04 -03:00
Victor Shyba fdb0e22656 cache search_ahead 2021-04-27 22:57:04 -03:00
Jack Robison 132ee1915f Merge pull request #3283 from lbryio/fix_multiprocessing_db
fix multiprocessing support on client db
2021-04-27 16:46:28 -04:00
Victor Shyba 44bf4f3c8f fix if statement from always evaluating a string 2021-04-27 17:10:04 -03:00
Alex Grintsvayg 6237767d5a Merge branch 'make-test'
* make-test:
  run tests using make
2021-04-23 15:26:26 -04:00
Alex Grintsvayg dec9d96417 run tests using make 2021-04-23 15:25:40 -04:00
Lex Berezhny b167c87267 v0.95.0 2021-04-23 14:55:38 -04:00
Lex Berezhny 2280fe8e8e default has_source to 1 2021-04-23 14:54:51 -04:00
Lex Berezhny 575d6dcd2d migration specifically for upgrading from client db v1.5 to v1.6 2021-04-23 14:54:51 -04:00
Lex Berezhny f729490c6b pending claims ordered towards top in claim_list 2021-04-23 11:00:58 -04:00
Lex Berezhny b32124cdd6 regenerate docs 2021-04-23 10:24:48 -04:00
Lex Berezhny 3d4321ee38 added --has_source/--has_no_source filters to claim_list 2021-04-23 10:24:48 -04:00
Alex Grintsvayg 85034b382e Revert "run tests using make"
This reverts commit 77a51d1ad4.
2021-04-21 11:41:16 -04:00
Alex Grintsvayg 77a51d1ad4 run tests using make 2021-04-20 15:41:09 -04:00
Alex Grin 33e0cdc2d7 Update docker-compose-wallet-server.yml 2021-04-16 11:58:02 -04:00
Alex Grin 6519faa2fe Update docker-compose-wallet-server.yml 2021-04-16 11:53:16 -04:00
Lex Berezhny 5e3a234cbe v0.94.1 2021-04-16 11:18:24 -04:00
Lex Berezhny e54c31d2d5 fix bug in how reserved balance is calculated 2021-04-16 11:17:51 -04:00
Alex Grin 66c0537251 Create ossar-analysis.yml 2021-04-15 15:24:26 -04:00
Alex Grin ac58516593 Create codeql-analysis.yml 2021-04-15 15:22:57 -04:00
Alex Grin c3da6322b5 Create SECURITY.md 2021-04-15 15:21:17 -04:00
Lex Berezhny 3d241500cf v0.94.0 2021-04-14 19:55:35 -04:00
Lex Berezhny ded8224f66 update docs 2021-04-14 19:52:50 -04:00
Lex Berezhny f8814881a1 ability to set sd_hash, file_name and file_hash when updating a stream claim 2021-04-14 19:52:50 -04:00
Victor Shyba cc2852cd48 new implementation for limit_claims_per_channel 2021-04-14 18:32:16 -04:00
Lex Berezhny 467637a9eb fix test 2021-04-14 11:24:58 -04:00
Lex Berezhny 3cfc292d84 lint 2021-04-14 11:24:58 -04:00
Lex Berezhny 6acf94a810 moved balance calculation to SQL 2021-04-14 11:24:58 -04:00
Jack Robison 31367fb4c4 show hostnames of spvs 2021-04-13 11:51:27 -04:00
Jack Robison 12d6074e3b fix typing 2021-04-13 11:51:27 -04:00
Lex Berezhny ff30386051 lint 2021-04-06 21:22:27 -04:00
shubhendra 601f99ac16 Remove unnecessary generator
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra 87fe5c6101 Refactor the comparison involving not
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra 68399ca31c Iterate dictionary directly
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra 2a6d7fd80f Remove methods with unnecessary super delegation.
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra 4725f510d8 Remove unnecessary use of comprehension
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra be0ba22222 Remove unnecessary comprehension
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
Lex Berezhny c8781392be added unit test for Access-Control HTTP headers 2021-04-06 17:12:05 -04:00
John Leith b97164fcfb adding access control headers 2021-04-06 17:12:05 -04:00
Lex Berezhny 0dfb92281b v0.93.0 2021-03-30 20:59:47 -04:00
Victor Shyba 4fe80c40da also apply to test:json-api 2021-03-30 17:00:15 -04:00
Victor Shyba f0fac5115a update tox to pass ELASTIC_HOST 2021-03-30 17:00:15 -04:00
Victor Shyba 46dd389d0d add elasticsearch service to gitlab 2021-03-30 17:00:15 -04:00
Jack Robison 1e28e21ab5 Merge pull request #3248 from lbryio/add-es-host-setting
add ELASTIC_HOST and ELASTIC_PORT settings to hub
2021-03-30 13:09:18 -04:00
Jack Robison 7832c62c5d add ELASTIC_HOST and ELASTIC_PORT settings to hub 2021-03-30 12:48:13 -04:00
Lex Berezhny d025ee9dbe revert release 2021-03-30 11:29:17 -04:00
Lex Berezhny a9a9cb4319 v0.93.0 2021-03-30 10:15:31 -04:00
Victor Shyba aa727cb9b1 show channels regardless of no_source 2021-03-30 09:47:08 -04:00
Victor Shyba b8c9a99f20 fix no_source for reposts 2021-03-30 09:47:08 -04:00
Lex Berezhny aff995b0d0 temporary fix for mempool sync failing during reorg 2021-03-29 16:11:03 -04:00
Jack Robison 2cc7e5dfdc Merge pull request #3153 from lbryio/elasticsearch
hub: use Elasticsearch for `claim_search` and `resolve` calls
2021-03-24 16:44:14 -04:00
Victor Shyba 5235a150b1 add prog name to sync arg parser 2021-03-24 17:07:17 -03:00
Victor Shyba c6372ea9de hub->lbry-hub 2021-03-24 17:03:57 -03:00
Victor Shyba 7df4cc44c4 fixes from review 2021-03-24 16:30:33 -03:00
Victor Shyba d47cf40544 add reader.py for test_sqldb tests 2021-03-19 19:58:13 -03:00
Victor Shyba 7f5d88e95c remove dead/broken/unused API 2021-03-19 19:58:13 -03:00
Victor Shyba d09663c066 remove flush call 2021-03-19 19:58:13 -03:00
Victor Shyba ef97c9b69f torba-server -> hub 2021-03-19 19:58:13 -03:00
Victor Shyba d855e6c8b1 move elasticsearch things into its own module 2021-03-19 19:58:13 -03:00
Victor Shyba cd66f7eb43 if not no_totals, use default page size 2021-03-19 19:58:13 -03:00
Victor Shyba 6a35a7ba4c expand content filtering tests for no_totals 2021-03-19 19:58:13 -03:00
Victor Shyba a3e146dc68 sort on index time 2021-03-19 19:58:13 -03:00
Victor Shyba b81305a4a9 index and allow has_source 2021-03-19 19:58:13 -03:00
Victor Shyba 73884b34bc apply no_totals 2021-03-19 19:58:13 -03:00
Victor Shyba 6166a34db2 check cache item before locking 2021-03-19 19:58:13 -03:00
Victor Shyba 6fa7da4b1c less slices 2021-03-19 19:58:13 -03:00
Victor Shyba c3e426c491 fix search by channel for invalid channel 2021-03-19 19:58:13 -03:00
Victor Shyba 21e023f0db fix search by channel 2021-03-19 19:58:13 -03:00
Victor Shyba 063be001b3 cache inner parsing 2021-03-19 19:58:13 -03:00
Victor Shyba 5dff02e8bc on resolve, get all claims at once 2021-03-19 19:58:13 -03:00
Victor Shyba 60a59407d8 cache the encoded output instead 2021-03-19 19:58:13 -03:00
Victor Shyba 20a5aecfca fix lib exception to asyncio TimeoutError 2021-03-19 19:58:13 -03:00
Victor Shyba c2e7b5a67d restore some of the interrupt metrics 2021-03-19 19:58:13 -03:00
Victor Shyba 8f32303d07 apply search timeout 2021-03-19 19:58:13 -03:00
Victor Shyba 891b1e7782 track results up to 200 2021-03-19 19:58:13 -03:00
Victor Shyba f26394fd3b report deletions on docs that doesnt exist, but dont raise 2021-03-19 19:58:13 -03:00
Victor Shyba 4d83d42b4c fix equality instead of mod 2021-03-19 19:58:13 -03:00
Victor Shyba 57f1108df2 fix query being json serializable 2021-03-19 19:58:13 -03:00
Victor Shyba 2641a9abe5 make better resolve cache 2021-03-19 19:58:13 -03:00
Victor Shyba 6b193ab350 make indexing cooperative 2021-03-19 19:58:13 -03:00
Victor Shyba b1bb37511c use right key on cache 2021-03-19 19:58:13 -03:00
Victor Shyba 319187d6d6 log mempool task exceptions 2021-03-19 19:58:13 -03:00
Victor Shyba 02eb789f84 caching for resolve 2021-03-19 19:58:13 -03:00
Victor Shyba 5a9338a27f use a dict on set_reference 2021-03-19 19:58:13 -03:00
Victor Shyba eb6924277f round time to 10 minutes and fetch referenced by id 2021-03-19 19:58:13 -03:00
Victor Shyba 325419404d update dockerfile 2021-03-19 19:58:13 -03:00
Victor Shyba bd8f371fd5 bump referenced rows query limit up 2021-03-19 19:58:13 -03:00
Victor Shyba 1783ff2845 dont delete claims on reorg 2021-03-19 19:58:13 -03:00
Victor Shyba d388527ffa log indexing errors 2021-03-19 19:58:13 -03:00
Victor Shyba 19494088bd generate from queue 2021-03-19 19:58:13 -03:00
Victor Shyba 920dad524a simplify sync and use asyncio Queue instead 2021-03-19 19:58:13 -03:00
Victor Shyba ec89bcac8e improve sync script for no-downtime maintenance 2021-03-19 19:58:13 -03:00
Victor Shyba a916c1f4ad check if db file exists before sync 2021-03-19 19:58:13 -03:00
Victor Shyba a9a0ac92d7 ignore unset flag 2021-03-19 19:58:13 -03:00
Victor Shyba da8a8bd1ef filter+fts and tests for edge cases 2021-03-19 19:58:13 -03:00
Victor Shyba d9c746891d pin python3.7 2021-03-19 19:58:13 -03:00
Victor Shyba 67817005b5 check ES synced without a process and wait for ES 2021-03-19 19:58:13 -03:00
Jack Robison 24d11de5a7 torba-elastic-sync 2021-03-19 19:58:13 -03:00
Victor Shyba 9251c87323 refresh after sync 2021-03-19 19:58:13 -03:00
Victor Shyba e12fab90d1 docker compose update 2021-03-19 19:58:13 -03:00
Victor Shyba 0a194b5b01 claim_ids query 2021-03-19 19:58:13 -03:00
Victor Shyba 8d028adc53 be a writer by default 2021-03-19 19:58:13 -03:00
Victor Shyba dfca15395e claim id is also a keyword 2021-03-19 19:58:13 -03:00
Victor Shyba e21f2362fe apply reorg deletion as well 2021-03-19 19:58:13 -03:00
Victor Shyba 1ce328e8a9 cache signature inspection 2021-03-19 19:58:13 -03:00
Victor Shyba 038a5f999f cache encoded headers 2021-03-19 19:58:13 -03:00
Victor Shyba 5d3704c7ea reader mode 2021-03-19 19:58:13 -03:00
Victor Shyba 87037c06c9 remove reader code 2021-03-19 19:58:13 -03:00
Victor Shyba dd412c0f50 delete sqlite fts 2021-03-19 19:58:13 -03:00
Victor Shyba bf44befff6 backport fixes from server 2021-03-19 19:58:13 -03:00
Victor Shyba e61874bb6f only repeat search if it has blocked items 2021-03-19 19:58:13 -03:00
Victor Shyba 1e5331768f fix some of the tests 2021-03-19 19:58:13 -03:00
Victor Shyba ec9a3a4f7c do not page filtered 2021-03-19 19:58:13 -03:00
Victor Shyba e439a3a8dc advanced resolve 2021-03-19 19:58:13 -03:00
Victor Shyba 19f70d7a11 create changelog trigger 2021-03-19 19:58:13 -03:00
Victor Shyba afe7ed5b05 adjust size 2021-03-19 19:58:13 -03:00
Victor Shyba d4bf004d74 use a thread pool to sync changes 2021-03-19 19:58:13 -03:00
Victor Shyba e4d06a088b include the channel being filtered/blocked 2021-03-19 19:58:13 -03:00
Victor Shyba 0929088b12 missing refresh step 2021-03-19 19:58:13 -03:00
Victor Shyba 7b4838fc9b dont update more than 400 items a time 2021-03-19 19:58:13 -03:00
Victor Shyba 0cf9533248 narrow update by query 2021-03-19 19:58:13 -03:00
Victor Shyba 84ff0b8a9f general timeout 2021-03-19 19:58:13 -03:00
Victor Shyba d467dcfeaf increase sync queue 2021-03-19 19:58:13 -03:00
Victor Shyba 8e68ba4751 fix join, refresh before update 2021-03-19 19:58:13 -03:00
Victor Shyba 0f2a85ba9f simplify sync 2021-03-19 19:58:13 -03:00
Victor Shyba 7674a0a91e backport fixes from testing server 2021-03-19 19:58:13 -03:00
Victor Shyba 5bc1a66572 32 slices and add censor type to fields 2021-03-19 19:58:13 -03:00
Victor Shyba 9b56067213 raise request timeout for content filtering 2021-03-19 19:58:13 -03:00
Victor Shyba 9a9df2fc3c apply filtering only to whats unfiltered 2021-03-19 19:58:13 -03:00
Victor Shyba 9989d8d1d4 refresh after delete 2021-03-19 19:58:13 -03:00
Victor Shyba f9471f297e apply filter and block from ES script lang 2021-03-19 19:58:13 -03:00
Victor Shyba 146b693e4a exclude title and description 2021-03-19 19:58:13 -03:00
Victor Shyba 7295b7e329 make sync parallel 2021-03-19 19:58:13 -03:00
Victor Shyba e2441ea3e7 use prefix from ES docs 2021-03-19 19:58:13 -03:00
Victor Shyba 119e51912e fix partial id 2021-03-19 19:58:13 -03:00
Victor Shyba dd950f5b0d tag can have empty space 2021-03-19 19:58:13 -03:00
Victor Shyba 78a9bad1e1 no indexer_task 2021-03-19 19:58:13 -03:00
Victor Shyba 0c6eaf5484 fix resolve partial id 2021-03-19 19:58:13 -03:00
Victor Shyba 1010068ddb disable refresh interval. start with 3 shards 2021-03-19 19:58:13 -03:00
Victor Shyba 82eec3d8d7 use multiple clients on sync script indexing 2021-03-19 19:58:13 -03:00
Victor Shyba ee7b37d3f3 also normalize the name supplied by user 2021-03-19 19:58:13 -03:00
Victor Shyba 143d82d242 normalized, not normalized_name 2021-03-19 19:58:13 -03:00
Victor Shyba 8b91b38855 update winners in one go 2021-03-19 19:58:13 -03:00
Victor Shyba 1098f0d2a3 use normalized name instead 2021-03-19 19:58:13 -03:00
Victor Shyba ab53cec022 fix is_controlling sync 2021-03-19 19:58:13 -03:00
Victor Shyba 6f5f8e5648 add elasticsearch dep 2021-03-19 19:58:13 -03:00
Victor Shyba edfd707c22 run ES on github actions 2021-03-19 19:58:13 -03:00
Victor Shyba 1870f30af8 add sync script 2021-03-19 19:58:13 -03:00
Victor Shyba 90106f5f08 all test_claim_commands tests green 2021-03-19 19:58:13 -03:00
Victor Shyba 9924b7b438 reposts and tag inheritance 2021-03-19 19:58:13 -03:00
Victor Shyba aa37faab0a use porter analyzer with weights on full text search 2021-03-19 19:58:13 -03:00
Victor Shyba dc10f8ce72 ignore errors when deleting 2021-03-19 19:58:13 -03:00
Victor Shyba 996686c1da claim search and resolve translated to ES queries 2021-03-19 19:58:13 -03:00
Victor Shyba 488785d013 add indexer task 2021-03-19 19:58:13 -03:00
Victor Shyba 3abdc01230 index ES during sync 2021-03-19 19:58:13 -03:00
Victor Shyba 8da04a584f start waiting before generate 2021-03-19 18:01:29 -03:00
Victor Shyba 27cc61d45e limit test time to 2 minutes, then consider it a failure and log what was running 2021-03-19 18:01:29 -03:00
Lex Berezhny 7371c30064 v0.92.0 2021-03-15 13:07:30 -04:00
Lex Berezhny 140d163895 removed redundant comment 2021-03-14 10:11:42 -04:00
Victor Shyba dc33bdc1dc update api json 2021-03-14 10:11:42 -04:00
Victor Shyba 74df4fab83 change column to has_source and document both flags 2021-03-14 10:11:42 -04:00
Victor Shyba 1e5cd3d7a1 typo, fix tests 2021-03-14 10:11:42 -04:00
Victor Shyba a54e9b64aa add no_source claim_search filter 2021-03-14 10:11:42 -04:00
Victor Shyba 74660704e3 fix update 2021-03-14 10:11:42 -04:00
Victor Shyba 7439893a2a fix get for sourceless claims 2021-03-14 10:11:42 -04:00
Victor Shyba e27e49e9dc call update only once 2021-03-14 10:11:42 -04:00
Victor Shyba 34ed729c59 there is no 'sd_hash' parameter for this API 2021-03-14 10:11:42 -04:00
Victor Shyba adaeeca3fd let file_path be optional 2021-03-14 10:11:42 -04:00
Jack Robison dac75563d3 add --no_file_path param to publish, stream_create, and stream_update 2021-03-14 10:11:42 -04:00
Alex Grintsvayg cbc76adcaa only return unspent txos if is_spent flag is not used. fixes #2923 2021-03-13 06:44:20 -05:00
Lex Berezhny 69a9cb383d oops 2021-03-12 13:29:55 -05:00
Lex Berezhny 4343073c00 clients can connect to wallet server even when they are not reachable by UDP 2021-03-12 13:29:55 -05:00
Jack Robison fe60d4be88 Merge pull request #3221 from lbryio/subscribe_hash_on_call
Improve performance of address subscriptions and transaction proofs
2021-03-10 15:58:50 -05:00
Victor Shyba ae337807f5 get merkles outside thread cooperatively 2021-03-10 13:05:17 -03:00
Victor Shyba 9ae30ac08e during subscribe, hash address only when its time 2021-03-10 12:51:58 -03:00
Lex Berezhny 62fa85c0a4 fix test 2021-03-09 13:27:36 -05:00
Lex Berezhny 7bb873dad9 removed connection_status field from the status command, use wallet.connected instead to determine if SDK is connected 2021-03-09 13:27:36 -05:00
Lex Berezhny 5f6c1c14cb v0.91.0 2021-03-04 00:04:25 -05:00
Lex Berezhny d43189ad33 regenerate docs 2021-03-04 00:03:16 -05:00
Lex Berezhny fcad76fc51 lint 2021-03-04 00:03:16 -05:00
Lex Berezhny 97e6e1684e simplifying 2021-03-04 00:03:16 -05:00
zeppi 67a0d3e926 update docs 2021-03-04 00:03:16 -05:00
zeppi 183fb9f9ff provide --resolve tag for collection claim, separate from resolving its contents
bugfix and docs generation

review changes
2021-03-04 00:03:16 -05:00
Lex Berezhny 9815ddef1f fixes stalling client reconnect issue 2021-03-03 23:31:59 -05:00
Lex Berezhny f6d0847453 v0.90.1 2021-03-01 11:57:40 -05:00
Lex Berezhny b0b9f0d65f regenerate docs 2021-03-01 11:52:17 -05:00
Lex Berezhny 0cec80f676 fixes transaction signing bug when tx had no change outputs 2021-03-01 11:20:08 -05:00
Patrick Keane 48c64143e3 Add no_totals argument to claim search 2021-02-26 10:41:15 -05:00
Patrick Keane a8712422bc Update daemon.py 2021-02-21 23:45:18 -05:00
Patrick Keane 97f65bd283 Minor spelling correction(s). 2021-02-21 23:45:18 -05:00
Lex Berezhny fd3c1c50f1 v0.90.0 2021-02-16 11:13:14 -05:00
Lex Berezhny b153e4bb9f added support to claim_search for filtering collections via --claim_type 2021-02-16 11:08:54 -05:00
Lex Berezhny db9856a8db use median exchange rate when several exchange rates are available 2021-02-15 14:09:32 -05:00
Lex Berezhny 75ecea265d ensures content purchase fails appropriately when exchange rate is not available 2021-02-11 13:57:33 -05:00
Lex Berezhny be8751cb73 integration test fixes 2021-02-10 22:14:13 -05:00
Lex Berezhny fb25ecb4a1 unittest 2021-02-10 22:14:13 -05:00
Lex Berezhny f1cb7d27ac lint 2021-02-10 22:14:13 -05:00
Lex Berezhny dee494e12f converting from USD, BTC to LBC is now done via several exchange rate providers: Bittrex, Cryptonator, CoinEx, hotbit and UPbit 2021-02-10 22:14:13 -05:00
Lex Berezhny b13a121915 v0.89.0 2021-02-09 10:12:05 -05:00
Lex Berezhny 7486ee9537 do not tx._reset() in generate_channel_private_key 2021-02-08 09:56:03 -05:00
Lex Berezhny 4a20ccc28e tx._reset() the transaction after more internal changes 2021-02-08 09:56:03 -05:00
Lex Berezhny f80dd2b307 tx._reset() the transaction after internal changes 2021-02-08 09:56:03 -05:00
Lex Berezhny b208cf6d32 significant performance improvement when creating transactions and with txo spend specifically 2021-02-08 09:56:03 -05:00
Lex Berezhny 39e78ff17e updated exchange rate manager to use v3 bittrex API (old one is deprecated) 2021-02-03 14:15:19 -05:00
Alex Grin a8177ea7fe Merge pull request #3171 from lbryio/fix-collectionChannel 2021-02-02 14:16:42 -05:00
Alex Grin bedcfc154b Merge branch 'master' into fix-collectionChannel 2021-02-02 11:25:52 -05:00
zeppi 4c38b4aa3c docs and reenable gitlab json generation 2021-02-01 19:19:46 -05:00
Victor Shyba f6cfe266e0 specify Returns correctly for docs gen 2021-02-01 19:19:46 -05:00
zeppi 4905e65f14 fix comment tests and json generat 2021-02-01 19:19:46 -05:00
Lex Berezhny ccb250b410 txo_list --resolve now works for signed supports by resolving the signing channel 2021-01-26 20:53:36 -05:00
zeppi aca57ffc62 allows cli to accept channel_id for collections 2021-01-26 10:02:01 -05:00
Dispatch 7f375f42d8 Fix / Add correct mimetypes for HLS / DASH 2021-01-25 15:55:56 -05:00
Lex Berezhny eedcc2034d fixes #3166 2021-01-25 10:56:48 -05:00
Jack Robison 24c9a167d7 dockerfile 2021-01-22 12:28:04 -05:00
Jack Robison 909df8ef1f dockerfile 2021-01-22 11:54:56 -05:00
Jack Robison 3b27cb3671 v0.88.0 2021-01-22 10:41:06 -05:00
Lex Berezhny 3fe0db4a7d fix integration tests 2021-01-22 07:51:13 -05:00
Lex Berezhny 8b55814ab2 fixes #2943 bug where get forced user to purchase their own priced stream 2021-01-22 07:51:13 -05:00
Jack Robison 575e471553 Merge pull request #3161 from lbryio/ledger-compatibility
handle connections from ledger clients in spv server
2021-01-22 00:09:50 -05:00
Jack Robison 0f5f1aebed accept connections from ledger clients in spv server 2021-01-21 21:37:39 -05:00
Jack Robison 50e17eb1ab Merge pull request #3148 from lbryio/udp-wallet-server-status
UDP based spv server status, improved server selection
2021-01-21 20:55:16 -05:00
Jack Robison 158cc2f660 skip test_single_server_payment 2021-01-21 19:29:59 -05:00
Jack Robison 1066a31acd fix test 2021-01-21 18:46:28 -05:00
Jack Robison 1f9d0f4582 pylint 2021-01-21 16:30:54 -05:00
Jack Robison a6d65233f1 fallback to getting external ip from spv servers instead of internal apis 2021-01-21 16:20:53 -05:00
Jack Robison eff2fe7a1b update tests 2021-01-21 16:20:01 -05:00
Jack Robison 20efdc70b3 use UDP ping for wallet server selection
-only connect to one spv server at a time
-remove session pool
2021-01-21 16:15:42 -05:00
Jack Robison f0d8fb8f1a add UDP based ping protocol for spv servers 2021-01-21 16:08:33 -05:00
Jack Robison f7a380e9b7 start prometheus before block processing 2021-01-21 16:01:33 -05:00
Jack Robison e9c7cf6f63 logging 2021-01-21 14:51:59 -05:00
Jack Robison 68f1661452 add LRUCache with no prometheus metrics 2021-01-21 14:37:08 -05:00
Jack Robison 36fd1b91ae cache server features 2021-01-21 14:37:08 -05:00
Jack Robison a4ec430ac0 improve subscription performance 2021-01-21 14:37:08 -05:00
Jack Robison 519614b2fd skip libtorrent component in tests 2021-01-21 14:37:08 -05:00
Lex Berezhny bf0118c8ef added support for --not_channel_id to txo_list commands 2021-01-21 14:03:59 -05:00
Jack Robison a4db0820bc Merge pull request #3136 from lbryio/fix-reflector-loop
fix reflector loop crashing upon an unexpected error
2021-01-08 12:40:18 -05:00
Jack Robison ee7528413e update tests 2021-01-08 11:54:02 -05:00
Jack Robison 7952fc8324 fix reflector loop propagating cancelled error 2021-01-08 09:53:12 -05:00
Jack Robison 652773d2cf Merge pull request #3129 from lbryio/signing_api
add `channel_sign` api for signing data with a channel identity
2021-01-05 14:13:09 -05:00
Victor Shyba 2a17787242 fix test 2021-01-04 18:04:06 -03:00
Victor Shyba 0a53ad5721 use the comment api signing implementation 2021-01-04 18:04:06 -03:00
Victor Shyba 6da6bdc863 adds jsonrpc_channel_sign signing api 2021-01-04 18:04:06 -03:00
Jack Robison 42ad2bb83f Merge pull request #3135 from lbryio/loop-metrics
add prometheus metrics for asyncio loop
2021-01-02 11:54:21 -05:00
Jack Robison f309a65cb4 fix 2021-01-01 15:25:31 -05:00
Jack Robison 77e19ab1a4 prometheus metrics for asyncio loop responsiveness 2021-01-01 15:25:31 -05:00
Jack Robison 1a996b6ef3 Merge pull request #3134 from lbryio/log_checkpoint
log checkpoint finishing so we can gather information on leftover files bug
2021-01-01 15:24:30 -05:00
Victor Shyba b882f1a010 log checkpoint finishing for gathering information on wal/shm file bug 2020-12-31 16:11:22 -03:00
Jack Robison 82a030e6ff Merge pull request #3130 from lbryio/block-cache-metrics
skip null args in `channel_ids` given to `claim_search`
2020-12-30 20:57:56 -05:00
Jack Robison 0758b85179 skip null args given to channel_ids in claim search 2020-12-29 21:41:07 -05:00
Jack Robison ab3d9bd080 block cache metrics 2020-12-29 21:40:39 -05:00
Thomas Zarebczan 8ff813f689 Take 5!
Why you try to pass version param to other installs?
2020-12-26 12:31:51 -05:00
Thomas Zarebczan 88ae72c0d3 Take 3
Why you fail on uninstall?
2020-12-24 20:28:50 -05:00
Thomas Zarebczan 312aa4be26 Fix Windows build take 2 2020-12-24 04:41:21 -05:00
Thomas Zarebczan cbb06fce9d Fix windows build, Merry christmas! 2020-12-24 04:13:03 -05:00
Thomas Zarebczan f259e497c4 Uninstall old python first 2020-12-24 03:48:17 -05:00
Thomas Zarebczan dd4172ac66 64 bit windows 2020-12-24 03:16:07 -05:00
Jack Robison 66029e60d3 v0.87.0 2020-12-23 21:51:13 -05:00
Jack Robison 364f484f04 Merge pull request #3127 from lbryio/lru-cache-metrics
Add metrics for lru caches
2020-12-23 21:08:01 -05:00
Jack Robison 9dd5159414 increase server cache sizes 2020-12-23 20:47:05 -05:00
Jack Robison 13e38d6fd8 use LRUCache instead of pylru in wallet server 2020-12-23 20:47:05 -05:00
Jack Robison 10dcb64715 lru cache metrics 2020-12-23 20:47:03 -05:00
Jack Robison 7551b51e7d Merge pull request #3128 from lbryio/sync_smaller_batches
Improve memory usage during wallet sync
2020-12-23 20:45:03 -05:00
Victor Shyba adb418aafc group cache hits in a single dict 2020-12-23 21:37:20 -03:00
Victor Shyba 270da80d64 return whole batches 2020-12-23 19:10:19 -03:00
Victor Shyba b2027cfd66 sync in batches of 10, clearing after 2020-12-23 18:30:52 -03:00
Jack Robison 7f1f4eeac6 Merge pull request #3125 from lbryio/fix_order42
fix wallet db integrity / sync bugs
2020-12-22 20:50:46 -05:00
Jack Robison 7a7446c8bd force resync blockchain.db 2020-12-22 20:08:26 -05:00
Jack Robison ddbae294e6 skip doc test on gitlab 2020-12-22 16:55:33 -05:00
Jack Robison 8c71b744f3 fix request_synced_transactions edge cases 2020-12-22 15:19:08 -05:00
Jack Robison 479b5d31a9 fix test 2020-12-22 14:06:53 -05:00
Jack Robison 4cbf4230e8 fix txi.txo_ref 2020-12-22 14:05:37 -05:00
Victor Shyba 6a610187e0 cache bypass 2020-12-22 13:01:30 -03:00
Victor Shyba eb2a4aebba unrestricted and reusing verified cache hits 2020-12-21 16:49:08 -03:00
Victor Shyba 21a2e67755 fix rebase 2020-12-20 21:39:36 -03:00
Victor Shyba 3b9e312615 fix verification 2020-12-20 21:25:26 -03:00
Victor Shyba 26dab04c9e checkpoint 2020-12-20 21:25:26 -03:00
Victor Shyba 00713c0d11 asyncgens 2020-12-20 21:25:26 -03:00
Victor Shyba 751b5f3027 refactor duplicate code 2020-12-20 21:25:26 -03:00
Victor Shyba e8261b000e wip, see jack 2020-12-20 21:23:58 -03:00
Victor Shyba 41ecb70297 join network can only happen after initial header sync returns 2020-12-20 21:15:11 -03:00
Jack Robison 09ee104b8c Merge pull request #3120 from lbryio/fix-reorg-crash
Fix off by one in wallet server reorg
2020-12-16 02:36:11 -05:00
Jack Robison e3a4964787 catch any remaining index errors in fs_tx_hash 2020-12-16 01:28:30 -05:00
Jack Robison 9bf72910a4 fix off by one error in reorg 2020-12-16 01:27:03 -05:00
Jack Robison ee39e20e6d logging 2020-12-16 01:26:19 -05:00
Jack Robison 399d6db6f6 Merge branch 'disable-refresh-histogram' 2020-12-15 20:13:20 -05:00
Jack Robison 0821ce44b5 remove unused heavy call 2020-12-15 19:22:12 -05:00
Jack Robison ea279111c6 Merge pull request #3113 from lbryio/leveldb-performance
Improve performance of fetching transactions and sending address notifications
2020-12-14 16:39:50 -05:00
Jack Robison 674ce02e58 logging 2020-12-14 14:38:36 -05:00
Jack Robison 8dfa2767ec new_touched 2020-12-14 13:52:26 -05:00
Jack Robison 20dad7f07f only notify hashxs touched since last notification 2020-12-14 13:42:20 -05:00
Jack Robison 751cc4c44d don't deserialize mempool in a thread 2020-12-13 13:48:50 -05:00
Jack Robison 2318e6d8e9 faster fs_transactions 2020-12-13 13:48:02 -05:00
Jack Robison 61b4a492c3 v0.86.1 2020-12-08 16:41:46 -05:00
Jack Robison 9db3d01e09 Merge pull request #3105 from lbryio/dont-rerequest-mempool
fix mempool txs being re-requested
2020-12-08 16:00:24 -05:00
Jack Robison 8da73ad3dd improve hash_to_hex_str performance 2020-12-08 15:39:03 -05:00
Jack Robison b8c16d8ac5 fix mempool txs being re-requested with each address notification 2020-12-08 15:39:03 -05:00
Jack Robison 429c0951f3 v0.86.0 2020-12-07 21:19:24 -05:00
Jack Robison 74e103c791 version 2020-12-07 21:15:15 -05:00
Jack Robison f941950ee2 fix comment_create docs 2020-12-07 21:13:43 -05:00
Jack Robison 846df2eef1 v0.86.0 2020-12-07 15:58:05 -05:00
Jack Robison 34ed058c97 Merge pull request #3103 from lbryio/fix-mempool-bloat
fix mempool notification bloat
2020-12-07 15:09:13 -05:00
Jack Robison eae0290978 fix mempool notification bloat 2020-12-07 13:16:37 -05:00
Jack Robison 561368570e Merge pull request #3102 from lbryio/faster-notifications
Improve performance of mempool processing loop
2020-12-03 22:43:11 -05:00
Jack Robison 3467d1fed0 fix tests 2020-12-03 22:16:28 -05:00
Jack Robison d02ff232e5 _single_batch 2020-12-03 20:50:25 -05:00
Jack Robison 2d1c6a5402 fix 2020-12-03 19:15:15 -05:00
Jack Robison eab3b65629 calculate notifications for all subscriptions instead of per session 2020-12-03 19:15:15 -05:00
Jack Robison 20b435732a Merge pull request #3099 from lbryio/increase-tx-cache-size
Increase default `tx_cache_size`
2020-12-03 19:14:34 -05:00
Jack Robison 929617273d resync blockchain.db 2020-12-03 15:56:35 -05:00
Jack Robison 2717bf7d49 increase default tx_cache_size setting 2020-12-03 15:56:35 -05:00
Jack Robison 5cd2ebc960 Merge pull request #3098 from lbryio/cached-txids
store txids in memory, faster address subscription/history
2020-12-02 14:08:07 -05:00
Jack Robison 9b4afe9816 store txids in memory, faster address subscription/history 2020-11-25 16:30:11 -05:00
Jack Robison 23bb5598d5 Merge pull request #3095 from lbryio/fix-transaction-getbatch-order
ensure transactions are returned in the order they're requested
2020-11-24 17:21:12 -05:00
Jack Robison af1d7813e9 ensure transactions are returned in the order they're requested 2020-11-24 11:52:07 -05:00
Jack Robison 16c2e5a585 Merge pull request #3093 from lbryio/fix-merkle
fix off-by-one when getting block txs for tx_merkle
2020-11-23 19:08:11 -05:00
Jack Robison c02750edbd fix tx_merkle 2020-11-23 16:48:24 -05:00
Jack Robison 7204ddafec Merge pull request #3092 from lbryio/faster-transaction-getbatch
faster `blockchain.transaction.get_batch`
2020-11-23 12:29:14 -05:00
Jack Robison faeba9a7e4 faster blockchain.transaction.get_batch 2020-11-23 12:05:47 -05:00
Jack Robison 190d238a1f faster read_headers 2020-11-23 12:03:11 -05:00
Jack Robison 715451b5fb v0.85.0 2020-11-21 00:14:20 -05:00
Jack Robison 87f1895405 Merge pull request #3090 from lbryio/fast-headers
store headers in memory on wallet server
2020-11-20 23:33:18 -05:00
Jack Robison 923d817751 store headers in memory 2020-11-20 22:37:42 -05:00
Jack Robison 0728209b66 Merge pull request #3089 from lbryio/force_checkpoint
Ensure no client .wal files remain on shutdown
2020-11-20 21:12:33 -05:00
Victor Shyba b8b9dcc2ee WAL_CHECKPOINT on close 2020-11-20 20:16:03 -03:00
Jack Robison f35e879852 Merge pull request #3088 from lbryio/batched-inflate-outputs
fix flooding with resolve/claim search
2020-11-20 17:43:56 -05:00
Jack Robison 34f4f12eb9 pylint 2020-11-20 16:15:57 -05:00
Jack Robison fa63bf758d delete single_call_context, use session pool 2020-11-20 15:52:11 -05:00
Jack Robison f6b396ae64 Merge branch 'fix-reorg' 2020-11-20 13:32:59 -05:00
Jack Robison 2c7fd58e34 threaded compress headers 2020-11-20 13:09:54 -05:00
Jack Robison 982f2c9634 fix test 2020-11-20 10:57:28 -05:00
Jack Robison f2fd42b47a use write_batch 2020-11-19 18:37:49 -05:00
Jack Robison 1b4ccad938 allow settings TRENDING_ALGORITHMS to nothing 2020-11-19 18:36:34 -05:00
Jack Robison a9de1ce8e0 fix fs_transactions 2020-11-19 12:40:46 -05:00
Jack Robison ac752d5ec2 Merge pull request #3082 from lbryio/faster-leveldb
store transactions for wallet server in leveldb instead of requesting from lbrycrd
2020-11-17 17:02:11 -05:00
Jack Robison 632d8d02d2 remove dead code 2020-11-17 16:17:24 -05:00
Jack Robison 48aeb26e02 threaded_get_merkle
-remove dead code
2020-11-17 16:05:08 -05:00
Jack Robison 1694af8b5e get transactions from leveldb instead of lbrycrd rpc 2020-11-17 15:30:48 -05:00
Jack Robison 83bcab9cd2 cleanup 2020-11-17 15:30:48 -05:00
Jack Robison bdc7f4b3f5 combine tx_count_db and hashes_db, add tx db 2020-11-17 15:30:48 -05:00
Lex Berezhny 39202a3d79 old sequence resolution test 2020-11-17 11:24:29 -05:00
Lex Berezhny 912065a121 remove url sequence feature from code and tests 2020-11-17 11:24:29 -05:00
Thomas Zarebczan c8466afac2 Remove * from invalid characters
This is already part of other URLs and channels, will be removed from spec as it won't be used for sequencing.
2020-11-16 14:09:04 -05:00
Victor Shyba 2619e162c1 recover invalid state described on #3026 2020-11-09 16:03:38 -05:00
Lex Berezhny e1112e17f8 v0.84.0 2020-11-08 19:42:33 -05:00
Jack Robison 92b2ead74c Merge pull request #3078 from lbryio/null-language
support `none` argument to `any_languages` filter in `claim_search`
2020-11-05 22:09:11 -05:00
Jack Robison bbed9b94c1 null language filtering for claim search 2020-11-05 21:23:49 -05:00
Jack Robison 73d07311db Merge branch 'brendon' 2020-11-05 21:21:24 -05:00
Brendon J. Brewer 1cdff47477 Rewrite of variable_decay.py for speed improvements 2020-11-05 21:15:55 -05:00
Jack Robison 511a5c3f82 Merge pull request #3058 from lbryio/faster-resolve
faster resolve and claim_search
2020-11-05 21:09:07 -05:00
Jack Robison 853885e2ff debug 2020-11-05 19:25:34 -05:00
Jack Robison d83936a66a fix uncaught error 2020-11-04 22:21:35 -05:00
Jack Robison 5517d2bf56 fix new_sdk_server arg for resolve 2020-11-03 16:23:31 -05:00
Jack Robison f21ab49ac5 bump aioupnp requirement 2020-11-03 10:39:38 -05:00
Jack Robison 925a458abe tags 2020-11-03 10:39:08 -05:00
Jack Robison 76946c447f use single_call_context for claim_search and resolve 2020-11-03 10:39:08 -05:00
Jack Robison 2faa29b1c4 fix dht_monitor script 2020-11-03 10:39:08 -05:00
Lex Berezhny 6826cc311d update test to use new url spec 2020-10-30 11:37:54 -04:00
Lex Berezhny 5e17ce0a0b minor fixup 2020-10-30 11:37:54 -04:00
Kevin Raoofi e8d299d3b6 Allow : or # for claim_id
This removes the code for trying multiple patterns and the setup for it

Added a few unit tests to check that the parsed URL is as expected
2020-10-30 11:37:54 -04:00
Kevin Raoofi 7637aa2ab6 Added support for spec compliant URL parsing
Legacy URLs are preserved by attempting to parse the new URL format and,
on failing that, it'll attempt the legacy one.

Tests had to be updated such that the correct things are asserted
against each other.
2020-10-30 11:37:54 -04:00
Jack Robison ab067d1d3a fix unclean closing of blockchain.db 2020-10-26 16:23:28 -04:00
Lex Berezhny e6f84666c7 added glorious data in dictionary doc for support_sum 2020-10-16 12:03:01 -04:00
Lex Berezhny 4c5429af15 fix lint 2020-10-16 12:03:01 -04:00
Alex Grintsvayg 0a0ac3b7c9 pass-through for new support_sum api 2020-10-16 12:03:01 -04:00
Lex Berezhny 24833ce9fb v0.83.0 2020-10-10 13:03:42 -04:00
Lex Berezhny ec2c18dc87 regenerate docs 2020-10-10 13:02:38 -04:00
jessop 7384609e74 support comment pinning 2020-10-10 13:02:38 -04:00
Lex Berezhny 3047649650 v0.82.0 2020-09-29 08:49:16 -04:00
kodxana d298dac3f3 Fixed docker-compose - no snapshot error
Curently new containers do not get provide snapshots from LBRYinc (Wallet and CRD) that causes containers to get into restart loop.
2020-09-29 08:43:49 -04:00
Lex Berezhny 1574bca8a8 Merge pull request #3054 from lbryio/ios-support
iOS support
2020-09-29 08:42:17 -04:00
Lex Berezhny ec2f6c6b80 lint 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 838cc60161 conditional import 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola 310c61a5cc check KIVY_BUILD env variable 2020-09-29 07:52:33 -04:00
Lex Berezhny 318cc15323 Merge pull request #3045 from lbryio/bencode-byte-keys
add forward compatibility for byte datagram keys
2020-09-29 07:23:49 -04:00
Jack Robison 3a64ceb4d6 add forward compatibility for byte datagram keys 2020-09-28 15:56:13 -04:00
Lex Berezhny d0f21c0095 Merge pull request #3050 from lbryio/language-indexes
add indexes for `any_languages` argument to `claim_search`
2020-09-28 15:52:41 -04:00
Jack Robison 46dc15dd29 use language indexes 2020-09-28 14:45:51 -04:00
Jack Robison 8dc654b513 add language table and indexes 2020-09-28 14:45:51 -04:00
Jack Robison 7000ac3f3f update indexed tags 2020-09-28 14:45:51 -04:00
Lex Berezhny 43c2e8d8e9 Merge pull request #3051 from lbryio/comments-v2
Comments v2 with Reactions
2020-09-28 14:44:54 -04:00
jessop 0231139b01 support anonymous react_list 2020-09-27 16:36:51 -04:00
jessop d6ee6446dd react list takes comment_ids 2020-09-26 15:21:26 -04:00
Lex Berezhny 7b666efcf8 lbrycrd download path 2020-09-24 20:11:40 -04:00
Lex Berezhny eb5d2198fc lint 2020-09-24 20:09:46 -04:00
Lex Berezhny 34e44ebd1c minor cleanup and lint fixes 2020-09-24 20:03:22 -04:00
jessop bf2f4bc040 wip 2020-09-24 17:47:59 -04:00
jessop 9dc4559aba new comment and reactions api 2020-09-23 16:43:28 -04:00
Lex Berezhny eba8856261 v0.81.0 2020-08-19 15:47:53 -04:00
Lex Berezhny a8c5aa471a update docs 2020-08-19 15:46:29 -04:00
Lex Berezhny 52f6dcf092 Merge pull request #3021 from lbryio/debug-reflector
Improve reflector upload cancellation handling
2020-08-19 15:44:27 -04:00
Jack Robison dec79f3742 pylint 2020-08-19 15:24:17 -04:00
Jack Robison 8bdcac0f3e close reader before deleting blob 2020-08-19 15:24:17 -04:00
Jack Robison 8426b674a3 improve reflector upload cancellation handling 2020-08-19 15:24:17 -04:00
Lex Berezhny da391bcc8d Merge pull request #3028 from lbryio/limit_claims_per_channel
added `--limit_claims_per_channel` argument to `claim_search` to only return up to the specified number of claims per channel
2020-08-19 15:22:53 -04:00
Lex Berezhny 2d7443d454 slight cleanup 2020-08-19 13:59:26 -04:00
Lex Berezhny 991987ed76 docopt fix 2020-08-19 12:08:41 -04:00
Lex Berezhny ec24ebf2cf added --limit_claims_per_channel argument to claim_search to only return up to the specified number of claims per channel 2020-08-19 10:51:31 -04:00
Lex Berezhny 6ba0976085 Merge pull request #3022 from btzr-io/patch-2
Add webvtt mime type ( subtitles, captions )
2020-08-12 18:58:07 -04:00
Baltazar Gomez 2b88d01a01 Add webvtt mime type ( subtitles, captions ) 2020-08-12 02:25:01 -05:00
Lex Berezhny 0c09f24cbf v0.80.0 2020-08-04 16:53:52 -04:00
Lex Berezhny 61d22afeba Merge pull request #3016 from lbryio/release_all_outputs_on_start
release reserved outputs on startup
2020-08-04 16:53:15 -04:00
Lex Berezhny 9f1ed6e8c3 release_all_outputs on startup 2020-08-04 15:03:58 -04:00
Lex Berezhny bbc4113cac Merge pull request #3015 from lbryio/new_sdk_forwarding
added support for using the new SDK for `resolve` and `claim_search` by passing `--new_sdk_server`
2020-08-04 15:03:39 -04:00
Lex Berezhny 91194bf422 doc fix 2020-08-04 14:12:40 -04:00
Lex Berezhny 9c5f940b00 claim search forwarding to new sdk 2020-08-04 12:11:02 -04:00
Lex Berezhny 455b4043b8 new resolve 2020-08-04 11:33:39 -04:00
101 changed files with 5914 additions and 2692 deletions

No files matched your search

+67
View File
@@ -0,0 +1,67 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '21 0 * * 4'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
language: [ 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
# Learn more:
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
+13 -3
View File
@@ -22,10 +22,9 @@ jobs:
with:
python-version: '3.7'
- run: make install tools
- working-directory: lbry
env:
- env:
HOME: /tmp
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
run: make test-unit-coverage
tests-integration:
name: "tests / integration"
@@ -37,6 +36,17 @@ jobs:
- blockchain
- other
steps:
- name: Configure sysctl limits
run: |
sudo swapoff -a
sudo sysctl -w vm.swappiness=1
sudo sysctl -w fs.file-max=262144
sudo sysctl -w vm.max_map_count=262144
- name: Runs Elasticsearch
uses: elastic/elastic-github-actions/elasticsearch@master
with:
stack-version: 7.6.0
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
+44
View File
@@ -0,0 +1,44 @@
# This workflow integrates a collection of open source static analysis tools
# with GitHub code scanning. For documentation, or to provide feedback, visit
# https://github.com/github/ossar-action
name: OSSAR
on:
push:
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '28 17 * * 1'
jobs:
OSSAR-Scan:
# OSSAR runs on windows-latest.
# ubuntu-latest and macos-latest support coming soon
runs-on: windows-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Ensure a compatible version of dotnet is installed.
# The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201.
# A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action.
# GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped.
# For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action:
# - name: Install .NET
# uses: actions/setup-dotnet@v1
# with:
# dotnet-version: '3.1.x'
# Run open source static analysis tools
- name: Run OSSAR
uses: github/ossar-action@v1
id: ossar
# Upload results to the Security tab
- name: Upload OSSAR results
uses: github/codeql-action/upload-sarif@v1
with:
sarif_file: ${{ steps.ossar.outputs.sarifFile }}
+3 -1
View File
@@ -6,10 +6,12 @@
/.coverage*
/lbry-venv
/venv
/lbry/blockchain
lbry.egg-info
__pycache__
_trial_temp/
trending*.log
/tests/integration/blockchain/files
/tests/.coverage.*
@@ -17,4 +19,4 @@ _trial_temp/
/lbry/wallet/bin
/.vscode
/.gitignore
/.gitignore
+14 -5
View File
@@ -1,5 +1,11 @@
default:
image: python:3.7
services:
- name: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
alias: elasticsearch
command: [ "bin/elasticsearch", "-Expack.security.enabled=false", "-Ediscovery.type=single-node" ]
variables:
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
#cache:
@@ -41,13 +47,14 @@ test:datanetwork-integration:
stage: test
script:
- pip install tox-travis
- tox -e datanetwork --recreate
- ELASTIC_HOST=elasticsearch tox -e datanetwork --recreate
test:blockchain-integration:
stage: test
script:
- curl "http://elasticsearch:9200/_cat/health"
- pip install tox-travis
- tox -e blockchain
- ELASTIC_HOST=elasticsearch tox -e blockchain
test:other-integration:
stage: test
@@ -55,13 +62,13 @@ test:other-integration:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e other
- ELASTIC_HOST=elasticsearch tox -e other
test:json-api:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
- ELASTIC_HOST=elasticsearch HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
@@ -121,9 +128,11 @@ build:windows:
GIT_DEPTH: 5
before_script:
- ./docker/install_choco.ps1
- choco install -y --x86 python3 7zip checksum
- choco install -y --x64 python --version=3.7.9
- choco install -y 7zip checksum
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
- pip --version # | findstr /E '\(python 3\.7\)'
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
- pip install virtualenv pywin32
- virtualenv venv
- venv/Scripts/activate.ps1
+10 -2
View File
@@ -1,4 +1,4 @@
.PHONY: install tools lint test idea
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
install:
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
@@ -16,7 +16,15 @@ lint:
pylint --rcfile=setup.cfg lbry
#mypy --ignore-missing-imports lbry
test:
test: test-unit test-integration
test-unit:
python -m unittest discover tests.unit
test-unit-coverage:
coverage run -p --source=lbry -m unittest discover -vv tests.unit
test-integration:
tox
idea:
+9
View File
@@ -0,0 +1,9 @@
# Security Policy
## Supported Versions
While we are not at v1.0 yet, only the latest release will be supported.
## Reporting a Vulnerability
See https://lbry.com/faq/security
+4 -2
View File
@@ -1,4 +1,4 @@
FROM ubuntu:19.10
FROM debian:10-slim
ARG user=lbry
ARG db_dir=/database
@@ -13,7 +13,9 @@ RUN apt-get update && \
wget \
tar unzip \
build-essential \
python3 \
pkg-config \
libleveldb-dev \
python3.7 \
python3-dev \
python3-pip \
python3-wheel \
+31 -20
View File
@@ -1,34 +1,45 @@
version: "3"
volumes:
lbrycrd:
wallet_server:
es01:
services:
lbrycrd:
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
restart: always
ports: # accessible from host
- "9246:9246" # rpc port
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
- "9245" # node-to-node comms port
volumes:
- "lbrycrd:/data/.lbrycrd"
environment:
- RUN_MODE=default
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- RPC_ALLOW_IP=0.0.0.0/0
wallet_server:
depends_on:
- es01
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
depends_on:
- lbrycrd
restart: always
network_mode: host
ports:
- "50001:50001" # rpc port
- "50005:50005" # websocket port
#- "2112:2112" # uncomment to enable prometheus
- "2112:2112" # uncomment to enable prometheus
volumes:
- "wallet_server:/database"
environment:
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
- TCP_PORT=50001
- PROMETHEUS_PORT=2112
- QUERY_TIMEOUT_MS=3000 # how long search queries allowed to run before cancelling, in milliseconds
- TRENDING_ALGORITHMS=variable_decay
- MAX_SEND=10000000000000 # deprecated. leave it high until its removed
- MAX_SUBS=1000000000000 # deprecated. leave it high until its removed
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 e4e230b131082f6b10c8f7994bbb83f29e8e6fb9
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- indices.query.bool.max_clause_count=4096
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- es01:/usr/share/elasticsearch/data
ports:
- 127.0.0.1:9200:9200
+3 -1
View File
@@ -20,4 +20,6 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
rm "$filename"
fi
/home/lbry/.local/bin/torba-server "$@"
/home/lbry/.local/bin/lbry-hub-elastic-sync /database/claims.db
echo 'starting server'
/home/lbry/.local/bin/lbry-hub "$@"
+669 -187
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.79.1"
__version__ = "0.96.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+1 -1
View File
@@ -337,9 +337,9 @@ class BlobFile(AbstractBlob):
return self.loop.create_task(write_blob())
def delete(self):
super().delete()
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
@classmethod
async def create_from_unencrypted(
+2 -2
View File
@@ -2,7 +2,7 @@ import os
import typing
import asyncio
import logging
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbry.stream.descriptor import StreamDescriptor
from lbry.connection_manager import ConnectionManager
@@ -32,7 +32,7 @@ class BlobManager:
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
+4 -2
View File
@@ -544,6 +544,7 @@ class Config(CLIConfig):
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
hub_timeout = Float("Timeout when making a hub request", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
@@ -565,6 +566,7 @@ class Config(CLIConfig):
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
@@ -611,7 +613,7 @@ class Config(CLIConfig):
('lbrynet4.lbry.com', 4444) # ASIA
])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
@@ -638,7 +640,7 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
transaction_cache_size = Integer("Transaction cache size", 100_000)
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
+3 -22
View File
@@ -1,14 +1,14 @@
import typing
import asyncio
import logging
import ipaddress
from binascii import hexlify
from dataclasses import dataclass, field
from functools import lru_cache
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
from lbry.dht import constants
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
ALLOW_LOCALHOST = False
log = logging.getLogger(__name__)
@@ -20,28 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
ALLOW_LOCALHOST = False
def is_valid_public_ipv4(address, allow_localhost: bool = False):
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
try:
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
return _is_valid_public_ipv4(address, allow_localhost)
class PeerManager:
+17 -7
View File
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
self.response = response.decode()
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
def _decode_datagram(datagram: bytes):
msg_types = {
REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram,
@@ -152,19 +152,29 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
}
primitive: typing.Dict = bdecode(datagram)
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
converted = {
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
}
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
else:
raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type]
decoded = {
k: primitive[i] # pylint: disable=unsubscriptable-object
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.required_fields)
if i in primitive # pylint: disable=unsupported-membership-test
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
}
for i, _ in enumerate(OPTIONAL_FIELDS):
if i + OPTIONAL_ARG_OFFSET in primitive:
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
return decoded, datagram_class
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
decoded, datagram_class = _decode_datagram(datagram)
return datagram_class(**decoded)
+1 -1
View File
@@ -132,7 +132,7 @@ class AnalyticsManager:
async def run(self):
while True:
if self.enabled:
self.external_ip = await utils.get_external_ip()
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
await self._send_heartbeat()
await asyncio.sleep(1800)
+31 -18
View File
@@ -22,41 +22,54 @@ def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
except KeyError:
pass
return False
def sign_comment(comment: dict, channel: Output, abandon=False):
def verify(channel, data, signature, channel_hash=None):
pieces = [
signature['signing_ts'].encode(),
channel_hash or channel.claim_hash,
data
]
return Output.is_signature_valid(
get_encoded_signature(signature['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
comment.update(sign(channel, signing_field.encode()))
def sign(channel, data):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
pieces = [timestamp.encode(), channel.claim_hash, data]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment.update({
return {
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
}
def sign_reaction(reaction: dict, channel: Output):
signing_field = reaction['channel_name']
reaction.update(sign(channel, signing_field.encode()))
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
+24 -18
View File
@@ -119,13 +119,14 @@ class WalletComponent(Component):
async def get_status(self):
if self.wallet_manager is None:
return
session_pool = self.wallet_manager.ledger.network.session_pool
sessions = session_pool.sessions
is_connected = self.wallet_manager.ledger.network.is_connected
sessions = []
connected = None
if self.wallet_manager.ledger.network.client:
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
if addr_and_port:
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
if is_connected:
addr, port = self.wallet_manager.ledger.network.client.server
connected = f"{addr}:{port}"
sessions.append(self.wallet_manager.ledger.network.client)
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
@@ -137,8 +138,8 @@ class WalletComponent(Component):
'availability': session.available,
} for session in sessions
],
'known_servers': len(sessions),
'available_servers': len(list(session_pool.available_sessions))
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
'available_servers': 1 if is_connected else 0
}
if self.wallet_manager.ledger.network.remote_height:
@@ -274,7 +275,7 @@ class DHTComponent(Component):
external_ip = upnp_component.external_ip
storage = self.component_manager.get_component(DATABASE_COMPONENT)
if not external_ip:
external_ip = await utils.get_external_ip()
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if not external_ip:
log.warning("failed to get external ip")
@@ -328,7 +329,7 @@ class HashAnnouncerComponent(Component):
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -351,7 +352,10 @@ class FileManagerComponent(Component):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
node = self.component_manager.get_component(DHT_COMPONENT) \
if self.component_manager.has_component(DHT_COMPONENT) else None
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
try:
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
except NameError:
torrent = None
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.file_manager = FileManager(
@@ -360,7 +364,7 @@ class FileManagerComponent(Component):
self.file_manager.source_managers['stream'] = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node,
)
if TorrentSession:
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
self.file_manager.source_managers['torrent'] = TorrentManager(
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
)
@@ -472,7 +476,7 @@ class UPnPComponent(Component):
pass
if external_ip and not is_valid_public_ipv4(external_ip):
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
external_ip = await utils.get_external_ip()
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
if external_ip:
@@ -530,13 +534,15 @@ class UPnPComponent(Component):
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await utils.get_external_ip()
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
return
success = False
await self._maintain_redirects()
if self.upnp:
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
if not self.upnp_redirects and not all(
x in self.component_manager.skip_components
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
):
log.error("failed to setup upnp")
else:
success = True
@@ -545,9 +551,9 @@ class UPnPComponent(Component):
else:
log.error("failed to setup upnp")
if not self.external_ip:
self.external_ip = await utils.get_external_ip()
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip:
log.info("detected external ip using lbry.com fallback")
log.info("detected external ip using %s fallback", probed_url)
if self.component_manager.analytics_manager:
self.component_manager.loop.create_task(
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
+373 -117
View File
@@ -21,6 +21,7 @@ import base58
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter
from google.protobuf.message import DecodeError
from lbry.wallet import (
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
Transaction, Output, Input, Account, database
@@ -118,13 +119,6 @@ STREAM_STAGES = [
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
]
CONNECTION_STATUS_CONNECTED = 'connected'
CONNECTION_STATUS_NETWORK = 'network_connection'
CONNECTION_MESSAGES = {
CONNECTION_STATUS_CONNECTED: 'No connection problems detected',
CONNECTION_STATUS_NETWORK: "Your internet connection appears to have been interrupted",
}
SHORT_ID_LEN = 20
MAX_UPDATE_FEE_ESTIMATE = 0.3
DEFAULT_PAGE_SIZE = 20
@@ -338,13 +332,13 @@ class Daemon(metaclass=JSONRPCServerType):
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
rpc_app = web.Application()
rpc_app.router.add_get('/lbryapi', self.handle_old_jsonrpc)
rpc_app.router.add_post('/lbryapi', self.handle_old_jsonrpc)
rpc_app.router.add_post('/', self.handle_old_jsonrpc)
rpc_app.router.add_options('/', self.add_cors_headers)
self.rpc_runner = web.AppRunner(rpc_app)
streaming_app = web.Application()
@@ -357,9 +351,6 @@ class Daemon(metaclass=JSONRPCServerType):
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.metrics_runner = web.AppRunner(prom_app)
self.need_connection_status_refresh = asyncio.Event()
self._connection_status_task: Optional[asyncio.Task] = None
@property
def dht_node(self) -> typing.Optional['Node']:
return self.component_manager.get_component(DHT_COMPONENT)
@@ -464,31 +455,10 @@ class Daemon(metaclass=JSONRPCServerType):
if not os.path.isdir(self.conf.download_dir):
os.makedirs(self.conf.download_dir)
async def update_connection_status(self):
connected = await utils.async_check_connection()
if connected and not self._connection_status[1]:
log.info("detected internet connection is working")
elif not connected and self._connection_status[1]:
log.warning("detected internet connection was lost")
self._connection_status = (self.component_manager.loop.time(), connected)
async def keep_connection_status_up_to_date(self):
while True:
try:
await asyncio.wait_for(self.need_connection_status_refresh.wait(), 300)
except asyncio.TimeoutError:
pass
await self.update_connection_status()
self.need_connection_status_refresh.clear()
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
self.need_connection_status_refresh.set()
self._connection_status_task = self.component_manager.loop.create_task(
self.keep_connection_status_up_to_date()
)
await self.analytics_manager.send_server_startup()
await self.rpc_runner.setup()
@@ -548,15 +518,19 @@ class Daemon(metaclass=JSONRPCServerType):
await self.component_startup_task
async def stop(self):
if self._connection_status_task:
if not self._connection_status_task.done():
self._connection_status_task.cancel()
self._connection_status_task = None
if self.component_startup_task is not None:
if self.component_startup_task.done():
await self.component_manager.stop()
else:
self.component_startup_task.cancel()
# the wallet component might have not started
try:
wallet_component = self.component_manager.get_actual_component('wallet')
except NameError:
pass
else:
await wallet_component.stop()
await self.component_manager.stop()
log.info("stopped api components")
await self.rpc_runner.cleanup()
await self.streaming_runner.cleanup()
@@ -566,6 +540,17 @@ class Daemon(metaclass=JSONRPCServerType):
self.analytics_manager.stop()
log.info("finished shutting down")
async def add_cors_headers(self, request):
if self.conf.allowed_origin:
return web.Response(
headers={
'Access-Control-Allow-Origin': self.conf.allowed_origin,
'Access-Control-Allow-Methods': self.conf.allowed_origin,
'Access-Control-Allow-Headers': self.conf.allowed_origin,
}
)
return None
async def handle_old_jsonrpc(self, request):
ensure_request_allowed(request, self.conf)
data = await request.json()
@@ -586,8 +571,16 @@ class Daemon(metaclass=JSONRPCServerType):
'After successfully executing the command, failed to encode result for JSON RPC response.',
{'traceback': format_exc()}
), ledger=ledger)
headers = {}
if self.conf.allowed_origin:
headers.update({
'Access-Control-Allow-Origin': self.conf.allowed_origin,
'Access-Control-Allow-Methods': self.conf.allowed_origin,
'Access-Control-Allow-Headers': self.conf.allowed_origin,
})
return web.Response(
text=encoded_result,
headers=headers,
content_type='application/json'
)
@@ -927,10 +920,6 @@ class Daemon(metaclass=JSONRPCServerType):
}
}
"""
if not self._connection_status[1]:
self.need_connection_status_refresh.set()
connection_code = CONNECTION_STATUS_CONNECTED if self._connection_status[1] else CONNECTION_STATUS_NETWORK
ffmpeg_status = await self._video_file_analyzer.status()
running_components = self.component_manager.get_components_status()
response = {
@@ -938,10 +927,6 @@ class Daemon(metaclass=JSONRPCServerType):
'is_running': all(running_components.values()),
'skipped_components': self.component_manager.skip_components,
'startup_status': running_components,
'connection_status': {
'code': connection_code,
'message': CONNECTION_MESSAGES[connection_code],
},
'ffmpeg_status': ffmpeg_status
}
for component in self.component_manager.components:
@@ -986,10 +971,12 @@ class Daemon(metaclass=JSONRPCServerType):
[--include_sent_supports]
[--include_sent_tips]
[--include_received_tips]
[--new_sdk_server=<new_sdk_server>]
Options:
--urls=<urls> : (str, list) one or more urls to resolve
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase receipts
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_purchase_receipt : (bool) lookup and include a receipt if this wallet
has purchased the claim being resolved
--include_is_my_output : (bool) lookup and include a boolean indicating
@@ -1101,7 +1088,7 @@ class Daemon(metaclass=JSONRPCServerType):
--download_directory=<download_directory> : (str) full path to the directory to download into
--timeout=<timeout> : (int) download timeout in number of seconds
--save_file=<save_file> : (bool) save the file to the downloads directory
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase receipts
Returns: {File}
"""
@@ -1569,7 +1556,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
account = wallet.get_account_or_default(account_id)
balance = await account.get_detailed_balance(
confirmations=confirmations, reserved_subtotals=True, read_only=True
confirmations=confirmations, read_only=True
)
return dict_values_to_lbc(balance)
@@ -2267,7 +2254,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...] [--is_spent]
[--channel_id=<channel_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>]
[--has_source | --has_no_source] [--page=<page>] [--page_size=<page_size>]
[--resolve] [--order_by=<order_by>] [--no_totals] [--include_received_tips]
Options:
@@ -2278,13 +2265,15 @@ class Daemon(metaclass=JSONRPCServerType):
--is_spent : (bool) shows previous claim updates and abandons
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--has_source : (bool) list claims containing a source field
--has_no_source : (bool) list claims not containing a source field
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--resolve : (bool) resolves each claim to provide additional metadata
--order_by=<order_by> : (str) field to order by: 'name', 'height', 'amount'
--no_totals : (bool) do not calculate the total number of pages and items in result set
(significant performance boost)
--include_received_tips : (bool) calculate the amount of tips recieved for claim outputs
--include_received_tips : (bool) calculate the amount of tips received for claim outputs
Returns: {Paginated[Output]}
"""
@@ -2293,6 +2282,41 @@ class Daemon(metaclass=JSONRPCServerType):
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(**kwargs)
async def jsonrpc_support_sum(self, claim_id, new_sdk_server, include_channel_content=False, **kwargs):
"""
List total staked supports for a claim, grouped by the channel that signed the support.
+
+ If claim_id is a channel claim, you can use --include_channel_content to also include supports for
+ content claims in the channel.
!!!! NOTE: PAGINATION DOES NOT DO ANYTHING AT THE MOMENT !!!!!
Usage:
support_sum <claim_id> <new_sdk_server>
[--include_channel_content]
[--page=<page>] [--page_size=<page_size>]
Options:
--claim_id=<claim_id> : (str) claim id
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
--include_channel_content : (bool) if claim_id is for a channel, include supports for claims in
that channel
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
Returns: {Paginated[Dict]}
"""
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
support_sums = await self.ledger.sum_supports(
new_sdk_server, claim_id=claim_id, include_channel_content=include_channel_content, **kwargs
)
return {
"items": support_sums,
"page": page_num,
"page_size": page_size
}
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
"""
@@ -2308,6 +2332,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel=<channel> |
[[--channel_ids=<channel_ids>...] [--not_channel_ids=<not_channel_ids>...]]]
[--has_channel_signature] [--valid_channel_signature | --invalid_channel_signature]
[--limit_claims_per_channel=<limit_claims_per_channel>]
[--is_controlling] [--release_time=<release_time>] [--public_key_id=<public_key_id>]
[--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
[--height=<height>] [--creation_height=<creation_height>]
@@ -2325,8 +2350,10 @@ class Daemon(metaclass=JSONRPCServerType):
[--not_languages=<not_languages>...]
[--any_locations=<any_locations>...] [--all_locations=<all_locations>...]
[--not_locations=<not_locations>...]
[--order_by=<order_by>...] [--page=<page>] [--page_size=<page_size>]
[--order_by=<order_by>...] [--no_totals] [--page=<page>] [--page_size=<page_size>]
[--wallet_id=<wallet_id>] [--include_purchase_receipt] [--include_is_my_output]
[--has_source | --has_no_source]
[--new_sdk_server=<new_sdk_server>]
Options:
--name=<name> : (str) claim name (normalized)
@@ -2355,6 +2382,8 @@ class Daemon(metaclass=JSONRPCServerType):
--invalid_channel_signature : (bool) claims with invalid channel signature or no signature,
use in conjunction with --has_channel_signature to
only get claims with invalid signatures
--limit_claims_per_channel=<limit_claims_per_channel>: (int) only return up to the specified
number of claims per channel
--is_controlling : (bool) winning claims of their respective name
--public_key_id=<public_key_id> : (str) only return channels having this public key id, this is
the same key as used in the wallet file to map
@@ -2401,7 +2430,7 @@ class Daemon(metaclass=JSONRPCServerType):
--reposted_claim_id=<reposted_claim_id>: (str) all reposts of the specified original claim id
--reposted=<reposted> : (int) claims reposted this many times (supports
equality constraints)
--claim_type=<claim_type> : (str) filter by 'channel', 'stream' or 'unknown'
--claim_type=<claim_type> : (str) filter by 'channel', 'stream', 'repost' or 'collection'
--stream_types=<stream_types> : (list) filter by 'video', 'image', 'document', etc
--media_types=<media_types> : (list) filter by 'video/mp4', 'image/png', etc
--fee_currency=<fee_currency> : (string) specify fee currency: LBC, BTC, USD
@@ -2427,11 +2456,14 @@ class Daemon(metaclass=JSONRPCServerType):
'trending_local', 'trending_global', 'activation_height'
--no_totals : (bool) do not calculate the total number of pages and items in result set
(significant performance boost)
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase reciepts
--wallet_id=<wallet_id> : (str) wallet to check for claim purchase receipts
--include_purchase_receipt : (bool) lookup and include a receipt if this wallet
has purchased the claim
--include_is_my_output : (bool) lookup and include a boolean indicating
if claim being resolved is yours
--has_source : (bool) find claims containing a source field
--has_no_source : (bool) find claims not containing a source field
--new_sdk_server=<new_sdk_server> : (str) URL of the new SDK server (EXPERIMENTAL)
Returns: {Paginated[Output]}
"""
@@ -2442,6 +2474,8 @@ class Daemon(metaclass=JSONRPCServerType):
kwargs['signature_valid'] = 1
if kwargs.pop('invalid_channel_signature', False):
kwargs['signature_valid'] = 0
if 'has_no_source' in kwargs:
kwargs['has_source'] = not kwargs.pop('has_no_source')
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **kwargs)
@@ -2738,6 +2772,39 @@ class Daemon(metaclass=JSONRPCServerType):
return tx
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_sign(
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
"""
Signs data using the specified channel signing key.
Usage:
channel_sign [<channel_name> | --channel_name=<channel_name>]
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
Returns:
(dict) Signature if successfully made, (None) or an error otherwise
{
"signature": (str) The signature of the comment,
"signing_ts": (str) The timestamp used to sign the comment,
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
signing_channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
return comment_client.sign(signing_channel, unhexlify(hexdata))
@requires(WALLET_COMPONENT)
async def jsonrpc_channel_abandon(
self, claim_id=None, txid=None, nout=None, account_id=None, wallet_id=None,
@@ -2822,7 +2889,7 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
kwargs['type'] = 'channel'
if 'is_spent' not in kwargs:
if 'is_spent' not in kwargs or not kwargs['is_spent']:
kwargs['is_not_spent'] = True
return self.jsonrpc_txo_list(*args, **kwargs)
@@ -2931,13 +2998,13 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
publish (<name> | --name=<name>) [--bid=<bid>] [--file_path=<file_path>]
[--validate_file] [--optimize_file]
[--file_name=<file_name>] [--file_hash=<file_hash>] [--validate_file] [--optimize_file]
[--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>] [--fee_address=<fee_address>]
[--title=<title>] [--description=<description>] [--author=<author>]
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
[--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>]
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
@@ -2947,6 +3014,8 @@ class Daemon(metaclass=JSONRPCServerType):
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
--bid=<bid> : (decimal) amount to back the claim
--file_path=<file_path> : (str) path to file to be associated with name.
--file_name=<file_name> : (str) name of file to be associated with stream.
--file_hash=<file_hash> : (str) hash of file to be associated with stream.
--validate_file : (bool) validate that the video container and encodings match
common web browser support or that optimization succeeds if specified.
FFmpeg is required
@@ -3009,6 +3078,7 @@ class Daemon(metaclass=JSONRPCServerType):
--width=<width> : (int) image/video width, automatically calculated from media file
--height=<height> : (int) image/video height, automatically calculated from media file
--duration=<duration> : (int) audio/video duration in seconds, automatically calculated
--sd_hash=<sd_hash> : (str) sd_hash of stream
--channel_id=<channel_id> : (str) claim id of the publisher channel
--channel_name=<channel_name> : (str) name of publisher channel
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
@@ -3035,8 +3105,6 @@ class Daemon(metaclass=JSONRPCServerType):
if len(claims) == 0:
if 'bid' not in kwargs:
raise Exception("'bid' is a required argument for new publishes.")
if 'file_path' not in kwargs:
raise Exception("'file_path' is a required argument for new publishes.")
return await self.jsonrpc_stream_create(name, **kwargs)
elif len(claims) == 1:
assert claims[0].claim.is_stream, f"Claim at name '{name}' is not a stream claim."
@@ -3120,7 +3188,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
async def jsonrpc_stream_create(
self, name, bid, file_path, allow_duplicate_name=False,
self, name, bid, file_path=None, allow_duplicate_name=False,
channel_id=None, channel_name=None, channel_account_id=None,
account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None,
preview=False, blocking=False, validate_file=False, optimize_file=False, **kwargs):
@@ -3128,15 +3196,15 @@ class Daemon(metaclass=JSONRPCServerType):
Make a new stream claim and announce the associated file to lbrynet.
Usage:
stream_create (<name> | --name=<name>) (<bid> | --bid=<bid>) (<file_path> | --file_path=<file_path>)
[--validate_file] [--optimize_file]
stream_create (<name> | --name=<name>) (<bid> | --bid=<bid>) [<file_path> | --file_path=<file_path>]
[--file_name=<file_name>] [--file_hash=<file_hash>] [--validate_file] [--optimize_file]
[--allow_duplicate_name=<allow_duplicate_name>]
[--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>] [--fee_address=<fee_address>]
[--title=<title>] [--description=<description>] [--author=<author>]
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
[--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>]
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
@@ -3146,6 +3214,8 @@ class Daemon(metaclass=JSONRPCServerType):
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
--bid=<bid> : (decimal) amount to back the claim
--file_path=<file_path> : (str) path to file to be associated with name.
--file_name=<file_name> : (str) name of file to be associated with stream.
--file_hash=<file_hash> : (str) hash of file to be associated with stream.
--validate_file : (bool) validate that the video container and encodings match
common web browser support or that optimization succeeds if specified.
FFmpeg is required
@@ -3210,13 +3280,14 @@ class Daemon(metaclass=JSONRPCServerType):
--width=<width> : (int) image/video width, automatically calculated from media file
--height=<height> : (int) image/video height, automatically calculated from media file
--duration=<duration> : (int) audio/video duration in seconds, automatically calculated
--sd_hash=<sd_hash> : (str) sd_hash of stream
--channel_id=<channel_id> : (str) claim id of the publisher channel
--channel_name=<channel_name> : (str) name of the publisher channel
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--account_id=<account_id> : (str) account to use for holding the transaction
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
it will be determined automatically from the account
--preview : (bool) do not broadcast the transaction
@@ -3242,20 +3313,24 @@ class Daemon(metaclass=JSONRPCServerType):
f"Use --allow-duplicate-name flag to override."
)
file_path, spec = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
kwargs.update(spec)
if file_path is not None:
file_path, spec = await self._video_file_analyzer.verify_or_repair(
validate_file, optimize_file, file_path, ignore_non_video=True
)
kwargs.update(spec)
claim = Claim()
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
if file_path is not None:
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
else:
claim.stream.update(**kwargs)
tx = await Transaction.claim_create(
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
)
new_txo = tx.outputs[0]
file_stream = None
if not preview:
if not preview and file_path is not None:
file_stream = await self.file_manager.create_stream(file_path)
claim.stream.source.sd_hash = file_stream.sd_hash
new_txo.script.generate()
@@ -3271,7 +3346,8 @@ class Daemon(metaclass=JSONRPCServerType):
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
if file_path is not None:
await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
self.component_manager.loop.create_task(save_claims())
self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish'))
@@ -3301,7 +3377,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--locations=<locations>...] [--clear_locations]
[--license=<license>] [--license_url=<license_url>] [--thumbnail_url=<thumbnail_url>]
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
[--channel_id=<channel_id> | --channel_name=<channel_name> | --clear_channel]
[--sd_hash=<sd_hash>] [--channel_id=<channel_id> | --channel_name=<channel_name> | --clear_channel]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
@@ -3380,6 +3456,7 @@ class Daemon(metaclass=JSONRPCServerType):
--width=<width> : (int) image/video width, automatically calculated from media file
--height=<height> : (int) image/video height, automatically calculated from media file
--duration=<duration> : (int) audio/video duration in seconds, automatically calculated
--sd_hash=<sd_hash> : (str) sd_hash of stream
--channel_id=<channel_id> : (str) claim id of the publisher channel
--channel_name=<channel_name> : (str) name of the publisher channel
--clear_channel : (bool) remove channel signature
@@ -3451,9 +3528,10 @@ class Daemon(metaclass=JSONRPCServerType):
if replace:
claim = Claim()
claim.stream.message.source.CopyFrom(
old_txo.claim.stream.message.source
)
if old_txo.claim.stream.has_source:
claim.stream.message.source.CopyFrom(
old_txo.claim.stream.message.source
)
stream_type = old_txo.claim.stream.stream_type
if stream_type:
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
@@ -3624,14 +3702,16 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
collection_create (<name> | --name=<name>) (<bid> | --bid=<bid>)
(<claims>... | --claims=<claims>...)
[--allow_duplicate_name]
[--title=<title>] [--description=<description>]
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
[--thumbnail_url=<thumbnail_url>]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
[--preview] [--blocking]
(--claims=<claims>...)
[--allow_duplicate_name]
[--title=<title>] [--description=<description>]
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
[--thumbnail_url=<thumbnail_url>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
[--preview] [--blocking]
Options:
--name=<name> : (str) name of the collection
@@ -3682,6 +3762,10 @@ class Daemon(metaclass=JSONRPCServerType):
... --locations="{'country': 'US', 'state': 'NH'}"
--thumbnail_url=<thumbnail_url>: (str) thumbnail url
--channel_id=<channel_id> : (str) claim id of the publisher channel
--channel_name=<channel_name> : (str) name of the publisher channel
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
for channel certificates, defaults to all accounts.
--account_id=<account_id> : (str) account to use for holding the transaction
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
@@ -3745,7 +3829,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--locations=<locations>...] [--clear_locations]
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--claim_address=<claim_address>] [--new_signing_key]
[--claim_address=<claim_address>]
[--funding_account_ids=<funding_account_ids>...]
[--preview] [--blocking] [--replace]
@@ -3803,7 +3887,6 @@ class Daemon(metaclass=JSONRPCServerType):
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
--claim_address=<claim_address>: (str) address where the collection is sent
--new_signing_key : (bool) generate a new signing key, will invalidate all previous publishes
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
--replace : (bool) instead of modifying specific values on
@@ -3867,8 +3950,6 @@ class Daemon(metaclass=JSONRPCServerType):
)
new_txo = tx.outputs[0]
new_txo.script.generate()
if channel:
new_txo.sign(channel)
await tx.sign(funding_accounts)
@@ -3906,15 +3987,18 @@ class Daemon(metaclass=JSONRPCServerType):
return await self.jsonrpc_stream_abandon(*args, **kwargs)
@requires(WALLET_COMPONENT)
def jsonrpc_collection_list(self, resolve_claims=0, account_id=None, wallet_id=None, page=None, page_size=None):
def jsonrpc_collection_list(
self, resolve_claims=0, resolve=False, account_id=None,
wallet_id=None, page=None, page_size=None):
"""
List my collection claims.
Usage:
collection_list [--resolve_claims=<resolve_claims>] [<account_id> | --account_id=<account_id>]
collection_list [--resolve_claims=<resolve_claims>] [--resolve] [<account_id> | --account_id=<account_id>]
[--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>]
Options:
--resolve : (bool) resolve collection claim
--resolve_claims=<resolve_claims> : (int) resolve every claim
--account_id=<account_id> : (str) id of the account to use
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
@@ -3931,7 +4015,10 @@ class Daemon(metaclass=JSONRPCServerType):
else:
collections = partial(self.ledger.get_collections, wallet=wallet, accounts=wallet.accounts)
collection_count = partial(self.ledger.get_collection_count, wallet=wallet, accounts=wallet.accounts)
return paginate_rows(collections, collection_count, page, page_size, resolve_claims=resolve_claims)
return paginate_rows(
collections, collection_count, page, page_size,
resolve=resolve, resolve_claims=resolve_claims
)
async def jsonrpc_collection_resolve(
self, claim_id=None, url=None, wallet_id=None, page=1, page_size=DEFAULT_PAGE_SIZE):
@@ -4263,8 +4350,10 @@ class Daemon(metaclass=JSONRPCServerType):
@staticmethod
def _constrain_txo_from_kwargs(
constraints, type=None, txid=None, # pylint: disable=redefined-builtin
claim_id=None, channel_id=None, name=None, reposted_claim_id=None,
claim_id=None, channel_id=None, not_channel_id=None,
name=None, reposted_claim_id=None,
is_spent=False, is_not_spent=False,
has_source=None, has_no_source=None,
is_my_input_or_output=None, exclude_internal_transfers=False,
is_my_output=None, is_not_my_output=None,
is_my_input=None, is_not_my_input=None):
@@ -4272,6 +4361,10 @@ class Daemon(metaclass=JSONRPCServerType):
constraints['is_spent'] = True
elif is_not_spent:
constraints['is_spent'] = False
if has_source:
constraints['has_source'] = True
elif has_no_source:
constraints['has_source'] = False
constraints['exclude_internal_transfers'] = exclude_internal_transfers
if is_my_input_or_output is True:
constraints['is_my_input_or_output'] = True
@@ -4286,6 +4379,7 @@ class Daemon(metaclass=JSONRPCServerType):
constraints['is_my_output'] = False
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
database.constrain_single_or_list(constraints, 'channel_id', channel_id)
database.constrain_single_or_list(constraints, 'channel_id', not_channel_id, negate=True)
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
database.constrain_single_or_list(constraints, 'claim_name', name)
database.constrain_single_or_list(constraints, 'txid', txid)
@@ -4300,9 +4394,9 @@ class Daemon(metaclass=JSONRPCServerType):
List my transaction outputs.
Usage:
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
[--is_spent | --is_not_spent]
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
[--name=<name>...] [--is_spent | --is_not_spent]
[--is_my_input_or_output |
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
]
@@ -4316,6 +4410,7 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str or list) transaction id of outputs
--claim_id=<claim_id> : (str or list) claim id
--channel_id=<channel_id> : (str or list) claims in this channel
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
--name=<name> : (str or list) claim name
--is_spent : (bool) only show spent txos
--is_not_spent : (bool) only show not spent txos
@@ -4330,7 +4425,7 @@ class Daemon(metaclass=JSONRPCServerType):
"--is_my_input --is_my_output --type=other"
this allows to exclude "change" payments, this
flag can be used in combination with any of the other flags
--include_received_tips : (bool) calculate the amount of tips recieved for claim outputs
--include_received_tips : (bool) calculate the amount of tips received for claim outputs
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--page=<page> : (int) page to return during paginating
@@ -4369,15 +4464,15 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT)
async def jsonrpc_txo_spend(
self, account_id=None, wallet_id=None, batch_size=500,
self, account_id=None, wallet_id=None, batch_size=100,
include_full_tx=False, preview=False, blocking=False, **kwargs):
"""
Spend transaction outputs, batching into multiple transactions as necessary.
Usage:
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
[--is_my_input | --is_not_my_input]
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
[--name=<name>...] [--is_my_input | --is_not_my_input]
[--exclude_internal_transfers] [--wallet_id=<wallet_id>]
[--preview] [--blocking] [--batch_size=<batch_size>] [--include_full_tx]
@@ -4387,6 +4482,7 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str or list) transaction id of outputs
--claim_id=<claim_id> : (str or list) claim id
--channel_id=<channel_id> : (str or list) claims in this channel
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
--name=<name> : (str or list) claim name
--is_my_input : (bool) show outputs created by you
--is_not_my_input : (bool) show outputs not created by you
@@ -4407,7 +4503,10 @@ class Daemon(metaclass=JSONRPCServerType):
accounts = [wallet.get_account_or_error(account_id)] if account_id else wallet.accounts
txos = await self.ledger.get_txos(
wallet=wallet, accounts=accounts, read_only=True,
**self._constrain_txo_from_kwargs({}, is_not_spent=True, is_my_output=True, **kwargs)
no_tx=True, no_channel_info=True,
**self._constrain_txo_from_kwargs(
{}, is_not_spent=True, is_my_output=True, **kwargs
)
)
txs = []
while txos:
@@ -4431,6 +4530,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
[--claim_id=<claim_id>...] [--name=<name>...]
[--is_spent] [--is_not_spent]
[--is_my_input_or_output |
@@ -4444,6 +4544,8 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str or list) transaction id of outputs
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--channel_id=<channel_id> : (str or list) claims in this channel
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
--is_spent : (bool) only show spent txos
--is_not_spent : (bool) only show not spent txos
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
@@ -4478,6 +4580,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent]
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
[--is_my_input_or_output |
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
]
@@ -4492,6 +4595,8 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str or list) transaction id of outputs
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--channel_id=<channel_id> : (str or list) claims in this channel
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
--is_spent : (bool) only show spent txos
--is_not_spent : (bool) only show not spent txos
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
@@ -5041,8 +5146,9 @@ class Daemon(metaclass=JSONRPCServerType):
if hidden ^ visible:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_hidden_comments',
'comment.List',
claim_id=claim_id,
visible=visible,
hidden=hidden,
page=page,
page_size=page_size
@@ -5050,7 +5156,7 @@ class Daemon(metaclass=JSONRPCServerType):
else:
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_claim_comments',
'comment.List',
claim_id=claim_id,
parent_id=parent_id,
page=page,
@@ -5058,15 +5164,11 @@ class Daemon(metaclass=JSONRPCServerType):
top_level=not include_replies
)
if not skip_validation:
channel_urls = []
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if channel_url:
channel_urls.append(channel_url)
resolve_response = await self.resolve([], channel_urls)
for comment in result.get('items', []):
channel_url = comment.get('channel_url')
if not channel_url:
continue
resolve_response = await self.resolve([], [channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
@@ -5087,7 +5189,7 @@ class Daemon(metaclass=JSONRPCServerType):
Usage:
comment_create (<comment> | --comment=<comment>)
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
(<claim_id> | --claim_id=<claim_id>) [--parent_id=<parent_id>]
(--channel_id=<channel_id> | --channel_name=<channel_name>)
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
@@ -5106,8 +5208,10 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"is_pinned": (boolean) Channel owner has pinned this comment,
"signature": (str) The signature of the comment,
"signing_ts": (str) The timestamp used to sign the comment,
"channel_url": (str) Channel's URI in the ClaimTrie,
@@ -5129,7 +5233,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Create', comment_body)
response.update({
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
})
@@ -5155,18 +5259,20 @@ class Daemon(metaclass=JSONRPCServerType):
{
"comment": (str) The actual string as inputted by the user,
"comment_id": (str) The Comment's unique identifier,
"claim_id": (str) The claim commented on,
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
"channel_id": (str) The Channel Claim ID that this comment was posted under,
"signature": (str) The signature of the comment,
"signing_ts": (str) Timestamp used to sign the most recent signature,
"channel_url": (str) Channel's URI in the ClaimTrie,
"is_pinned": (boolean) Channel owner has pinned this comment,
"parent_id": (str) Comment this is replying to, (None) if this is the root,
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
}
"""
channel = await comment_client.jsonrpc_post(
self.conf.comment_server,
'get_channel_from_comment_id',
'comment.GetChannelFromCommentID',
comment_id=comment_id
)
if 'error' in channel:
@@ -5183,7 +5289,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
comment_client.sign_comment(edited_comment, channel_claim)
return await comment_client.jsonrpc_post(
self.conf.comment_server, 'edit_comment', edited_comment
self.conf.comment_server, 'comment.Edit', edited_comment
)
@requires(WALLET_COMPONENT)
@@ -5209,7 +5315,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
self.conf.comment_server, 'comment.GetChannelFromCommentID', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'abandoned': False}}
@@ -5218,8 +5324,8 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'abandon_comment', abandon_comment_body)
comment_client.sign_comment(abandon_comment_body, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Abandon', abandon_comment_body)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_hide(self, comment_ids: typing.Union[str, list], wallet_id=None):
@@ -5263,9 +5369,159 @@ class Daemon(metaclass=JSONRPCServerType):
for_signing=True
)
piece = {'comment_id': comment['comment_id']}
comment_client.sign_comment(piece, channel, abandon=True)
comment_client.sign_comment(piece, channel, sign_comment_id=True)
pieces.append(piece)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'hide_comments', pieces=pieces)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Hide', pieces=pieces)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_pin(self, comment_id=None, channel_id=None, channel_name=None,
channel_account_id=None, remove=False, wallet_id=None):
"""
Pin a comment published to a claim you control.
Usage:
comment_pin (<comment_id> | --comment_id=<comment_id>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
[--remove]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_id=<comment_id> : (str) Hash identifying the comment to pin
--channel_id=<claim_id> : (str) The ID of channel owning the commented claim
--channel_name=<claim_name> : (str) The name of channel owning the commented claim
--remove : (bool) remove the pin
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--wallet_id=<wallet_id : (str) restrict operation to specific wallet
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
comment_pin_args = {
'comment_id': comment_id,
'channel_name': channel_name,
'channel_id': channel_id,
'remove': remove,
}
comment_client.sign_comment(comment_pin_args, channel, sign_comment_id=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'comment.Pin', comment_pin_args)
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, remove=False, clear_types=None, react_type=None, wallet_id=None):
"""
Create and associate a reaction emoji with a comment using your channel identity.
Usage:
comment_react (--comment_ids=<comment_ids>)
(--channel_id=<channel_id>)
(--channel_name=<channel_name>)
(--react_type=<react_type>)
[(--remove) | (--clear_types=<clear_types>)]
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
Options:
--comment_ids=<comment_ids> : (str) one or more comment id reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--channel_account_id=<channel_account_id> : (str) one or more account ids for accounts to look in
--react_type=<react_type> : (str) name of reaction type
--remove : (bool) remove specified react_type
--clear_types=<clear_types> : (str) types to clear when adding another type
Returns:
(dict) Reaction object if successfully made, (None) otherwise
{
"Reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction
...
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_body = {
'comment_ids': comment_ids,
'channel_id': channel_id,
'channel_name': channel.claim_name,
'type': react_type,
'remove': remove,
'clear_types': clear_types,
}
comment_client.sign_reaction(react_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.React', react_body)
return response
@requires(WALLET_COMPONENT)
async def jsonrpc_comment_react_list(
self, comment_ids, channel_name=None, channel_id=None,
channel_account_id=None, react_types=None, wallet_id=None):
"""
List reactions emoji with a claim using your channel identity.
Usage:
comment_react_list (--comment_ids=<comment_ids>)
[(--channel_id=<channel_id>)(--channel_name=<channel_name>)]
[--react_types=<react_types>]
Options:
--comment_ids=<comment_ids> : (str) The comment ids reacted to, comma delimited
--channel_id=<claim_id> : (str) The ID of channel reacting
--channel_name=<claim_name> : (str) The name of the channel reacting
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
--react_types=<react_type> : (str) comma delimited reaction types
Returns:
(dict) Comment object if successfully made, (None) otherwise
{
"my_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
"other_reactions": {
<comment_id>: {
<reaction_type>: (int) Count for this reaction type
...
}
}
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
react_list_body = {
'comment_ids': comment_ids,
}
if channel_id:
channel = await self.get_channel_or_error(
wallet, channel_account_id, channel_id, channel_name, for_signing=True
)
react_list_body['channel_id'] = channel_id
react_list_body['channel_name'] = channel.claim_name
if react_types:
react_list_body['types'] = react_types
if channel_id:
comment_client.sign_reaction(react_list_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'reaction.List', react_list_body)
return response
async def broadcast_or_release(self, tx, blocking=False):
await self.wallet_manager.broadcast_or_release(tx, blocking)
+109 -57
View File
@@ -2,6 +2,7 @@ import json
import time
import asyncio
import logging
from statistics import median
from decimal import Decimal
from typing import Optional, Iterable, Type
from aiohttp.client_exceptions import ContentTypeError
@@ -58,9 +59,12 @@ class MarketFeed:
raise NotImplementedError()
async def get_response(self):
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
async with aiohttp_request(
'get', self.url, params=self.params,
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
) as response:
try:
self._last_response = await response.json()
self._last_response = await response.json(content_type=None)
except ContentTypeError as e:
self._last_response = {}
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
@@ -104,70 +108,115 @@ class MarketFeed:
self.event.clear()
class BittrexFeed(MarketFeed):
class BaseBittrexFeed(MarketFeed):
name = "Bittrex"
market = "BTCLBC"
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
params = {'market': 'BTC-LBC', 'count': 50}
market = None
url = None
fee = 0.0025
def get_rate_from_response(self, json_response):
if 'result' not in json_response:
if 'lastTradeRate' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
trades = json_response['result']
if len(trades) == 0:
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
totals = sum([i['Total'] for i in trades])
qtys = sum([i['Quantity'] for i in trades])
if totals <= 0 or qtys <= 0:
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
vwap = totals / qtys
return float(1.0 / vwap)
return 1.0 / float(json_response['lastTradeRate'])
class LBRYFeed(MarketFeed):
name = "lbry.com"
class BittrexBTCFeed(BaseBittrexFeed):
market = "BTCLBC"
url = "https://api.lbry.com/lbc/exchange_rate"
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
class BittrexUSDFeed(BaseBittrexFeed):
market = "USDLBC"
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
class BaseCryptonatorFeed(MarketFeed):
name = "Cryptonator"
market = None
url = None
def get_rate_from_response(self, json_response):
if 'data' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / json_response['data']['lbc_btc']
class LBRYBTCFeed(LBRYFeed):
market = "USDBTC"
def get_rate_from_response(self, json_response):
if 'data' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / json_response['data']['btc_usd']
class CryptonatorFeed(MarketFeed):
name = "cryptonator.com"
market = "BTCLBC"
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
def get_rate_from_response(self, json_response):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
'success' not in json_response or json_response['success'] is not True:
if 'ticker' not in json_response or 'price' not in json_response['ticker']:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return float(json_response['ticker']['price'])
class CryptonatorBTCFeed(CryptonatorFeed):
market = "USDBTC"
url = "https://api.cryptonator.com/api/ticker/usd-btc"
class CryptonatorBTCFeed(BaseCryptonatorFeed):
market = "BTCLBC"
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
class CryptonatorUSDFeed(BaseCryptonatorFeed):
market = "USDLBC"
url = "https://api.cryptonator.com/api/ticker/usd-lbc"
class BaseCoinExFeed(MarketFeed):
name = "CoinEx"
market = None
url = None
def get_rate_from_response(self, json_response):
if 'data' not in json_response or \
'ticker' not in json_response['data'] or \
'last' not in json_response['data']['ticker']:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / float(json_response['data']['ticker']['last'])
class CoinExBTCFeed(BaseCoinExFeed):
market = "BTCLBC"
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
class CoinExUSDFeed(BaseCoinExFeed):
market = "USDLBC"
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
class BaseHotbitFeed(MarketFeed):
name = "hotbit"
market = None
url = "https://api.hotbit.io/api/v1/market.last"
def get_rate_from_response(self, json_response):
if 'result' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / float(json_response['result'])
class HotbitBTCFeed(BaseHotbitFeed):
market = "BTCLBC"
params = {"market": "LBC/BTC"}
class HotbitUSDFeed(BaseHotbitFeed):
market = "USDLBC"
params = {"market": "LBC/USDT"}
class UPbitBTCFeed(MarketFeed):
name = "UPbit"
market = "BTCLBC"
url = "https://api.upbit.com/v1/ticker"
params = {"markets": "BTC-LBC"}
def get_rate_from_response(self, json_response):
if len(json_response) != 1 or 'trade_price' not in json_response[0]:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / float(json_response[0]['trade_price'])
FEEDS: Iterable[Type[MarketFeed]] = (
LBRYFeed,
LBRYBTCFeed,
BittrexFeed,
# CryptonatorFeed,
# CryptonatorBTCFeed,
BittrexBTCFeed,
BittrexUSDFeed,
CryptonatorBTCFeed,
CryptonatorUSDFeed,
CoinExBTCFeed,
CoinExUSDFeed,
HotbitBTCFeed,
HotbitUSDFeed,
UPbitBTCFeed,
)
@@ -191,20 +240,23 @@ class ExchangeRateManager:
source.stop()
def convert_currency(self, from_currency, to_currency, amount):
rates = [market.rate for market in self.market_feeds]
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
log.debug(
"Converting %f %s to %s, rates: %s",
amount, from_currency, to_currency,
[market.rate for market in self.market_feeds]
)
if from_currency == to_currency:
return round(amount, 8)
rates = []
for market in self.market_feeds:
if (market.has_rate and market.is_online and
market.rate.currency_pair == (from_currency, to_currency)):
return round(amount * Decimal(market.rate.spot), 8)
for market in self.market_feeds:
if (market.has_rate and market.is_online and
market.rate.currency_pair[0] == from_currency):
return round(self.convert_currency(
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
rates.append(market.rate.spot)
if rates:
return round(amount * Decimal(median(rates)), 8)
raise CurrencyConversionError(
f'Unable to convert {amount} from {from_currency} to {to_currency}')
+16 -4
View File
@@ -85,8 +85,11 @@ class FileManager:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(
self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
resolve_timeout
self.wallet_manager.ledger.resolve(
wallet.accounts, [uri],
include_purchase_receipt=True,
include_is_my_output=True
), resolve_timeout
)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
@@ -114,9 +117,11 @@ class FileManager:
if claim.stream.source.bt_infohash:
source_manager = self.source_managers['torrent']
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
else:
elif claim.stream.source.sd_hash:
source_manager = self.source_managers['stream']
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
else:
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
# resume or update an existing stream, if the stream changed: download it and delete the old one after
to_replace, updated_stream = None, None
@@ -170,7 +175,14 @@ class FileManager:
# pay fee
####################
if not to_replace and txo.has_price and not txo.purchase_receipt:
needs_purchasing = (
not to_replace and
not txo.is_my_output and
txo.has_price and
not txo.purchase_receipt
)
if needs_purchasing:
payment = await self.wallet_manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
+37 -1
View File
@@ -1,14 +1,44 @@
import time
import logging
import asyncio
import asyncio.tasks
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
from prometheus_client import Counter, Histogram, Gauge
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
def get_loop_metrics(delay=1):
loop = asyncio.get_event_loop()
def callback(started):
PROBE_TIMES.observe(time.perf_counter() - started - delay)
PROBES_FINISHED.inc()
async def monitor_loop_responsiveness():
while True:
now = time.perf_counter()
loop.call_later(delay, callback, now)
PROBES_IN_FLIGHT.inc()
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
await asyncio.sleep(delay)
return loop.create_task(monitor_loop_responsiveness())
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
self._monitor_loop_task = None
async def start(self, interface: str, port: int):
self.logger.info("start prometheus metrics")
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
@@ -16,7 +46,10 @@ class PrometheusServer:
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
self.logger.info(
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
)
self._monitor_loop_task = get_loop_metrics()
async def handle_metrics_get_request(self, request: web.Request):
try:
@@ -29,4 +62,7 @@ class PrometheusServer:
raise
async def stop(self):
if self._monitor_loop_task and not self._monitor_loop_task.done():
self._monitor_loop_task.cancel()
self._monitor_loop_task = None
await self.runner.cleanup()
+4
View File
@@ -303,6 +303,10 @@ class Stream(BaseClaim):
def has_fee(self) -> bool:
return self.message.HasField('fee')
@property
def has_source(self) -> bool:
return self.message.HasField('source')
@property
def source(self) -> Source:
return Source(self.message.source)
+5 -2
View File
@@ -46,8 +46,8 @@ types_map = {
'.ksh': ('text/plain', 'document'),
'.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'video'),
'.m3u': ('application/x-mpegurl', 'audio'),
'.m3u8': ('application/x-mpegurl', 'video'),
'.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'),
@@ -66,6 +66,7 @@ types_map = {
'.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'),
'.ms': ('application/x-troff-ms', 'binary'),
'.m4s': ('video/iso.segment', 'binary'),
'.nc': ('application/x-netcdf', 'binary'),
'.nws': ('message/rfc822', 'document'),
'.o': ('application/octet-stream', 'binary'),
@@ -119,10 +120,12 @@ types_map = {
'.tif': ('image/tiff', 'image'),
'.tiff': ('image/tiff', 'image'),
'.tr': ('application/x-troff', 'binary'),
'.ts': ('video/mp2t', 'video'),
'.tsv': ('text/tab-separated-values', 'document'),
'.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'),
'.vtt': ('text/vtt', 'document'),
'.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'),
+34 -38
View File
@@ -13,50 +13,45 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
def set_reference(reference, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
return
def set_reference(reference, txo_row):
if txo_row:
reference.tx_hash = txo_row['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo_row['txo_hash'][32:])[0]
reference.height = txo_row['height']
class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total'
NOT_CENSORED = 0
SEARCH = 1
RESOLVE = 2
def __init__(self, streams: dict = None, channels: dict = None):
self.streams = streams or {}
self.channels = channels or {}
__slots__ = 'censor_type', 'censored'
def __init__(self, censor_type):
self.censor_type = censor_type
self.censored = {}
self.total = 0
def is_censored(self, row):
return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
def apply(self, rows):
return [row for row in rows if not self.censor(row)]
def censor(self, row) -> bool:
was_censored = False
for claim_hash, lookup in (
(row['claim_hash'], self.streams),
(row['claim_hash'], self.channels),
(row['channel_hash'], self.channels),
(row['reposted_claim_hash'], self.streams),
(row['reposted_claim_hash'], self.channels)):
censoring_channel_hash = lookup.get(claim_hash)
if censoring_channel_hash:
was_censored = True
self.censored.setdefault(censoring_channel_hash, 0)
self.censored[censoring_channel_hash] += 1
break
if was_censored:
self.total += 1
return was_censored
if self.is_censored(row):
censoring_channel_hash = row['censoring_channel_hash']
self.censored.setdefault(censoring_channel_hash, set())
self.censored[censoring_channel_hash].add(row['tx_hash'])
return True
return False
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
outputs.blocked_total = self.total
def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
for censoring_channel_hash, count in self.censored.items():
blocked = outputs.blocked.add()
blocked.count = count
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
blocked.count = len(count)
set_reference(blocked.channel, extra_txo_rows.get(censoring_channel_hash))
outputs.blocked_total += len(count)
class Outputs:
@@ -161,6 +156,7 @@ class Outputs:
@classmethod
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
extra_txo_rows = {row['claim_hash']: row for row in extra_txo_rows}
page = OutputsMessage()
page.offset = offset
if total is not None:
@@ -169,12 +165,12 @@ class Outputs:
blocked.to_message(page, extra_txo_rows)
for row in txo_rows:
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
for row in extra_txo_rows:
for row in extra_txo_rows.values():
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
return page.SerializeToString()
@classmethod
def row_to_message(cls, txo, txo_message, extra_txo_rows):
def row_to_message(cls, txo, txo_message, extra_row_dict: dict):
if isinstance(txo, Exception):
txo_message.error.text = txo.args[0]
if isinstance(txo, ValueError):
@@ -183,7 +179,7 @@ class Outputs:
txo_message.error.code = ErrorMessage.NOT_FOUND
elif isinstance(txo, ResolveCensoredError):
txo_message.error.code = ErrorMessage.BLOCKED
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(txo.censor_hash))
return
txo_message.tx_hash = txo['txo_hash'][:32]
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
@@ -206,5 +202,5 @@ class Outputs:
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
set_reference(txo_message.claim.channel, extra_row_dict.get(txo['channel_hash']))
set_reference(txo_message.claim.repost, extra_row_dict.get(txo['reposted_claim_hash']))
+10 -9
View File
@@ -22,8 +22,7 @@ def _create_url_regex():
return _group(
_named(name+"_name", prefix + invalid_names_regex) +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
@@ -50,28 +49,31 @@ def normalize_name(name):
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
def normalized(self):
return normalize_name(self.name)
@property
def is_shortid(self):
return self.claim_id is not None and len(self.claim_id) < 40
@property
def is_fullid(self):
return self.claim_id is not None and len(self.claim_id) == 40
def to_dict(self):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
def __str__(self):
if self.claim_id is not None:
return f"{self.name}#{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
return f"{self.name}:{self.claim_id}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
@@ -118,7 +120,6 @@ class URL(NamedTuple):
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
+2 -2
View File
@@ -183,11 +183,11 @@ class StreamDescriptor:
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
if decoded['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in decoded['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
descriptor = cls(
loop, blob_dir,
+1 -1
View File
@@ -40,7 +40,7 @@ class StreamDownloader:
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2)
if self.blob_manager.decrypted_blob_lru_cache:
if self.blob_manager.decrypted_blob_lru_cache is not None:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob
)
+12 -3
View File
@@ -354,15 +354,24 @@ class ManagedStream(ManagedDownloadSource):
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
except ConnectionError:
return sent
except OSError:
# raised if a blob is deleted while it's being sent
except (OSError, Exception) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
elif isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
return sent
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+22 -6
View File
@@ -64,12 +64,12 @@ class StreamReflectorClient(asyncio.Protocol):
self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request
except (AttributeError, asyncio.CancelledError):
except (AttributeError, asyncio.CancelledError) as err:
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise asyncio.TimeoutError()
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
finally:
self.pending_request = None
@@ -94,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
needed = response.get('needed_blobs', [])
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await sd_blob.sendfile(self)
if sent == -1:
log.warning("failed to send sd blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -118,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await asyncio.wait_for(self.response_queue.get(), 30)
try:
sent = await blob.sendfile(self)
if sent == -1:
log.warning("failed to send blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+2 -2
View File
@@ -160,13 +160,13 @@ class StreamManager(SourceManager):
async def reflect_streams(self):
try:
return await self._reflact_streams()
return await self._reflect_streams()
except asyncio.CancelledError:
raise
except Exception:
log.exception("reflector task encountered an unexpected error!")
async def _reflact_streams(self):
async def _reflect_streams(self):
# todo: those debug statements are temporary for #2987 - remove them if its closed
while True:
if self.config.reflect_streams and self.config.reflector_servers:
+24 -8
View File
@@ -23,12 +23,13 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
from lbry.extras.daemon.components import Component, WalletComponent
from lbry.extras.daemon.components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
)
from lbry.extras.daemon.componentmanager import ComponentManager
from lbry.extras.daemon.exchange_rate_manager import (
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
)
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.blob.blob_manager import BlobManager
@@ -83,6 +84,7 @@ class AsyncioTestCase(unittest.TestCase):
# https://bugs.python.org/issue32972
LOOP_SLOW_CALLBACK_DURATION = 0.2
TIMEOUT = 120.0
maxDiff = None
@@ -136,6 +138,8 @@ class AsyncioTestCase(unittest.TestCase):
with outcome.testPartExecutor(self, isTest=True):
maybe_coroutine = testMethod()
if asyncio.iscoroutine(maybe_coroutine):
if self.TIMEOUT:
self.loop.call_later(self.TIMEOUT, self.cancel)
self.loop.run_until_complete(maybe_coroutine)
outcome.expecting_failure = False
with outcome.testPartExecutor(self):
@@ -188,6 +192,12 @@ class AsyncioTestCase(unittest.TestCase):
if asyncio.iscoroutine(maybe_coroutine):
self.loop.run_until_complete(maybe_coroutine)
def cancel(self):
for task in asyncio.all_tasks(self.loop):
if not task.done():
task.print_stack()
task.cancel()
class AdvanceTimeTestCase(AsyncioTestCase):
@@ -281,8 +291,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
def get_fake_exchange_rate_manager(rates=None):
return FakeExchangeRateManager(
[LBRYFeed(), LBRYBTCFeed()],
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
[BittrexBTCFeed(), BittrexUSDFeed()],
rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
)
@@ -320,6 +330,7 @@ class CommandTestCase(IntegrationTestCase):
self.server_blob_manager = None
self.server = None
self.reflector = None
self.skip_libtorrent = True
async def asyncSetUp(self):
await super().asyncSetUp()
@@ -340,6 +351,7 @@ class CommandTestCase(IntegrationTestCase):
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_config = Config()
self.server_config.transaction_cache_size = 10000
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
await self.server_storage.open()
@@ -389,10 +401,13 @@ class CommandTestCase(IntegrationTestCase):
conf.fixed_peers = [('127.0.0.1', 5567)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.transaction_cache_size = 10000
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
]
if self.skip_libtorrent:
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
wallet_node.manager.config = conf
def wallet_maker(component_manager):
@@ -404,7 +419,7 @@ class CommandTestCase(IntegrationTestCase):
daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
'BTCLBC': 1.0, 'USDBTC': 2.0
'BTCLBC': 1.0, 'USDLBC': 2.0
})
))
await daemon.initialize()
@@ -433,8 +448,9 @@ class CommandTestCase(IntegrationTestCase):
async def generate(self, blocks):
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
@@ -474,7 +490,7 @@ class CommandTestCase(IntegrationTestCase):
async def stream_create(
self, name='hovercraft', bid='1.0', file_path=None,
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
if file_path is None:
if file_path is None and data is not None:
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
return await self.confirm_and_render(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
+186 -15
View File
@@ -20,8 +20,11 @@ import pkg_resources
import certifi
import aiohttp
from prometheus_client import Counter
from prometheus_client.registry import REGISTRY
from lbry.schema.claim import Claim
log = logging.getLogger(__name__)
@@ -102,10 +105,6 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
return False
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
@@ -189,6 +188,8 @@ def cache_concurrent(async_fn):
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
if url.lower() == 'localhost':
return '127.0.0.1'
try:
if ipaddress.ip_address(url):
return url
@@ -203,18 +204,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
))[0][4][0]
class LRUCache:
class LRUCacheWithMetrics:
__slots__ = [
'capacity',
'cache'
'cache',
'_track_metrics',
'hits',
'misses'
]
def __init__(self, capacity):
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
self.capacity = capacity
self.cache = collections.OrderedDict()
if metric_name is None:
self._track_metrics = False
self.hits = self.misses = None
else:
self._track_metrics = True
try:
self.hits = Counter(
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
)
self.misses = Counter(
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
)
except ValueError as err:
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
self._track_metrics = False
self.hits = self.misses = None
def get(self, key):
value = self.cache.pop(key)
def get(self, key, default=None):
try:
value = self.cache.pop(key)
if self._track_metrics:
self.hits.inc()
except KeyError:
if self._track_metrics:
self.misses.inc()
return default
self.cache[key] = value
return value
@@ -226,16 +253,94 @@ class LRUCache:
self.cache.popitem(last=False)
self.cache[key] = value
def clear(self):
self.cache.clear()
def pop(self, key):
return self.cache.pop(key)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool:
return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
if self._track_metrics: # needed for tests
try:
REGISTRY.unregister(self.hits)
REGISTRY.unregister(self.misses)
except AttributeError:
pass
class LRUCache:
__slots__ = [
'capacity',
'cache'
]
def __init__(self, capacity: int):
self.capacity = capacity
self.cache = collections.OrderedDict()
def get(self, key, default=None):
try:
value = self.cache.pop(key)
except KeyError:
return default
self.cache[key] = value
return value
def set(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def clear(self):
self.cache.clear()
def pop(self, key):
return self.cache.pop(key)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool:
return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = override_lru_cache or LRUCache(cache_size)
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
def wrapper(async_fn):
@@ -266,18 +371,84 @@ def get_ssl_context() -> ssl.SSLContext:
@contextlib.asynccontextmanager
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
async with aiohttp.ClientSession() as session:
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
async with session.request(method, url, **kwargs) as response:
yield response
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
def is_valid_public_ipv4(address, allow_localhost: bool = False):
try:
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
try:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
return response['data']['ip'], None
except Exception:
return
return None, None
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
# used if upnp is disabled or non-functioning
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
hostname_to_ip = {}
ip_to_hostnames = collections.defaultdict(list)
async def resolve_spv(server, port):
try:
server_addr = await resolve_host(server, port, 'udp')
hostname_to_ip[server] = (server_addr, port)
ip_to_hostnames[(server_addr, port)].append(server)
except Exception:
log.exception("error looking up dns for spv servers")
# accumulate the dns results
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
loop = asyncio.get_event_loop()
pong_responses = asyncio.Queue()
connection = SPVStatusClientProtocol(pong_responses)
try:
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
# could raise OSError if it cant bind
randomized_servers = list(ip_to_hostnames.keys())
random.shuffle(randomized_servers)
for server in randomized_servers:
connection.ping(server)
try:
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
if is_valid_public_ipv4(pong.ip_address):
return pong.ip_address, ip_to_hostnames[server][0]
except asyncio.TimeoutError:
pass
return None, None
finally:
connection.close()
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
ip_from_spv_servers = await _get_external_ip(default_servers)
if not ip_from_spv_servers[1]:
return await fallback_get_external_ip()
return ip_from_spv_servers
def is_running_from_bundle():
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+9 -31
View File
@@ -5,7 +5,6 @@ import logging
import typing
import asyncio
import random
from functools import partial
from hashlib import sha256
from string import hexdigits
from typing import Type, Dict, Tuple, Optional, Any, List
@@ -16,7 +15,7 @@ from lbry.crypto.crypt import aes_encrypt, aes_decrypt
from .bip32 import PrivateKey, PubKey, from_extended_key_string
from .mnemonic import Mnemonic
from .constants import COIN, CLAIM_TYPES, TXO_TYPES
from .constants import COIN, TXO_TYPES
from .transaction import Transaction, Input, Output
if typing.TYPE_CHECKING:
@@ -566,35 +565,14 @@ class Account:
if gap_changed:
self.wallet.save()
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False, read_only=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0
get_total_balance = partial(self.get_balance, read_only=read_only, confirmations=confirmations,
include_claims=True)
total = await get_total_balance()
if reserved_subtotals:
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
for txo in await self.get_support_summary():
if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1):
continue
if txo.is_my_input:
supports_balance += txo.amount
else:
tips_balance += txo.amount
reserved = claims_balance + supports_balance + tips_balance
else:
reserved = await self.get_balance(
confirmations=confirmations, include_claims=True, txo_type__gt=0
)
return {
'total': total,
'available': total - reserved,
'reserved': reserved,
'reserved_subtotals': {
'claims': claims_balance,
'supports': supports_balance,
'tips': tips_balance
} if reserved_subtotals else None
}
async def get_detailed_balance(self, confirmations=0, read_only=False):
constraints = {}
if confirmations > 0:
height = self.ledger.headers.height - (confirmations-1)
constraints.update({'height__lte': height, 'height__gt': 0})
return await self.ledger.db.get_detailed_balance(
accounts=[self], read_only=read_only, **constraints
)
def get_transaction_history(self, read_only=False, **constraints):
return self.ledger.get_transaction_history(
+141
View File
@@ -740,4 +740,145 @@ HASHES = {
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
751000: 'cedaf12415dac1314942e58ced80830b92fbfabc41f42a0b0f054f0672ef9822',
752000: '293606bcd9fbbee5584724301b2cf86bb69204820023e1fb46c238ddfbc660ab',
753000: 'f4d43cbb38b7d97919dedc0f5a6dc8007896c4f443b76f3e5693e25bc46760cf',
754000: 'fcaad22fd815311280fe451086516375d1d9d92b2990c7c351407df5aa19011e',
755000: 'b9276f10d1844cb5b0308766c8db960490ac34a73c4653d0a91202789a6ccb9b',
756000: '2fe5581f1110c1c8dcea46cad647551bd6bd640cb37738d863e189bd8f368347',
757000: 'b9d915f366f0b010429a52245b0fb02774157eb9fd8f66bce32dcd3acc71c2a1',
758000: '62d1854fc15db56b5d0e05ceeb54c1297966bf9dc7f7a0a14b42c059fc485d1b',
759000: 'f4ca9f69d16d092f4a0ea5102e6343b21204c4ea9cd9b22cddd77dbb5d68ade3',
760000: 'df3bb86641330d8cc7f55a2fd0da28251219e95babe960a308b18e08a7d88fc8',
761000: 'a93029475de4bc7569b6ae802d658cd91c84cc253772712a279f140a6c3b91b1',
762000: '307e289dc6ec8bcd62ca8831e4159d5edd780f2fae55ba55dd446225450f46f8',
763000: '293f73514abca24f374473bd0394179812952a04ea13dc60ef5ada5331fa274f',
764000: 'dd8b082db9281e3d9bacf15d6b352fda186d2d2923c7731844d0d4764dd71db8',
765000: '201239e562d2571bf47347b3522fff89632aecea3b2d8cef05151f88b2b0bcdb',
766000: '4a55a538b51b5650979e64521998cd5c5ad055ba9f3ac0e3e2a28febc6cc2798',
767000: '3916666f2adbb05ea98ec1961f9546b9afa0f6910ec95e42ce37267f2ae4f79c',
768000: 'dc0ad881eedcb5fd4954238f462080d6e7636b058d481698ed1c077e0ce2207e',
769000: 'eaf10a1e1ec6e129289b8479a05df03e0808f1f0946f1995de6524e9ebe7a461',
770000: '7200c64f22e32de7f999583361c933680fc9a2ffcb9a5ab73d3076fd49ec7537',
771000: 'd883111a2eeacff80ce31df35ab6c943805b9e48877b413fccf371e5dbfa7fb2',
772000: '3977d3c60edb9c80c97bb2b759b1659cbb650ad2d3a6f61d2caec83f1b2ae84c',
773000: '9c7175fb8646a1a82383b4c534fd01bcf92d65c43d87ae854d51a784b04dc77e',
774000: 'e0e92485f86e5fffa87b3497424e43b02a37710517d9d3f272392e8cdc56e5e9',
775000: '6395229113d3aa2105afbaeb8b59621a536fc61fe272314b2fc3bdda98dd66cc',
776000: 'b4b00207328b5f032bd4f0b634f91323ff520ada8c8bfec241b23c8e4bfd5a4e',
777000: '14cdc6f5f7b4bd5bad745dfe6fcd114e9194026412a2e1b3f345be2eef433d16',
778000: 'd3cd7b68be504c32117b670d38d59d44b02dcf3d65811efc2ca5531d902623cc',
779000: 'afcd220e4040cb5f92d4b38fc204e59822df2218f767f2c4b33597b238a35f77',
780000: '78252a9cfc289a70192ed8dd3dddeb1b9a4f9b8eff9a5d0ac259b3254472cf68',
781000: '02ebc3f17d947481a311b4771c254f1e002b6a9198d4a5258ce6c13165aadddc',
782000: '8dd9f1f372ee6d688a0bcdc3b342c77804ba5a646a218be4bc2aa02d846206c0',
783000: 'e46b0d02ec2ef488fae455665e107520e1bd2b4f35ca52af7ad8addd2f72fa73',
784000: '9ee8a8de94231e3ae3a610b82fdbca48dc14d9b80791d20af6c365a31822df6f',
785000: '21e1cc12def8173a50158b2833bd91a62140c61646f5e08aecaee3e6da20735e',
786000: 'b3e659f84d73de42888cc0f2b69bae71dd5fa6756a437a4b21958b182faa316e',
787000: 'a9be7ba00ea6a9ea6bd03d8412ec014ca7e8cda6bdc33382f165e702811b8836',
788000: 'a4c14729f8a68c03f5a0ccd890ac6a92b39c143f1f752fe81ad051eb52d8dce0',
789000: '5cf66d224e5645097efc9c3c0392b51c8ca8ea1295151921a7912a2f04ee1274',
790000: '676769ade71c33bc102bce416e66eb2c6794b03d7b8f5a590c87c380da463775',
791000: '0228e074451797bf6bfbc941bcafcbadc972d32e4e1e0c5da015513f65714217',
792000: '0fa3d00a1f19c5ac060e10a410cf7cea18eac5f89018d79ce51ac3fc66bbb365',
793000: '5f68d0868b424e32f5ce3d8e7d9f18979da7b831b8ef4e3974d62fb20ff53a97',
794000: '34508c56423739c00a837801b654b07decb274d02b383eff396d23c4d64bc0e9',
795000: '7f70910c855d1fd88cd7f9be8a3b94314ee408a31a2da6301404bf8deb07c12c',
796000: 'b74ab8813b1d2a0967fea0e66597572e5f0b5a285e21f5150fcc9d5f757de130',
797000: 'bba27b1491d907ab1baa456cb651dc5b071231b1b6ad27b62d351ca12c25dbfd',
798000: 'e75dcb15b2fc91f02e75e600dde9f6f46c09672533bc82a5d6916c4a2cd8613a',
799000: 'adf62c826a3e0b33af439a7881918ae4ce19c5fb2ca37d21243415f7d716aa65',
800000: 'd8f0ca13a8c8a19c254a3a6ba15150a34711dca96f2d877162cc44aa2acfb268',
801000: '2a8c7104c4040a2bc31913ae25e9361df5bac9477368c708f86c1ca640480887',
802000: '1f3b09d3561c4a8a056b263289bd492dc6c0d604c3fa195935e735d1c0ddc40e',
803000: '037769628c40a701fdb4b16d79084b8fbb319fde79770a7ac842f3cdc813099e',
804000: 'a0c6a089e5fa1e3589ca282085fe7201a5705776d81b257ffd252b2947fa6428',
805000: 'b2ac99bfc4a488e7b7624b31ee061991a6dd0881bb005cd13f3dd2e66a08fe19',
806000: 'ffe63cb999a278280b80a667d2dcb60c40e43a53f733914d8bec808b694ebf83',
807000: 'eddb09fc6c4869a59b520d0befb1fb6ac952333f3cc5de086539c85ea8558778',
808000: '0f4fb3f9172e52897ea992d9f3a2024126c4d2e63e9888739f11fb1f5e4c1f46',
809000: '9641dd720d23ced2f1cb6e5cf46ac4e547afb9f56263c4cf58e3b19d407cf401',
810000: 'de6dc953acd7e5ef213b3aaf1c4a9ee1d5b756bfce5525ee105214647e243a85',
811000: 'c52c83712ca12b24b2db1b4a575e7f352b1d560cbf702e121a03bdca9e8be23d',
812000: '83143734bb965318a53a38a7e403dcdb3e3fadedb01ab12c370417fc2a0655c0',
813000: 'e480deff10c5a84fc957e3aed936690e24b74dd08fa8858a8a953c2f7383b914',
814000: '810d33afcee07b9abe16c6cdc3a041038daa131c476b0daf48a080007f08b490',
815000: 'b4aeb9e16fddd27844b2d56bc2b221134039bb5642c9e9ba88372afbdeac3972',
816000: '86e73b67aae3d248011b8f66ed414cb8a9ba4b2a3cf7e32773cfbff055d719b7',
817000: '3ebb8b83752b48242016cb682f0f6bd14e15371bf1163a5933193eaa0edeb351',
818000: '4d925e17f642f220bbf317d3d5355d2f41fbce325f190f8c3b32dc0b337d24d6',
819000: 'b9cc126d620f6b99d90a00d35957b0e428aaaa7c986bc9e816a60e4334572961',
820000: '9c2f8c142bed1f94dca29276f7c83958be8cfe11773bb9b56c808fbcf7d3b1f8',
821000: 'e5509eb98895cfa12a8da5d54c1df3f52472ffcbdf707adbf84a4a9c5d356203',
822000: '764aada4802ebfe4ef935ab50af06a4f83aa556c49fdde3d9e12e1abd230c16b',
823000: '1dbd745c2e96a365d865f990d109137d32d42977f503af55d8c00b109d31d3c3',
824000: '954304a0b0c8f549c3bffd5ff46b5b8f05b0f0fde2a36f24fd5af9d774fb3079',
825000: '17808b14f2056c1a5d46cb7617e9de9be6a1a6084edbc1bdb778586467a72297',
826000: '3ca1167d4cac8b187829b23001b438617c43704b42462c4eb001b0d434cb9651',
827000: '246d1607245e4a202f420393ac2e30e9cbf5eb5570dc997073b897f6d8643023',
828000: '1764730a8dc3e89d02d168ff6bb54e8c903820b74711af6ff27bd0c8545577e7',
829000: 'd9f3ab0cd823c6305bd8b95a96188bb4f2ca90b4d66c5d12293e8b6192bac0f2',
830000: 'd4ff51f0092b04aedf8d39937680d8e8309b1be21d36e7833ed36f8e30aad6ea',
831000: '3e92e76721b962396dce52993fa7606552f0907b38f7b2bd7b21ada98c145f47',
832000: 'df12fcdb4cbe53ba627ace6de898298de175f8671d3d90170732d110fcdc34b8',
833000: '25167ff38ae4a5964b618cabe0a12d4de62ac7a4c47448cdb4499e09e108d5b9',
834000: 'd31f5309ea179a1e386e835fc372e47dcda6871a3a239abfba50c4f368994f13',
835000: 'aff7e8dd3e55ea807fcbe284014075f420b3a23f1b0eb47bacdc1c91d2899813',
836000: '3b5ac6d64c470739bb17d1544a285affb40f2d33e92687e5ba7c5ac602e0d72a',
837000: 'd5619cbfe4f27c55f2bf9351b4891636cf64fef88212a5eeeae7bd3de47fe0bd',
838000: '1f9102a49c6ac470cb5d0050e5300b1443840d6d65719b835e3bea484aafb2ec',
839000: '3f63e391f0fbc5787fbe4ace3bada3816261294ea1c6ee435001801023682f90',
840000: '777894fd12bd0d6dee7bcde2995c68e55e7094e3122da38571e4b6c4304b75e0',
841000: 'ceb0c598c788e25e43e25aa4beff5c7377035824844cf1675eaea537074df028',
842000: '8661cf2065dc713d2ba043f0b81f0effcc940eeb3e91906a21ff22c210561dcd',
843000: '0dc2766f90415009d0c86bedffee6ebcf58042eb08262c0c67c4e9ed86b2aec8',
844000: '26d072da864cab268a12794977b04ec44fb69ef3978e2342e82225974dac54dd',
845000: '95e93bb60be8d5f07a1f4d26290c914957a82fc9d26ae8a3f20082eda27406ff',
846000: 'f1bdc39af7705e58ab8b6c31dc70dce1e115db1cfd8cc9b037949dfbec82a59a',
847000: 'f5f10f06396ecf2765d8a081141d489737c1d8d57c281f28f57c4cb2f90db883',
848000: '331b8ef08605bae8d749893af9ed54f0df4f07a5a002108a2a0aea82d0360979',
849000: '75b5f6233ab9a1bbc3c8b2893e5b22a0aa98e7ea635261255dc3c281f67d2260',
850000: '5d7e6fe83e0ea1910a54a00090704737671d6f44df4228e21440ad1fc15e595f',
851000: '7822db25d3ff0f6695ee38bad91edf317b5c6611673d28f1d22053110bb558be',
852000: '2f0effad83a3561fc1a2806a562786a641d9ddb18d16bb9308006e7d324a21e9',
853000: 'f603b2eaff11d5296377d990651317d40a1b2599ad2c5250eab131090f4b9458',
854000: '34d59b26a50f18a9f250736d0f2e69d28b7e196fbef9b8a26c6b0b75c16aa194',
855000: '76dd1ffff3946c0878969886fcf177ce5ab5560df19ddf006f9bcb02ae3e4e4f',
856000: '74ff0b6f64e9dd5802fec2aac1d3ae194d28b9264114adaf0a882b46c8c918fe',
857000: '7b5badfa2e4f40aa597a504d7ebe83c3705a2c6169a8c168ce293db223bc2d32',
858000: '2bb0767a0f72b20d45ecfc3e34517dbda16d85758e040cf0e147f4cbd0cc57ac',
859000: '3d741b9c365a91ed76f85824b94d19ec19b608d232660840ba59c7aa4b2cb67f',
860000: 'd481a5a117878c0e3acd1f5844e150fb30e617577947d9846b1d214d703b71b0',
861000: '54033424e488a3f1ad6946d4a6d9acb48465d6b1dbe8e1c2504a54cc84d7cad4',
862000: '464bc3820a8cc8844dc9e26c388009e9982c656d46ef4b4fd0a2cb0e4eea0aaa',
863000: 'd1aa94be2174f66780c4f226b9da3f6712b0f37af8dec33360bea83ca261b342',
864000: '8c16008f11de5bc395d88cd802514ff647450f1bc136724b9aaf2ccce10a494f',
865000: '3dae86012e97a201e2e1a47c899001ac00f78dc108026ed7c4194858c6c6dd5a',
866000: 'afe5b0ccab995e1a1fa25fbc24c1d4b1a92c43042d03395f8743dcd806e72fd8',
867000: 'c83716ac171aa9ab0d414833db340fa30e82bfda6cc616d3038529caab9b5600',
868000: '8c409fe03cd35ef2d8e366818788b40eaeb4c8f6ae91450d75f4a66ca5f69cad',
869000: '1d47909ceba790b8e1ce2e9902ee2775ea99e58efdb95668f9803a8ccf95f286',
870000: '9adf5da1476388f053aa42de636da169d1cf1c9652cdf7cd9ad4fb18a0eb3388',
871000: '8ad57fb1e74bcba0b5614fbac003be2bb32275dd85b38f2d28a0585005a99cfc',
872000: '84a32e92012a356106e9657da8dab1a5491ea588fc29d411c69b20680c666420',
873000: 'adf5921bbbfaa43929f67e6a070975313b77b456e262c700a27be611fceb17ae',
874000: '09eaa7c4b18c79a46a2895190333f72336826223d5c986849a06f5153f49f2a5',
875000: '235d7e4f31966507312149ea4c5e294aa84c695cf840117f0ef5963be7a0bda1',
876000: '9aa9cb806ccbec0475ac330b496c5b2edeba38ba3f1e13ddd54a01457634a288',
877000: 'c1e7f9b2b20bb1c4c0deadbc786d31fdf36f262325342aa23d1a66e2846b22bc',
878000: 'ee0d2b20ac28ce23ab38698a57c6beff14f12b7af9d027c05cc92f652695f46b',
879000: '0eb0810f4b81d1845b0a88f05449408df2e45715c9210a656f45278c5fdf7956',
880000: 'e7d613027e3b4ca38d09bbef07998b57db237c6d67f1e8ea50024d2e0d9a1a72',
881000: '21af4d355d8756b8bf0369b2d79b5c824148ae069026ba5c14f9dd6b7555e1db',
}
+130 -50
View File
@@ -7,8 +7,6 @@ from binascii import hexlify
from collections import defaultdict
from dataclasses import dataclass
from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from prometheus_client import Gauge, Counter, Histogram
@@ -19,6 +17,12 @@ from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Inpu
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
if platform.system() == 'Windows' or ({'ANDROID_ARGUMENT', 'KIVY_BUILD'} & os.environ.keys()):
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
else:
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
log = logging.getLogger(__name__)
sqlite3.enable_callback_tracebacks(True)
@@ -62,12 +66,6 @@ def run_read_only_fetchone(sql, params):
raise
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
ReaderExecutorClass = ThreadPoolExecutor
else:
ReaderExecutorClass = ProcessPoolExecutor
class AIOSQLite:
reader_executor: ReaderExecutorClass
@@ -123,7 +121,13 @@ class AIOSQLite:
if self._closing:
return
self._closing = True
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
def __checkpoint_and_close(conn: sqlite3.Connection):
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
log.info("DB checkpoint finished.")
conn.close()
await asyncio.get_event_loop().run_in_executor(
self.writer_executor, __checkpoint_and_close, self.writer_connection)
self.writer_executor.shutdown(wait=True)
self.reader_executor.shutdown(wait=True)
self.read_ready.clear()
@@ -147,7 +151,7 @@ class AIOSQLite:
self.waiting_reads_metric.inc()
self.read_count_metric.inc()
try:
while self.writers: # more writes can come in while we are waiting for the first
while self.writers and not self._closing: # more writes can come in while we are waiting for the first
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
# throttle the writes if they pile up
self.urgent_read_done.clear()
@@ -155,6 +159,8 @@ class AIOSQLite:
# wait until the running writes have finished
await self.read_ready.wait()
still_waiting = True
if self._closing:
raise asyncio.CancelledError()
return await asyncio.get_event_loop().run_in_executor(
self.reader_executor, read_only_fn, sql, parameters
)
@@ -197,6 +203,8 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError()
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
@@ -232,6 +240,8 @@ class AIOSQLite:
self.read_ready.clear()
try:
async with self.write_lock:
if self._closing:
raise asyncio.CancelledError()
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
@@ -377,14 +387,31 @@ def interpolate(sql, values):
return sql
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
def constrain_single_or_list(constraints, column, value, convert=lambda x: x, negate=False):
if value is not None:
if isinstance(value, list):
value = [convert(v) for v in value]
if len(value) == 1:
constraints[column] = value[0]
if negate:
constraints[f"{column}__or"] = {
f"{column}__is_null": True,
f"{column}__not": value[0]
}
else:
constraints[column] = value[0]
elif len(value) > 1:
constraints[f"{column}__in"] = value
if negate:
constraints[f"{column}__or"] = {
f"{column}__is_null": True,
f"{column}__not_in": value
}
else:
constraints[f"{column}__in"] = value
elif negate:
constraints[f"{column}__or"] = {
f"{column}__is_null": True,
f"{column}__not": convert(value)
}
else:
constraints[column] = convert(value)
return constraints
@@ -419,9 +446,13 @@ class SQLiteMixin:
version = await self.db.execute_fetchone("SELECT version FROM version LIMIT 1;")
if version == (self.SCHEMA_VERSION,):
return
if version == ("1.5",) and self.SCHEMA_VERSION == "1.6":
await self.db.execute("ALTER TABLE txo ADD COLUMN has_source bool DEFAULT 1;")
await self.db.execute("UPDATE version SET version = ?", (self.SCHEMA_VERSION,))
return
await self.db.executescript('\n'.join(
f"DROP TABLE {table};" for table in tables
))
) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;')
await self.db.execute(self.CREATE_VERSION_TABLE)
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
await self.db.executescript(self.CREATE_TABLES_QUERY)
@@ -565,7 +596,7 @@ def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: L
class Database(SQLiteMixin):
SCHEMA_VERSION = "1.3"
SCHEMA_VERSION = "1.6"
PRAGMAS = """
pragma journal_mode=WAL;
@@ -619,6 +650,7 @@ class Database(SQLiteMixin):
txo_type integer not null default 0,
claim_id text,
claim_name text,
has_source bool,
channel_id text,
reposted_claim_id text
@@ -663,7 +695,8 @@ class Database(SQLiteMixin):
'address': txo.get_address(self.ledger),
'position': txo.position,
'amount': txo.amount,
'script': sqlite3.Binary(txo.script.source)
'script': sqlite3.Binary(txo.script.source),
'has_source': False,
}
if txo.is_claim:
if txo.can_decode_claim:
@@ -671,12 +704,18 @@ class Database(SQLiteMixin):
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
if claim.is_repost:
row['reposted_claim_id'] = claim.repost.reference.claim_id
row['has_source'] = True
if claim.is_signed:
row['channel_id'] = claim.signing_channel_id
if claim.is_stream:
row['has_source'] = claim.stream.has_source
else:
row['txo_type'] = TXO_TYPES['stream']
elif txo.is_support:
row['txo_type'] = TXO_TYPES['support']
support = txo.can_decode_support
if support and support.is_signed:
row['channel_id'] = support.signing_channel_id
elif txo.purchase is not None:
row['txo_type'] = TXO_TYPES['purchase']
row['claim_id'] = txo.purchased_claim_id
@@ -935,16 +974,18 @@ class Database(SQLiteMixin):
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)
async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraints):
async def get_txos(self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints):
include_is_spent = constraints.get('include_is_spent', False)
include_is_my_input = constraints.get('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
include_received_tips = constraints.pop('include_received_tips', False)
select_columns = [
"tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, "
"tx.txid, tx.height, tx.position as tx_position, tx.is_verified, "
"txo_type, txo.position as txo_position, amount, script"
]
if not no_tx:
select_columns.append("raw")
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
my_accounts_sql = ""
@@ -983,7 +1024,7 @@ class Database(SQLiteMixin):
if 'order_by' not in constraints or constraints['order_by'] == 'height':
constraints['order_by'] = [
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
"tx.height in (0, -1) DESC", "tx.height DESC", "tx.position DESC", "txo.position"
]
elif constraints.get('order_by', None) == 'none':
del constraints['order_by']
@@ -1022,32 +1063,33 @@ class Database(SQLiteMixin):
txo.received_tips = row['received_tips']
txos.append(txo)
channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
txo.private_key = private_key
break
if channel_ids:
channels = {
txo.claim_id: txo for txo in
(await self.get_channels(
wallet=wallet,
claim_id__in=channel_ids,
read_only=read_only
))
}
if not no_channel_info:
channel_ids = set()
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
txo.channel = channels.get(txo.claim.signing_channel_id, None)
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
txo.private_key = private_key
break
if channel_ids:
channels = {
txo.claim_id: txo for txo in
(await self.get_channels(
wallet=wallet,
claim_id__in=channel_ids,
read_only=read_only
))
}
for txo in txos:
if txo.is_claim and txo.can_decode_claim:
txo.channel = channels.get(txo.claim.signing_channel_id, None)
return txos
@@ -1109,6 +1151,41 @@ class Database(SQLiteMixin):
)
return balance[0]['total'] or 0
async def get_detailed_balance(self, accounts, read_only=False, **constraints):
constraints['accounts'] = accounts
result = (await self.select_txos(
f"COALESCE(SUM(amount), 0) AS total,"
f"COALESCE(SUM("
f" CASE WHEN"
f" txo_type NOT IN ({TXO_TYPES['other']}, {TXO_TYPES['purchase']})"
f" THEN amount ELSE 0 END), 0) AS reserved,"
f"COALESCE(SUM("
f" CASE WHEN"
f" txo_type IN ({','.join(map(str, CLAIM_TYPES))})"
f" THEN amount ELSE 0 END), 0) AS claims,"
f"COALESCE(SUM(CASE WHEN txo_type = {TXO_TYPES['support']} THEN amount ELSE 0 END), 0) AS supports,"
f"COALESCE(SUM("
f" CASE WHEN"
f" txo_type = {TXO_TYPES['support']} AND"
f" TXI.address IS NOT NULL AND"
f" TXI.address IN (SELECT address FROM account_address WHERE account = :$account__in0)"
f" THEN amount ELSE 0 END), 0) AS my_supports",
is_spent=False,
include_is_my_input=True,
read_only=read_only,
**constraints
))[0]
return {
"total": result["total"],
"available": result["total"] - result["reserved"],
"reserved": result["reserved"],
"reserved_subtotals": {
"claims": result["claims"],
"supports": result["my_supports"],
"tips": result["supports"] - result["my_supports"]
}
}
async def select_addresses(self, cols, read_only=False, **constraints):
return await self.db.execute_fetchall(*query(
f"SELECT {cols} FROM pubkey_address JOIN account_address USING (address)",
@@ -1254,13 +1331,16 @@ class Database(SQLiteMixin):
self.constrain_collections(constraints)
return self.get_utxo_count(**constraints)
async def release_all_outputs(self, account):
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
async def release_all_outputs(self, account=None):
if account is None:
await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1")
else:
await self.db.execute_fetchall(
"UPDATE txo SET is_reserved = 0 WHERE"
" is_reserved = 1 AND txo.address IN ("
" SELECT address from account_address WHERE account = ?"
" )", (account.public_key.address, )
)
def get_supports_summary(self, read_only=False, **constraints):
return self.get_txos(
+162 -212
View File
@@ -10,11 +10,11 @@ from collections import defaultdict
from binascii import hexlify, unhexlify
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
import pylru
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
from lbry.schema.url import URL
from lbry.crypto.hash import hash160, double_sha256, sha256
from lbry.crypto.base58 import Base58
from lbry.utils import LRUCacheWithMetrics
from .tasks import TaskGroup
from .database import Database
@@ -29,7 +29,6 @@ from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
from .bip32 import PubKey, PrivateKey
from .coinselection import CoinSelector
log = logging.getLogger(__name__)
LedgerType = Type['BaseLedger']
@@ -124,7 +123,6 @@ class Ledger(metaclass=LedgerRegistry):
self.network: Network = self.config.get('network') or Network(self)
self.network.on_header.listen(self.receive_header)
self.network.on_status.listen(self.process_status_update)
self.network.on_connected.listen(self.join_network)
self.accounts = []
self.fee_per_byte: int = self.config.get('fee_per_byte', self.default_fee_per_byte)
@@ -157,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
self._on_ready_controller = StreamController()
self.on_ready = self._on_ready_controller.stream
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 100_000))
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
self._update_tasks = TaskGroup()
self._other_tasks = TaskGroup() # that we dont need to start
self._utxo_reservation_lock = asyncio.Lock()
@@ -169,7 +167,7 @@ class Ledger(metaclass=LedgerRegistry):
self._known_addresses_out_of_sync = set()
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self._balance_cache = pylru.lrucache(100000)
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
@classmethod
def get_id(cls):
@@ -233,7 +231,7 @@ class Ledger(metaclass=LedgerRegistry):
async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]):
estimators = []
for account in funding_accounts:
utxos = await account.get_utxos()
utxos = await account.get_utxos(no_tx=True, no_channel_info=True)
for utxo in utxos:
estimators.append(utxo.get_estimator(self))
return estimators
@@ -330,7 +328,10 @@ class Ledger(metaclass=LedgerRegistry):
await self.network.on_connected.first
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
self.network.on_connected.listen(self.join_network)
asyncio.ensure_future(self.join_network())
await fully_synced
await self.db.release_all_outputs()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -414,6 +415,7 @@ class Ledger(metaclass=LedgerRegistry):
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
height, height+rewound
)
self._tx_cache.clear()
else:
raise IndexError(f"headers.connect() returned negative number ({added})")
@@ -478,12 +480,14 @@ class Ledger(metaclass=LedgerRegistry):
for address, remote_status in zip(batch, results):
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
addresses_remaining = addresses_remaining[batch_size:]
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *self.network.client.server_address_and_port)
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*self.network.client.server_address_and_port
)
if self.network.client and self.network.client.server_address_and_port:
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
len(addresses), *self.network.client.server_address_and_port)
if self.network.client and self.network.client.server_address_and_port:
log.info(
"finished subscribing to %i addresses on %s:%i", len(addresses),
*self.network.client.server_address_and_port
)
def process_status_update(self, update):
address, remote_status = update
@@ -510,11 +514,8 @@ class Ledger(metaclass=LedgerRegistry):
)
return True
acquire_lock_tasks = []
synced_txs = []
to_request = {}
pending_synced_history = {}
updated_cached_items = {}
already_synced = set()
already_synced_offset = 0
@@ -524,17 +525,6 @@ class Ledger(metaclass=LedgerRegistry):
already_synced.add((txid, remote_height))
already_synced_offset += 1
continue
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
for txid, remote_height in remote_history[already_synced_offset:]:
cache_item = self._tx_cache[txid]
acquire_lock_tasks.append(asyncio.create_task(cache_item.lock.acquire()))
if acquire_lock_tasks:
await asyncio.wait(acquire_lock_tasks)
tx_indexes = {}
@@ -542,29 +532,18 @@ class Ledger(metaclass=LedgerRegistry):
tx_indexes[txid] = i
if (txid, remote_height) in already_synced:
continue
cache_item = self._tx_cache.get(txid)
cache_item.pending_verifications += 1
updated_cached_items[txid] = cache_item
assert cache_item is not None, 'cache item is none'
assert cache_item.lock.locked(), 'cache lock is not held?'
# tx = cache_item.tx
# if cache_item.tx is not None and \
# cache_item.tx.height >= remote_height and \
# (cache_item.tx.is_verified or remote_height < 1):
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
# continue
to_request[i] = (txid, remote_height)
log.debug(
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
len(remote_history), address
)
requested_txes = await self._request_transaction_batch(to_request, len(remote_history), address)
for tx in requested_txes:
remote_history_txids = {txid for txid, _ in remote_history}
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
synced_txs.append(tx)
if len(pending_synced_history) % 100 == 0:
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
assert len(pending_synced_history) == len(remote_history), \
f"{len(pending_synced_history)} vs {len(remote_history)}"
@@ -575,21 +554,7 @@ class Ledger(metaclass=LedgerRegistry):
if f"{txid}:{height}:" != pending_synced_history[i]:
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
synced_history += pending_synced_history[i]
cache_size = self.config.get("tx_cache_size", 100_000)
for txid, cache_item in updated_cached_items.items():
cache_item.pending_verifications -= 1
if cache_item.pending_verifications < 0:
log.warning("config value tx cache size %i needs to be increased", cache_size)
cache_item.pending_verifications = 0
try:
cache_item.lock.release()
except RuntimeError:
log.warning("lock was already released?")
await self.db.save_transaction_io_batch(
[], address, self.address_to_hash160(address), synced_history
)
await self.db.set_address_history(address, synced_history)
if address_manager is None:
address_manager = await self.get_address_manager_for_address(address)
@@ -597,13 +562,6 @@ class Ledger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
for txid, cache_item in updated_cached_items.items():
if self._tx_cache.get(txid) is not cache_item:
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
if reattempt_update:
return await self.update_history(address, remote_status, address_manager, False)
return False
local_status, local_history = \
await self.get_local_status_and_history(address, synced_history)
@@ -635,150 +593,105 @@ class Ledger(metaclass=LedgerRegistry):
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
return True
async def cache_transaction(self, txid, remote_height, check_local=True):
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = self._tx_cache[txid] = TransactionCacheItem()
elif cache_item.tx is not None and \
cache_item.tx.height >= remote_height and \
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
cache_item.pending_verifications += 1
try:
async with cache_item.lock:
tx = cache_item.tx
if tx is None and check_local:
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
cache_item.tx = tx # make sure it's saved before caching it
tx.height = remote_height
if merkle and 0 < remote_height < len(self.headers):
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
finally:
cache_item.pending_verifications -= 1
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx.height = remote_height
cached = self._tx_cache.get(tx.id)
if not cached:
# cache txs looked up by transaction_show too
cached = TransactionCacheItem()
self._tx_cache[tx.id] = cached
cached.tx = tx
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
if 0 < remote_height < len(self.headers):
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
if not merkle:
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
if 'merkle' not in merkle:
return
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
return tx
async def _request_transaction_batch(self, to_request, remote_history_size, address):
header_cache = {}
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
batches = [[]]
remote_heights = {}
synced_txs = []
heights_in_batch = 0
last_height = 0
for idx in sorted(to_request):
txid = to_request[idx][0]
height = to_request[idx][1]
cache_hits = set()
for txid, height in sorted(to_request, key=lambda x: x[1]):
if cached:
cached_tx = self._tx_cache.get(txid)
if cached_tx is not None:
if cached_tx.tx is not None and cached_tx.tx.is_verified:
cache_hits.add(txid)
continue
else:
self._tx_cache[txid] = TransactionCacheItem()
remote_heights[txid] = height
if height != last_height:
heights_in_batch += 1
last_height = height
if len(batches[-1]) == 100 or heights_in_batch == 20:
if len(batches[-1]) == 100:
batches.append([])
heights_in_batch = 1
batches[-1].append(txid)
if not batches[-1]:
batches.pop()
if cached and cache_hits:
yield {txid: self._tx_cache[txid].tx for txid in cache_hits}
last_showed_synced_count = 0
async def _single_batch(batch):
this_batch_synced = []
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
merkle_height = merkle['block_height']
cache_item = self._tx_cache.get(txid)
if cache_item is None:
cache_item = TransactionCacheItem()
self._tx_cache[txid] = cache_item
tx = cache_item.tx or Transaction(unhexlify(raw), height=remote_height)
tx.height = remote_height
cache_item.tx = tx
if 'merkle' in merkle and remote_heights[txid] > 0:
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
try:
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
except IndexError:
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
else:
header_cache[remote_heights[txid]] = header
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
check_db_for_txos = []
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
if cache_item is not None:
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
else:
check_db_for_txos.append(txi.txo_ref.id)
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
)
}
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
referenced_txo = referenced_txos.get(txi.txo_ref.id)
if referenced_txo is not None:
txi.txo_ref = referenced_txo.ref
continue
cache_item = self._tx_cache.get(txi.txo_ref.id)
if cache_item is None:
cache_item = self._tx_cache[txi.txo_ref.id] = TransactionCacheItem()
if cache_item.tx is not None:
txi.txo_ref = cache_item.tx.ref
synced_txs.append(tx)
this_batch_synced.append(tx)
await self.db.save_transaction_io_batch(
this_batch_synced, address, self.address_to_hash160(address), ""
)
await asyncio.wait([
self._on_transaction_controller.add(TransactionEvent(address, tx))
for tx in this_batch_synced
])
nonlocal last_showed_synced_count
if last_showed_synced_count + 100 < len(synced_txs):
log.info("synced %i/%i transactions for %s", len(synced_txs), remote_history_size, address)
last_showed_synced_count = len(synced_txs)
for batch in batches:
await _single_batch(batch)
return synced_txs
txs = await self._single_batch(batch, remote_heights)
if cached:
for txid, tx in txs.items():
self._tx_cache[txid].tx = tx
yield txs
async def request_synced_transactions(self, to_request, remote_history, address):
async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())):
for tx in txs.values():
yield tx
await self._sync_and_save_batch(address, remote_history, txs)
async def _single_batch(self, batch, remote_heights):
heights = {remote_heights[txid] for txid in batch}
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
txs = {}
for txid, (raw, merkle) in batch_result.items():
remote_height = remote_heights[txid]
tx = Transaction(unhexlify(raw), height=remote_height)
txs[tx.id] = tx
await self.maybe_verify_transaction(tx, remote_height, merkle)
return txs
async def _sync_and_save_batch(self, address, remote_history, pending_txs):
await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values()))
await self.db.save_transaction_io_batch(
pending_txs.values(), address, self.address_to_hash160(address), ""
)
while pending_txs:
self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1]))
async def _sync(self, tx, remote_history, pending_txs):
check_db_for_txos = {}
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
continue
wanted_txid = txi.txo_ref.tx_ref.id
if wanted_txid not in remote_history:
continue
if wanted_txid in pending_txs:
txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref
else:
check_db_for_txos[txi] = txi.txo_ref.id
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(
txoid__in=list(check_db_for_txos.values()), order_by='txo.txoid', no_tx=True
)
}
for txi in check_db_for_txos:
if txi.txo_ref.id in referenced_txos:
txi.txo_ref = referenced_txos[txi.txo_ref.id].ref
else:
tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id)
if tx_from_db is None:
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
else:
txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref
return tx
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
details = await self.db.get_address(address=address)
@@ -846,11 +759,10 @@ class Ledger(metaclass=LedgerRegistry):
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
txs: List[Transaction] = []
if len(outputs.txs) > 0:
txs: List[Transaction] = await asyncio.gather(*(
self.cache_transaction(*tx) for tx in outputs.txs
))
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
txs.extend(tx.values())
_txos, blocked = outputs.inflate(txs)
@@ -922,13 +834,21 @@ class Ledger(metaclass=LedgerRegistry):
txo.received_tips = tips
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls, **kwargs):
resolve = partial(self.network.retriable_call, self.network.resolve)
urls_copy = list(urls)
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
txos = []
urls_copy = list(urls)
if new_sdk_server:
resolve = partial(self.network.new_resolve, new_sdk_server)
else:
resolve = partial(self.network.retriable_call, self.network.resolve)
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
batch, urls_copy = urls_copy[:100], urls_copy[100:]
txos.extend(
(await self._inflate_outputs(
resolve(batch), accounts, **kwargs
))[0]
)
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -941,13 +861,20 @@ class Ledger(metaclass=LedgerRegistry):
result[url] = txo
return result
async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]:
return await self.network.sum_supports(new_sdk_server, **kwargs)
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
**kwargs) -> Tuple[List[Output], dict, int, int]:
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
if new_sdk_server:
claim_search = partial(self.network.new_claim_search, new_sdk_server)
else:
claim_search = self.network.claim_search
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
include_is_my_output=include_is_my_output,
)
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
@@ -1010,6 +937,11 @@ class Ledger(metaclass=LedgerRegistry):
return self.db.get_purchase_count(**constraints)
async def _resolve_for_local_results(self, accounts, txos):
txos = await self._resolve_for_local_claim_results(accounts, txos)
txos = await self._resolve_for_local_support_results(accounts, txos)
return txos
async def _resolve_for_local_claim_results(self, accounts, txos):
results = []
response = await self.resolve(
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
@@ -1025,6 +957,23 @@ class Ledger(metaclass=LedgerRegistry):
results.append(txo)
return results
async def _resolve_for_local_support_results(self, accounts, txos):
channel_ids = set()
signed_support_txos = []
for txo in txos:
support = txo.can_decode_support
if support and support.signing_channel_id:
channel_ids.add(support.signing_channel_id)
signed_support_txos.append(txo)
if channel_ids:
channels = {
channel.claim_id: channel for channel in
(await self.claim_search(accounts, claim_ids=list(channel_ids)))[0]
}
for txo in signed_support_txos:
txo.channel = channels.get(txo.support.signing_channel_id)
return txos
async def get_claims(self, resolve=False, **constraints):
claims = await self.db.get_claims(**constraints)
if resolve:
@@ -1053,7 +1002,7 @@ class Ledger(metaclass=LedgerRegistry):
return self.db.get_channel_count(**constraints)
async def resolve_collection(self, collection, offset=0, page_size=1):
claim_ids = collection.claim.collection.claims.ids[offset:page_size+offset]
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
try:
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
except Exception as err:
@@ -1073,8 +1022,10 @@ class Ledger(metaclass=LedgerRegistry):
claims.append(None)
return claims
async def get_collections(self, resolve_claims=0, **constraints):
async def get_collections(self, resolve_claims=0, resolve=False, **constraints):
collections = await self.db.get_collections(**constraints)
if resolve:
collections = await self._resolve_for_local_results(constraints.get('accounts', []), collections)
if resolve_claims > 0:
for collection in collections:
collection.claims = await self.resolve_collection(collection, page_size=resolve_claims)
@@ -1102,17 +1053,17 @@ class Ledger(metaclass=LedgerRegistry):
'txid': tx.id,
'timestamp': ts,
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
'confirmations': (headers.height + 1) - tx.height if tx.height > 0 else 0,
'claim_info': [],
'update_info': [],
'support_info': [],
'abandon_info': [],
'purchase_info': []
}
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
is_my_inputs = all(txi.is_my_input for txi in tx.inputs)
if is_my_inputs:
# fees only matter if we are the ones paying them
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee)
item['fee'] = dewies_to_lbc(-tx.fee)
else:
# someone else paid the fees
@@ -1135,13 +1086,13 @@ class Ledger(metaclass=LedgerRegistry):
if txi.txo_ref.txo is not None:
other_txo = txi.txo_ref.txo
if (other_txo.is_claim or other_txo.script.is_support_claim) \
and other_txo.claim_id == txo.claim_id:
and other_txo.claim_id == txo.claim_id:
previous = other_txo
break
if previous is not None:
item['update_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(previous.amount-txo.amount),
'balance_delta': dewies_to_lbc(previous.amount - txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
@@ -1219,8 +1170,8 @@ class Ledger(metaclass=LedgerRegistry):
for account in accounts:
balance = self._balance_cache.get(account.id)
if not balance:
balance = self._balance_cache[account.id] =\
await account.get_detailed_balance(confirmations, reserved_subtotals=True)
balance = self._balance_cache[account.id] = \
await account.get_detailed_balance(confirmations)
for key, value in balance.items():
if key == 'reserved_subtotals':
for subkey, subvalue in value.items():
@@ -1238,7 +1189,6 @@ class TestNetLedger(Ledger):
extended_private_key_prefix = unhexlify('04358394')
checkpoints = {}
class RegTestLedger(Ledger):
network_name = 'regtest'
headers_class = UnvalidatedHeaders
+4
View File
@@ -182,7 +182,9 @@ class WalletManager:
ledger_config = {
'auto_connect': True,
'hub_timeout': config.hub_timeout,
'default_servers': config.lbryum_servers,
'concurrent_hub_requests': config.concurrent_hub_requests,
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
@@ -226,6 +228,8 @@ class WalletManager:
self.ledger.config = {
'auto_connect': True,
'default_servers': self.config.lbryum_servers,
'hub_timeout': self.config.hub_timeout,
'concurrent_hub_requests': self.config.concurrent_hub_requests,
'data_path': self.config.wallet_dir,
}
await self.ledger.stop()
+232 -193
View File
@@ -1,34 +1,39 @@
import logging
import asyncio
import json
import socket
import random
from time import perf_counter
from operator import itemgetter
from collections import defaultdict
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
from lbry.utils import resolve_host
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
from lbry.wallet.stream import StreamController
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.framer.max_size = self.max_errors = 1 << 32
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
self._concurrency = asyncio.Semaphore(concurrency)
@property
def concurrency(self):
return self._concurrency._value
@property
def available(self):
@@ -54,9 +59,9 @@ class ClientSession(BaseClientSession):
return result
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
try:
await self._concurrency.acquire()
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
request = asyncio.ensure_future(super().send_request(method, args))
@@ -66,7 +71,7 @@ class ClientSession(BaseClientSession):
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
if (perf_counter() - self.last_packet_received) < self.timeout:
continue
log.info("timeout sending %s to %s:%i", method, *self.server)
log.warning("timeout sending %s to %s:%i", method, *self.server)
raise asyncio.TimeoutError
if done:
try:
@@ -86,43 +91,11 @@ class ClientSession(BaseClientSession):
self.synchronous_close()
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
log.warning("cancelled sending %s to %s:%i", method, *self.server)
# self.synchronous_close()
raise
finally:
self.pending_amount -= 1
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
# TODO: change to 'ping' on newer protocol (above 1.2)
retry_delay = default_delay = 1.0
while True:
try:
if self.is_closing():
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except RPCError as e:
await self.close()
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
retry_delay = 60 * 60
except IncompatibleWalletServerError:
await self.close()
retry_delay = 60 * 60
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
pass
finally:
self.trigger_urgent_reconnect.clear()
self._concurrency.release()
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
@@ -133,6 +106,25 @@ class ClientSession(BaseClientSession):
raise IncompatibleWalletServerError(*self.server)
return response
async def keepalive_loop(self, timeout=3, max_idle=60):
try:
while True:
now = perf_counter()
if min(self.last_send, self.last_packet_received) + max_idle < now:
await asyncio.wait_for(
self.send_request('server.ping', []), timeout=timeout
)
else:
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
except Exception as err:
if isinstance(err, asyncio.CancelledError):
log.warning("closing connection to %s:%i", *self.server)
else:
log.exception("lost connection to spv")
finally:
if not self.is_closing():
self._close()
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
start = perf_counter()
@@ -144,12 +136,14 @@ class ClientSession(BaseClientSession):
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
log.warning("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self._on_disconnect_controller.add(True)
# self._on_disconnect_controller.add(True)
if self.network:
self.network.disconnect()
class Network:
@@ -159,13 +153,11 @@ class Network:
def __init__(self, ledger):
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
# self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
self._concurrency = asyncio.Semaphore(16)
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
@@ -181,70 +173,193 @@ class Network:
'blockchain.address.subscribe': self._on_status_controller,
}
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
self._urgent_need_reconnect = asyncio.Event()
self._loop_task: Optional[asyncio.Task] = None
self._keepalive_task: Optional[asyncio.Task] = None
@property
def config(self):
return self.ledger.config
async def switch_forever(self):
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
def disconnect(self):
if self._keepalive_task and not self._keepalive_task.done():
self._keepalive_task.cancel()
self._keepalive_task = None
async def start(self):
self.running = True
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
if not self.running:
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self.on_header.listen(self._update_remote_height)
self._loop_task = asyncio.create_task(self.network_loop())
self._urgent_need_reconnect.set()
def loop_task_done_callback(f):
try:
f.result()
except Exception:
if self.running:
log.exception("wallet server connection loop crashed")
self._loop_task.add_done_callback(loop_task_done_callback)
async def resolve_spv_dns(self):
hostname_to_ip = {}
ip_to_hostnames = defaultdict(list)
async def resolve_spv(server, port):
try:
server_addr = await resolve_host(server, port, 'udp')
hostname_to_ip[server] = (server_addr, port)
ip_to_hostnames[(server_addr, port)].append(server)
except socket.error:
log.warning("error looking up dns for spv server %s:%i", server, port)
except Exception:
log.exception("error looking up dns for spv server %s:%i", server, port)
# accumulate the dns results
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
return hostname_to_ip, ip_to_hostnames
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]:
loop = asyncio.get_event_loop()
pong_responses = asyncio.Queue()
connection = SPVStatusClientProtocol(pong_responses)
sent_ping_timestamps = {}
_, ip_to_hostnames = await self.resolve_spv_dns()
n = len(ip_to_hostnames)
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['default_servers']))
pongs = {}
try:
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
# could raise OSError if it cant bind
start = perf_counter()
for server in ip_to_hostnames:
connection.ping(server)
sent_ping_timestamps[server] = perf_counter()
while len(pongs) < n:
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
latency = ts - start
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
pong.available, pong.height)
if pong.available:
pongs[(ip_to_hostnames[remote][0], remote[1])] = pong
return pongs
except asyncio.TimeoutError:
if pongs:
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
return pongs
else:
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
len(sent_ping_timestamps),
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
random_server = random.choice(list(ip_to_hostnames.keys()))
host, port = random_server
log.warning("trying fallback to randomly selected spv: %s:%i", host, port)
return {(host, port): None}
finally:
connection.close()
async def connect_to_fastest(self) -> Optional[ClientSession]:
fastest_spvs = await self.get_n_fastest_spvs()
for (host, port) in fastest_spvs:
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
concurrency=self.config['concurrent_hub_requests'])
try:
await client.create_connection()
log.warning("Connected to spv server %s:%i", host, port)
await client.ensure_server_version()
return client
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
log.warning("Connecting to %s:%d failed", host, port)
client._close()
return
async def network_loop(self):
sleep_delay = 30
while self.running:
await asyncio.wait(
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
)
if self._urgent_need_reconnect.is_set():
sleep_delay = 30
self._urgent_need_reconnect.clear()
if not self.is_connected:
client = await self.connect_to_fastest()
if not client:
log.warning("failed to connect to any spv servers, retrying later")
sleep_delay *= 2
sleep_delay = min(sleep_delay, 300)
continue
log.debug("get spv server features %s:%i", *client.server)
features = await client.send_request('server.features', [])
self.client, self.server_features = client, features
log.info("subscribe to headers %s:%i", *client.server)
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
server_str = "%s:%i" % client.server
log.info("maintaining connection to spv server %s", server_str)
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
try:
if not self._urgent_need_reconnect.is_set():
await asyncio.wait(
[self._keepalive_task, self._urgent_need_reconnect.wait()],
return_when=asyncio.FIRST_COMPLETED
)
else:
await self._keepalive_task
if self._urgent_need_reconnect.is_set():
log.warning("urgent reconnect needed")
self._urgent_need_reconnect.clear()
if self._keepalive_task and not self._keepalive_task.done():
self._keepalive_task.cancel()
except asyncio.CancelledError:
pass
finally:
self._keepalive_task = None
self.client = None
self.server_features = None
log.warning("connection lost to %s", server_str)
log.info("network loop finished")
async def stop(self):
if self.running:
self.running = False
self._switch_task.cancel()
self.session_pool.stop()
self.running = False
self.disconnect()
if self._loop_task and not self._loop_task.done():
self._loop_task.cancel()
self._loop_task = None
if self.aiohttp_session:
await self.aiohttp_session.close()
self.aiohttp_session = None
@property
def is_connected(self):
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args, restricted=True):
session = self.client if restricted else self.session_pool.fastest_session
if session and not session.is_closing():
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
if session or self.is_connected:
session = session or self.client
return session.send_request(list_or_method, args)
else:
self.session_pool.trigger_nodelay_connect()
self._urgent_need_reconnect.set()
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def retriable_call(self, function, *args, **kwargs):
async with self._concurrency:
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
pass
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
log.warning("connection error")
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
@@ -255,9 +370,9 @@ class Network:
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_batch(self, txids):
def get_transaction_batch(self, txids, restricted=True):
# use any server if its old, otherwise restrict to who gave us the history
return self.rpc('blockchain.transaction.get_batch', txids, True)
return self.rpc('blockchain.transaction.get_batch', txids, restricted)
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
@@ -307,103 +422,27 @@ class Network:
def get_server_features(self):
return self.rpc('server.features', (), restricted=True)
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def resolve(self, urls, session_override=None):
return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def claim_search(self, session_override=None, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
async def new_resolve(self, server, urls):
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
async def new_claim_search(self, server, **kwargs):
kwargs['protobuf'] = True
message = {"method": "claim_search", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
class SessionPool:
def __init__(self, network: Network, timeout: float):
self.network = network
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.timeout = timeout
self.new_connection_event = asyncio.Event()
@property
def online(self):
return any(not session.is_closing() for session in self.sessions)
@property
def available_sessions(self):
return (session for session in self.sessions if session.available)
@property
def fastest_session(self):
if not self.online:
return None
return min(
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions] or [(0, None)],
key=itemgetter(0)
)[1]
def _get_session_connect_callback(self, session: ClientSession):
loop = asyncio.get_event_loop()
def callback():
duplicate_connections = [
s for s in self.sessions
if s is not session and s.server_address_and_port == session.server_address_and_port
]
already_connected = None if not duplicate_connections else duplicate_connections[0]
if already_connected:
self.sessions.pop(session).cancel()
session.synchronous_close()
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
session.server[0], already_connected.server[0])
loop.call_later(3600, self._connect_session, session.server)
return
self.new_connection_event.set()
log.info("connected to %s:%i", *session.server)
return callback
def _connect_session(self, server: Tuple[str, int]):
session = None
for s in self.sessions:
if s.server == server:
session = s
break
if not session:
session = ClientSession(
network=self.network, server=server
)
session._on_connect_cb = self._get_session_connect_callback(session)
task = self.sessions.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
def start(self, default_servers):
for server in default_servers:
self._connect_session(server)
def stop(self):
for session, task in self.sessions.items():
task.cancel()
session.synchronous_close()
self.sessions.clear()
def ensure_connections(self):
for session in self.sessions:
self._connect_session(session.server)
def trigger_nodelay_connect(self):
# used when other parts of the system sees we might have internet back
# bypasses the retry interval
for session in self.sessions:
session.trigger_urgent_reconnect.set()
async def wait_for_fastest_session(self):
while not self.fastest_session:
self.trigger_nodelay_connect()
self.new_connection_event.clear()
await self.new_connection_event.wait()
return self.fastest_session
async def sum_supports(self, server, **kwargs):
message = {"method": "support_sum", "params": kwargs}
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
+11 -2
View File
@@ -11,6 +11,7 @@ import importlib
from binascii import hexlify
from typing import Type, Optional
import urllib.request
from uuid import uuid4
import lbry
from lbry.wallet.server.server import Server
@@ -130,7 +131,9 @@ class WalletNode:
self.ledger_class.get_id(): {
'api_port': self.port,
'default_servers': [(spv_node.hostname, spv_node.port)],
'data_path': self.data_path
'data_path': self.data_path,
'hub_timeout': 30,
'concurrent_hub_requests': 32,
}
},
'wallets': [wallet_file_name]
@@ -168,6 +171,7 @@ class SPVNode:
self.server = None
self.hostname = 'localhost'
self.port = 50001 + node_number # avoid conflict with default daemon
self.udp_port = self.port
self.session_timeout = 600
self.rpc_port = '0' # disabled by default
@@ -182,10 +186,13 @@ class SPVNode:
'REORG_LIMIT': '100',
'HOST': self.hostname,
'TCP_PORT': str(self.port),
'UDP_PORT': str(self.udp_port),
'SESSION_TIMEOUT': str(self.session_timeout),
'MAX_QUERY_WORKERS': '0',
'INDIVIDUAL_TAG_INDEXES': '',
'RPC_PORT': self.rpc_port
'RPC_PORT': self.rpc_port,
'ES_INDEX_PREFIX': uuid4().hex,
'ES_MODE': 'writer',
}
if extraconf:
conf.update(extraconf)
@@ -197,6 +204,8 @@ class SPVNode:
async def stop(self, cleanup=True):
try:
await self.server.db.search_index.delete_index()
await self.server.db.search_index.stop()
await self.server.stop()
finally:
cleanup and self.cleanup()
-3
View File
@@ -108,9 +108,6 @@ class Response:
class CodeMessageError(Exception):
def __init__(self, code, message):
super().__init__(code, message)
@property
def code(self):
return self.args[0]
+3
View File
@@ -32,10 +32,13 @@ import inspect
# other_params: None means cannot be called with keyword arguments only
# any means any name is good
from functools import lru_cache
SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args '
'required_names other_names')
@lru_cache(256)
def signature_info(func):
params = inspect.signature(func).parameters
min_args = max_args = 0
+95 -74
View File
@@ -5,12 +5,12 @@ from concurrent.futures.thread import ThreadPoolExecutor
from typing import Optional
from prometheus_client import Gauge, Histogram
import lbry
from lbry.schema.claim import Claim
from lbry.wallet.server.db.writer import SQLDB
from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.udp import StatusServer
class Prefetcher:
@@ -171,7 +171,8 @@ class BlockProcessor:
# Caches of unflushed items.
self.headers = []
self.tx_hashes = []
self.block_hashes = []
self.block_txs = []
self.undo_infos = []
# UTXO cache
@@ -184,6 +185,7 @@ class BlockProcessor:
self.search_cache = {}
self.history_cache = {}
self.status_server = StatusServer()
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
@@ -212,17 +214,24 @@ class BlockProcessor:
if hprevs == chain:
start = time.perf_counter()
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
if self.sql:
await self.db.search_index.claim_consumer(self.sql.claim_producer())
for cache in self.search_cache.values():
cache.clear()
self.history_cache.clear()
self.notifications.notified_mempool_txs.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
self.block_count_metric.set(self.height)
self.block_update_time_metric.observe(processed_time)
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
if not self.db.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
if self._caught_up_event.is_set():
if self.sql:
await self.db.search_index.apply_filters(self.sql.blocked_streams, self.sql.blocked_channels,
self.sql.filtered_streams, self.sql.filtered_channels)
await self.notifications.on_block(self.touched, self.height)
self.touched = set()
elif hprevs[0] != chain[0]:
@@ -246,7 +255,7 @@ class BlockProcessor:
self.logger.info('chain reorg detected')
else:
self.logger.info(f'faking a reorg of {count:,d} blocks')
await self.flush(True)
async def get_raw_blocks(last_height, hex_hashes):
heights = range(last_height, last_height - len(hex_hashes), -1)
@@ -263,17 +272,27 @@ class BlockProcessor:
self.touched.discard(None)
self.db.flush_backup(self.flush_data(), self.touched)
start, last, hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
for hex_hashes in chunks(hashes, 50):
raw_blocks = await get_raw_blocks(last, hex_hashes)
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
try:
await self.flush(True)
start, last, hashes = await self.reorg_hashes(count)
# Reverse and convert to hex strings.
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
self.logger.info("reorg %i block hashes", len(hashes))
for hex_hashes in chunks(hashes, 50):
raw_blocks = await get_raw_blocks(last, hex_hashes)
self.logger.info("got %i raw blocks", len(raw_blocks))
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
except:
self.logger.exception("boom")
raise
finally:
self.logger.info("done with reorg")
async def reorg_hashes(self, count):
"""Return a pair (start, last, hashes) of blocks to back up during a
@@ -336,8 +355,8 @@ class BlockProcessor:
def flush_data(self):
"""The data for a flush. The lock must be taken."""
assert self.state_lock.locked()
return FlushData(self.height, self.tx_count, self.headers,
self.tx_hashes, self.undo_infos, self.utxo_cache,
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
self.block_txs, self.undo_infos, self.utxo_cache,
self.db_deletes, self.tip)
async def flush(self, flush_utxos):
@@ -392,7 +411,8 @@ class BlockProcessor:
for block in blocks:
height += 1
undo_info = self.advance_txs(
height, block.transactions, self.coin.electrum_header(block.header, height)
height, block.transactions, self.coin.electrum_header(block.header, height),
self.coin.header_hash(block.header)
)
if height >= min_height:
self.undo_infos.append((undo_info, height))
@@ -403,25 +423,26 @@ class BlockProcessor:
self.headers.extend(headers)
self.tip = self.coin.header_hash(headers[-1])
def advance_txs(self, height, txs, header):
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
def advance_txs(self, height, txs, header, block_hash):
self.block_hashes.append(block_hash)
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
# Use local vars for speed in the loops
undo_info = []
tx_num = self.tx_count
script_hashX = self.coin.hashX_from_script
s_pack = pack
hashXs_by_tx = []
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
undo_info_append = undo_info.append
update_touched = self.touched.update
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
append_hashX_by_tx = hashXs_by_tx.append
hashX_from_script = self.coin.hashX_from_script
for tx, tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
tx_numb = s_pack('<I', tx_num)
tx_numb = pack('<I', tx_num)
# Spend the inputs
for txin in tx.inputs:
@@ -434,18 +455,17 @@ class BlockProcessor:
# Add the new UTXOs
for idx, txout in enumerate(tx.outputs):
# Get the hashX. Ignore unspendable outputs
hashX = script_hashX(txout.pk_script)
hashX = hashX_from_script(txout.pk_script)
if hashX:
append_hashX(hashX)
put_utxo(tx_hash + s_pack('<H', idx),
hashX + tx_numb + s_pack('<Q', txout.value))
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
append_hashXs(hashXs)
append_hashX_by_tx(hashXs)
update_touched(hashXs)
self.db.total_transactions.append(tx_hash)
tx_num += 1
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
self.tx_count = tx_num
self.db.tx_counts.append(tx_num)
@@ -462,6 +482,7 @@ class BlockProcessor:
coin = self.coin
for raw_block in raw_blocks:
self.logger.info("backup block %i", self.height)
# Check and update self.tip
block = coin.block(raw_block, self.height)
header_hash = coin.header_hash(block.header)
@@ -487,20 +508,16 @@ class BlockProcessor:
# Use local vars for speed in the loops
s_pack = pack
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 12 + HASHX_LEN
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
hashX = script_hashX(txout.pk_script)
hashX = self.coin.hashX_from_script(txout.pk_script)
if hashX:
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-12])
cache_value = self.spend_utxo(tx_hash, idx)
self.touched.add(cache_value[:-12])
# Restore the inputs
for txin in reversed(tx.inputs):
@@ -508,9 +525,10 @@ class BlockProcessor:
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
undo_item)
touched.add(undo_item[:-12])
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
self.touched.add(undo_item[:-12])
self.db.total_transactions.pop()
assert n == 0
self.tx_count -= len(txs)
@@ -587,15 +605,18 @@ class BlockProcessor:
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
candidates = {db_key: hashX for db_key, hashX
in self.db.utxo_db.iterator(prefix=prefix)}
candidates = dict(self.db.utxo_db.iterator(prefix=prefix))
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-4:]
if len(candidates) > 1:
tx_num, = unpack('<I', tx_num_packed)
hash, height = self.db.fs_tx_hash(tx_num)
try:
hash, height = self.db.fs_tx_hash(tx_num)
except IndexError:
self.logger.error("data integrity error for hashx history: %s missing tx #%s (%s:%s)",
hashX.hex(), tx_num, hash_to_hex_str(tx_hash), tx_idx)
continue
if hash != tx_hash:
assert hash is not None # Should always be found
continue
@@ -604,12 +625,17 @@ class BlockProcessor:
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-6:]
utxo_value_packed = self.db.utxo_db.get(udb_key)
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
if utxo_value_packed is None:
self.logger.warning(
"%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), tx_idx, hash_to_hex_str(hashX)
)
raise ChainError(f"{hash_to_hex_str(tx_hash)}:{tx_idx} is not found in UTXO db for {hash_to_hex_str(hashX)}")
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
self.logger.error('UTXO {hash_to_hex_str(tx_hash)} / {tx_idx} not found in "h" table')
raise ChainError('UTXO {} / {:,d} not found in "h" table'
.format(hash_to_hex_str(tx_hash), tx_idx))
@@ -622,12 +648,16 @@ class BlockProcessor:
self._caught_up_event.set()
await self.blocks_event.wait()
self.blocks_event.clear()
if self.reorg_count:
if self.reorg_count: # this could only happen by calling the reorg rpc
await self.reorg_chain(self.reorg_count)
self.reorg_count = 0
else:
blocks = self.prefetcher.get_prefetched_blocks()
await self.check_and_advance_blocks(blocks)
try:
await self.check_and_advance_blocks(blocks)
except Exception:
self.logger.exception("error while processing txs")
raise
async def _first_caught_up(self):
self.logger.info(f'caught up to height {self.height}')
@@ -661,9 +691,11 @@ class BlockProcessor:
disk before exiting, as otherwise a significant amount of work
could be lost.
"""
self._caught_up_event = caught_up_event
try:
await self._first_open_dbs()
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
await asyncio.wait([
self.prefetcher.main_loop(self.height),
self._process_prefetched_blocks()
@@ -674,6 +706,7 @@ class BlockProcessor:
self.logger.exception("Block processing failed!")
raise
finally:
self.status_server.stop()
# Shut down block processing
self.logger.info('flushing to DB for a clean shutdown...')
await self.flush(True)
@@ -693,7 +726,6 @@ class BlockProcessor:
class Timer:
def __init__(self, name):
self.name = name
self.total = 0
@@ -756,38 +788,27 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer = Timer('BlockProcessor')
def advance_blocks(self, blocks):
self.sql.begin()
if self.sql:
self.sql.begin()
try:
self.timer.run(super().advance_blocks, blocks)
except:
self.logger.exception(f'Error while advancing transaction in new block.')
raise
finally:
self.sql.commit()
if self.db.first_sync and self.height == self.daemon.cached_height():
if self.sql:
self.sql.commit()
if self.sql and self.db.first_sync and self.height == self.daemon.cached_height():
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
def advance_txs(self, height, txs, header):
def advance_txs(self, height, txs, header, block_hash):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
if self.sql:
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
self.timer.show(height=height)
return undo
def _checksig(self, value, address):
try:
claim_dict = Claim.from_bytes(value)
cert_id = claim_dict.signing_channel_hash
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = Claim.from_bytes(cert_claim.value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception:
pass
+3 -1
View File
@@ -8,7 +8,7 @@ from lbry.wallet.server.server import Server
def get_argument_parser():
parser = argparse.ArgumentParser(
prog="torba-server"
prog="lbry-hub"
)
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
nargs="?", default="lbry.wallet.server.coin.LBC")
@@ -27,6 +27,8 @@ def main():
coin_class = get_coin_class(args.spvserver)
logging.basicConfig(level=logging.INFO)
logging.info('lbry.server starting')
logging.getLogger('aiohttp').setLevel(logging.WARNING)
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
try:
server = Server(Env(coin_class))
server.run()
+7 -6
View File
@@ -3,11 +3,10 @@ import itertools
import json
import time
from functools import wraps
from pylru import lrucache
import aiohttp
from prometheus_client import Gauge, Histogram
from lbry.utils import LRUCacheWithMetrics
from lbry.wallet.rpc.jsonrpc import RPCError
from lbry.wallet.server.util import hex_to_bytes, class_logger
from lbry.wallet.rpc import JSONRPC
@@ -55,8 +54,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(100000)
self._block_cache = lrucache(10000)
self._block_hash_cache = LRUCacheWithMetrics(100000)
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
async def close(self):
if self.connector:
@@ -250,8 +249,10 @@ class Daemon:
async def deserialised_block(self, hex_hash):
"""Return the deserialised block with the given hex hash."""
if not self._block_cache.get(hex_hash):
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
if hex_hash not in self._block_cache:
block = await self._send_single('getblock', (hex_hash, True))
self._block_cache[hex_hash] = block
return block
return self._block_cache[hex_hash]
async def raw_blocks(self, hex_hashes):
+401 -202
View File
@@ -1,7 +1,8 @@
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
'repost': 3
'repost': 3,
'collection': 4,
}
STREAM_TYPES = {
@@ -10,212 +11,410 @@ STREAM_TYPES = {
'image': 3,
'document': 4,
'binary': 5,
'model': 6
'model': 6,
}
# 9/21/2020
MOST_USED_TAGS = {
"gaming",
"people & blogs",
"entertainment",
"music",
"pop culture",
"education",
"technology",
"blockchain",
"news",
"funny",
"science & technology",
"learning",
"gameplay",
"news & politics",
"comedy",
"bitcoin",
"beliefs",
"nature",
"art",
"economics",
"film & animation",
"lets play",
"games",
"sports",
"howto & style",
"game",
"cryptocurrency",
"playstation 4",
"automotive",
"crypto",
"mature",
"sony interactive entertainment",
"walkthrough",
"tutorial",
"video game",
"weapons",
"playthrough",
"pc",
"anime",
"how to",
"btc",
"fun",
"ethereum",
"food",
"travel & events",
"minecraft",
"science",
"autos & vehicles",
"play",
"politics",
"commentary",
"twitch",
"ps4live",
"love",
"ps4",
"nonprofits & activism",
"ps4share",
"fortnite",
"xbox",
"porn",
"video games",
"trump",
"español",
"money",
"music video",
"nintendo",
"movie",
"coronavirus",
"donald trump",
"steam",
"trailer",
"android",
"podcast",
"xbox one",
"survival",
"audio",
"linux",
"travel",
"funny moments",
"litecoin",
"animation",
"gamer",
"lets",
"playstation",
"bitcoin news",
"history",
"xxx",
"fox news",
"dance",
"god",
"adventure",
"liberal",
"2020",
"horror",
"government",
"freedom",
"reaction",
"meme",
"photography",
"truth",
"health",
"lbry",
"family",
"online",
"eth",
"crypto news",
"diy",
"trading",
"gold",
"memes",
"world",
"space",
"lol",
"covid-19",
"rpg",
"humor",
"democrat",
"film",
"call of duty",
"tech",
"religion",
"conspiracy",
"rap",
"cnn",
"hangoutsonair",
"unboxing",
"fiction",
"conservative",
"cars",
"hoa",
"epic",
"programming",
"progressive",
"cryptocurrency news",
"classical",
"jesus",
"movies",
"book",
"ps3",
"republican",
"fitness",
"books",
"multiplayer",
"animals",
"pokemon",
"bitcoin price",
"facebook",
"sharefactory",
"criptomonedas",
"cod",
"bible",
"business",
"stream",
"comics",
"how",
"fail",
"nsfw",
"new music",
"satire",
"pets & animals",
"computer",
"classical music",
"indie",
"musica",
"msnbc",
"fps",
"mod",
"sport",
"sony",
"ripple",
"auto",
"rock",
"marvel",
"complete",
"mining",
"political",
"mobile",
"pubg",
"hip hop",
"flat earth",
"xbox 360",
"reviews",
"vlogging",
"latest news",
"hack",
"tarot",
"iphone",
"media",
"cute",
"christian",
"free speech",
"trap",
"war",
"remix",
"ios",
"xrp",
"spirituality",
"song",
"league of legends",
"cat"
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
def normalize_tag(tag):
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
}
INDEXED_LANGUAGES = [
'none',
'en',
'aa',
'ab',
'ae',
'af',
'ak',
'am',
'an',
'ar',
'as',
'av',
'ay',
'az',
'ba',
'be',
'bg',
'bh',
'bi',
'bm',
'bn',
'bo',
'br',
'bs',
'ca',
'ce',
'ch',
'co',
'cr',
'cs',
'cu',
'cv',
'cy',
'da',
'de',
'dv',
'dz',
'ee',
'el',
'eo',
'es',
'et',
'eu',
'fa',
'ff',
'fi',
'fj',
'fo',
'fr',
'fy',
'ga',
'gd',
'gl',
'gn',
'gu',
'gv',
'ha',
'he',
'hi',
'ho',
'hr',
'ht',
'hu',
'hy',
'hz',
'ia',
'id',
'ie',
'ig',
'ii',
'ik',
'io',
'is',
'it',
'iu',
'ja',
'jv',
'ka',
'kg',
'ki',
'kj',
'kk',
'kl',
'km',
'kn',
'ko',
'kr',
'ks',
'ku',
'kv',
'kw',
'ky',
'la',
'lb',
'lg',
'li',
'ln',
'lo',
'lt',
'lu',
'lv',
'mg',
'mh',
'mi',
'mk',
'ml',
'mn',
'mr',
'ms',
'mt',
'my',
'na',
'nb',
'nd',
'ne',
'ng',
'nl',
'nn',
'no',
'nr',
'nv',
'ny',
'oc',
'oj',
'om',
'or',
'os',
'pa',
'pi',
'pl',
'ps',
'pt',
'qu',
'rm',
'rn',
'ro',
'ru',
'rw',
'sa',
'sc',
'sd',
'se',
'sg',
'si',
'sk',
'sl',
'sm',
'sn',
'so',
'sq',
'sr',
'ss',
'st',
'su',
'sv',
'sw',
'ta',
'te',
'tg',
'th',
'ti',
'tk',
'tl',
'tn',
'to',
'tr',
'ts',
'tt',
'tw',
'ty',
'ug',
'uk',
'ur',
'uz',
've',
'vi',
'vo',
'wa',
'wo',
'xh',
'yi',
'yo',
'za',
'zh',
'zu'
]
@@ -0,0 +1 @@
from .search import SearchIndex
@@ -0,0 +1,61 @@
INDEX_DEFAULT_SETTINGS = {
"settings":
{"analysis":
{"analyzer": {
"default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}},
"index":
{"refresh_interval": -1,
"number_of_shards": 1,
"number_of_replicas": 0,
"sort": {
"field": ["trending_mixed", "release_time"],
"order": ["desc", "desc"]
}}
},
"mappings": {
"properties": {
"claim_id": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text",
"index_prefixes": {
"min_chars": 1,
"max_chars": 10
}
},
"height": {"type": "integer"},
"claim_type": {"type": "byte"},
"censor_type": {"type": "byte"},
"trending_mixed": {"type": "float"},
"release_time": {"type": "long"},
}
}
}
FIELDS = {'is_controlling', 'last_take_over_height', 'claim_id', 'claim_name', 'normalized', 'tx_position', 'amount',
'timestamp', 'creation_timestamp', 'height', 'creation_height', 'activation_height', 'expiration_height',
'release_time', 'short_url', 'canonical_url', 'title', 'author', 'description', 'claim_type', 'reposted',
'stream_type', 'media_type', 'fee_amount', 'fee_currency', 'duration', 'reposted_claim_hash', 'censor_type',
'claims_in_channel', 'channel_join', 'signature_valid', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'channel_id', 'tx_id', 'tx_nout',
'signature', 'signature_digest', 'public_key_bytes', 'public_key_hash', 'public_key_id', '_id', 'tags',
'reposted_claim_id'}
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id',
'media_type', 'normalized', 'public_key_bytes', 'public_key_hash', 'short_url', 'signature',
'signature_digest', 'stream_type', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'}
RANGE_FIELDS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed', 'censor_type',
'trending_local', 'trending_global',
}
REPLACEMENTS = {
'name': 'normalized',
'txid': 'tx_id',
'claim_hash': '_id'
}
@@ -0,0 +1,590 @@
import asyncio
import struct
from binascii import unhexlify
from collections import Counter, deque
from decimal import Decimal
from operator import itemgetter
from typing import Optional, List, Iterable, Union
from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
from elasticsearch.helpers import async_streaming_bulk
from lbry.crypto.base58 import Base58
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
from lbry.schema.result import Outputs, Censor
from lbry.schema.tags import clean_tags
from lbry.schema.url import URL, normalize_name
from lbry.utils import LRUCache
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES
from lbry.wallet.server.db.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \
RANGE_FIELDS
from lbry.wallet.server.util import class_logger
class ChannelResolution(str):
@classmethod
def lookup_error(cls, url):
return LookupError(f'Could not find channel in "{url}".')
class StreamResolution(str):
@classmethod
def lookup_error(cls, url):
return LookupError(f'Could not find claim at "{url}".')
class SearchIndex:
def __init__(self, index_prefix: str, search_timeout=3.0, elastic_host='localhost', elastic_port=9200):
self.search_timeout = search_timeout
self.sync_timeout = 600 # wont hit that 99% of the time, but can hit on a fresh import
self.search_client: Optional[AsyncElasticsearch] = None
self.sync_client: Optional[AsyncElasticsearch] = None
self.index = index_prefix + 'claims'
self.logger = class_logger(__name__, self.__class__.__name__)
self.claim_cache = LRUCache(2 ** 15)
self.short_id_cache = LRUCache(2 ** 17)
self.search_cache = LRUCache(2 ** 17)
self.resolution_cache = LRUCache(2 ** 17)
self._elastic_host = elastic_host
self._elastic_port = elastic_port
async def start(self):
if self.sync_client:
return
hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
self.search_client = AsyncElasticsearch(hosts, timeout=self.search_timeout)
while True:
try:
await self.sync_client.cluster.health(wait_for_status='yellow')
break
except ConnectionError:
self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
await asyncio.sleep(1)
res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
return res.get('acknowledged', False)
def stop(self):
clients = [self.sync_client, self.search_client]
self.sync_client, self.search_client = None, None
return asyncio.ensure_future(asyncio.gather(*(client.close() for client in clients)))
def delete_index(self):
return self.sync_client.indices.delete(self.index, ignore_unavailable=True)
async def _consume_claim_producer(self, claim_producer):
count = 0
for op, doc in claim_producer:
if op == 'delete':
yield {'_index': self.index, '_op_type': 'delete', '_id': doc}
else:
yield extract_doc(doc, self.index)
count += 1
if count % 100 == 0:
self.logger.info("Indexing in progress, %d claims.", count)
self.logger.info("Indexing done for %d claims.", count)
async def claim_consumer(self, claim_producer):
touched = set()
async for ok, item in async_streaming_bulk(self.sync_client, self._consume_claim_producer(claim_producer),
raise_on_error=False):
if not ok:
self.logger.warning("indexing failed for an item: %s", item)
else:
item = item.popitem()[1]
touched.add(item['_id'])
await self.sync_client.indices.refresh(self.index)
self.logger.info("Indexing done.")
def update_filter_query(self, censor_type, blockdict, channels=False):
blockdict = {key[::-1].hex(): value[::-1].hex() for key, value in blockdict.items()}
if channels:
update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
else:
update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
key = 'channel_id' if channels else 'claim_id'
update['script'] = {
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_hash=params[ctx._source.{key}]",
"lang": "painless",
"params": blockdict
}
return update
async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels):
if filtered_streams:
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4)
await self.sync_client.indices.refresh(self.index)
if filtered_channels:
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4)
await self.sync_client.indices.refresh(self.index)
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4)
await self.sync_client.indices.refresh(self.index)
if blocked_streams:
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4)
await self.sync_client.indices.refresh(self.index)
if blocked_channels:
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4)
await self.sync_client.indices.refresh(self.index)
await self.sync_client.update_by_query(
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
await self.sync_client.indices.refresh(self.index)
self.search_cache.clear()
self.short_id_cache.clear()
self.claim_cache.clear()
self.resolution_cache.clear()
async def session_query(self, query_name, kwargs):
offset, total = kwargs.get('offset', 0) if isinstance(kwargs, dict) else 0, 0
total_referenced = []
if query_name == 'resolve':
total_referenced, response, censor = await self.resolve(*kwargs)
else:
cache_item = ResultCacheItem.from_cache(str(kwargs), self.search_cache)
if cache_item.result is not None:
return cache_item.result
async with cache_item.lock:
if cache_item.result:
return cache_item.result
censor = Censor(Censor.SEARCH)
if kwargs.get('no_totals'):
response, offset, total = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
else:
response, offset, total = await self.search(**kwargs)
censor.apply(response)
total_referenced.extend(response)
if censor.censored:
response, _, _ = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
total_referenced.extend(response)
result = Outputs.to_base64(
response, await self._get_referenced_rows(total_referenced), offset, total, censor
)
cache_item.result = result
return result
return Outputs.to_base64(response, await self._get_referenced_rows(total_referenced), offset, total, censor)
async def resolve(self, *urls):
censor = Censor(Censor.RESOLVE)
results = [await self.resolve_url(url) for url in urls]
# just heat the cache
await self.populate_claim_cache(*filter(lambda x: isinstance(x, str), results))
results = [self._get_from_cache_or_error(url, result) for url, result in zip(urls, results)]
censored = [
result if not isinstance(result, dict) or not censor.censor(result)
else ResolveCensoredError(url, result['censoring_channel_hash'])
for url, result in zip(urls, results)
]
return results, censored, censor
def _get_from_cache_or_error(self, url: str, resolution: Union[LookupError, StreamResolution, ChannelResolution]):
cached = self.claim_cache.get(resolution)
return cached or (resolution if isinstance(resolution, LookupError) else resolution.lookup_error(url))
async def get_many(self, *claim_ids):
await self.populate_claim_cache(*claim_ids)
return filter(None, map(self.claim_cache.get, claim_ids))
async def populate_claim_cache(self, *claim_ids):
missing = [claim_id for claim_id in claim_ids if self.claim_cache.get(claim_id) is None]
if missing:
results = await self.search_client.mget(
index=self.index, body={"ids": missing}
)
for result in expand_result(filter(lambda doc: doc['found'], results["docs"])):
self.claim_cache.set(result['claim_id'], result)
async def full_id_from_short_id(self, name, short_id, channel_id=None):
key = '#'.join((channel_id or '', name, short_id))
if key not in self.short_id_cache:
query = {'name': name, 'claim_id': short_id}
if channel_id:
query['channel_id'] = channel_id
query['order_by'] = ['^channel_join']
query['signature_valid'] = True
else:
query['order_by'] = '^creation_height'
result, _, _ = await self.search(**query, limit=1)
if len(result) == 1:
result = result[0]['claim_id']
self.short_id_cache[key] = result
return self.short_id_cache.get(key, None)
async def search(self, **kwargs):
if 'channel' in kwargs:
kwargs['channel_id'] = await self.resolve_url(kwargs.pop('channel'))
if not kwargs['channel_id'] or not isinstance(kwargs['channel_id'], str):
return [], 0, 0
try:
if 'limit_claims_per_channel' in kwargs:
return await self.search_ahead(**kwargs), 0, 0
else:
result = (await self.search_client.search(
expand_query(**kwargs), index=self.index,
track_total_hits=False if kwargs.get('no_totals') else 10_000
))['hits']
except NotFoundError:
return [], 0, 0
return expand_result(result['hits']), 0, result.get('total', {}).get('value', 0)
async def search_ahead(self, **kwargs):
# 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return
per_channel_per_page = kwargs.pop('limit_claims_per_channel')
page_size = kwargs.pop('limit', 10)
offset = kwargs.pop('offset', 0)
kwargs['limit'] = 1000
cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache)
if cache_item.result is not None:
reordered_hits = cache_item.result
else:
async with cache_item.lock:
if cache_item.result:
reordered_hits = cache_item.result
else:
query = expand_query(**kwargs)
reordered_hits = await self.__search_ahead(query, page_size, per_channel_per_page)
cache_item.result = reordered_hits
return list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)])))
async def __search_ahead(self, query: dict, page_size: int, per_channel_per_page: int):
search_hits = deque((await self.search_client.search(
query, index=self.index, track_total_hits=False, _source_includes=['_id', 'channel_id']
))['hits']['hits'])
reordered_hits = []
channel_counters = Counter()
next_page_hits_maybe_check_later = deque()
while search_hits or next_page_hits_maybe_check_later:
if reordered_hits and len(reordered_hits) % page_size == 0:
channel_counters.clear()
elif not reordered_hits:
pass
else:
break # means last page was incomplete and we are left with bad replacements
for _ in range(len(next_page_hits_maybe_check_later)):
claim_id, channel_id = next_page_hits_maybe_check_later.popleft()
if channel_counters[channel_id] < per_channel_per_page:
reordered_hits.append((claim_id, channel_id))
channel_counters[channel_id] += 1
else:
next_page_hits_maybe_check_later.append((claim_id, channel_id))
while search_hits:
hit = search_hits.popleft()
hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id']
if hit_channel_id is None:
reordered_hits.append((hit_id, hit_channel_id))
elif channel_counters[hit_channel_id] < per_channel_per_page:
reordered_hits.append((hit_id, hit_channel_id))
channel_counters[hit_channel_id] += 1
if len(reordered_hits) % page_size == 0:
break
else:
next_page_hits_maybe_check_later.append((hit_id, hit_channel_id))
return reordered_hits
async def resolve_url(self, raw_url):
if raw_url not in self.resolution_cache:
self.resolution_cache[raw_url] = await self._resolve_url(raw_url)
return self.resolution_cache[raw_url]
async def _resolve_url(self, raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
return e
stream = LookupError(f'Could not find claim at "{raw_url}".')
channel_id = await self.resolve_channel_id(url)
if isinstance(channel_id, LookupError):
return channel_id
stream = (await self.resolve_stream(url, channel_id if isinstance(channel_id, str) else None)) or stream
if url.has_stream:
return StreamResolution(stream)
else:
return ChannelResolution(channel_id)
async def resolve_channel_id(self, url: URL):
if not url.has_channel:
return
if url.channel.is_fullid:
return url.channel.claim_id
if url.channel.is_shortid:
channel_id = await self.full_id_from_short_id(url.channel.name, url.channel.claim_id)
if not channel_id:
return LookupError(f'Could not find channel in "{url}".')
return channel_id
query = url.channel.to_dict()
if set(query) == {'name'}:
query['is_controlling'] = True
else:
query['order_by'] = ['^creation_height']
matches, _, _ = await self.search(**query, limit=1)
if matches:
channel_id = matches[0]['claim_id']
else:
return LookupError(f'Could not find channel in "{url}".')
return channel_id
async def resolve_stream(self, url: URL, channel_id: str = None):
if not url.has_stream:
return None
if url.has_channel and channel_id is None:
return None
query = url.stream.to_dict()
if url.stream.claim_id is not None:
if url.stream.is_fullid:
claim_id = url.stream.claim_id
else:
claim_id = await self.full_id_from_short_id(query['name'], query['claim_id'], channel_id)
return claim_id
if channel_id is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount', '^height']
else:
query['order_by'] = ['^channel_join']
query['channel_id'] = channel_id
query['signature_valid'] = True
elif set(query) == {'name'}:
query['is_controlling'] = True
matches, _, _ = await self.search(**query, limit=1)
if matches:
return matches[0]['claim_id']
async def _get_referenced_rows(self, txo_rows: List[dict]):
txo_rows = [row for row in txo_rows if isinstance(row, dict)]
referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows)))
referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows)))
referenced_ids |= set(map(parse_claim_id, filter(None, (row['censoring_channel_hash'] for row in txo_rows))))
referenced_txos = []
if referenced_ids:
referenced_txos.extend(await self.get_many(*referenced_ids))
referenced_ids = set(filter(None, (row['channel_id'] for row in referenced_txos)))
if referenced_ids:
referenced_txos.extend(await self.get_many(*referenced_ids))
return referenced_txos
def extract_doc(doc, index):
doc['claim_id'] = doc.pop('claim_hash')[::-1].hex()
if doc['reposted_claim_hash'] is not None:
doc['reposted_claim_id'] = doc.pop('reposted_claim_hash')[::-1].hex()
else:
doc['reposted_claim_id'] = None
channel_hash = doc.pop('channel_hash')
doc['channel_id'] = channel_hash[::-1].hex() if channel_hash else channel_hash
channel_hash = doc.pop('censoring_channel_hash')
doc['censoring_channel_hash'] = channel_hash[::-1].hex() if channel_hash else channel_hash
txo_hash = doc.pop('txo_hash')
doc['tx_id'] = txo_hash[:32][::-1].hex()
doc['tx_nout'] = struct.unpack('<I', txo_hash[32:])[0]
doc['is_controlling'] = bool(doc['is_controlling'])
doc['signature'] = (doc.pop('signature') or b'').hex() or None
doc['signature_digest'] = (doc.pop('signature_digest') or b'').hex() or None
doc['public_key_bytes'] = (doc.pop('public_key_bytes') or b'').hex() or None
doc['public_key_hash'] = (doc.pop('public_key_hash') or b'').hex() or None
doc['signature_valid'] = bool(doc['signature_valid'])
doc['claim_type'] = doc.get('claim_type', 0) or 0
doc['stream_type'] = int(doc.get('stream_type', 0) or 0)
doc['has_source'] = bool(doc['has_source'])
return {'doc': doc, '_id': doc['claim_id'], '_index': index, '_op_type': 'update', 'doc_as_upsert': True}
def expand_query(**kwargs):
if "amount_order" in kwargs:
kwargs["limit"] = 1
kwargs["order_by"] = "effective_amount"
kwargs["offset"] = int(kwargs["amount_order"]) - 1
if 'name' in kwargs:
kwargs['name'] = normalize_name(kwargs.pop('name'))
if kwargs.get('is_controlling') is False:
kwargs.pop('is_controlling')
query = {'must': [], 'must_not': []}
collapse = None
for key, value in kwargs.items():
key = key.replace('claim.', '')
many = key.endswith('__in') or isinstance(value, list)
if many:
key = key.replace('__in', '')
value = list(filter(None, value))
if value is None or isinstance(value, list) and len(value) == 0:
continue
key = REPLACEMENTS.get(key, key)
if key in FIELDS:
partial_id = False
if key == 'claim_type':
if isinstance(value, str):
value = CLAIM_TYPES[value]
else:
value = [CLAIM_TYPES[claim_type] for claim_type in value]
if key == '_id':
if isinstance(value, Iterable):
value = [item[::-1].hex() for item in value]
else:
value = value[::-1].hex()
if not many and key in ('_id', 'claim_id') and len(value) < 20:
partial_id = True
if key == 'public_key_id':
key = 'public_key_hash'
value = Base58.decode(value)[1:21].hex()
if key == 'signature_valid':
continue # handled later
if key in TEXT_FIELDS:
key += '.keyword'
ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'}
if partial_id:
query['must'].append({"prefix": {"claim_id": value}})
elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops:
operator_length = 2 if value[:2] in ops else 1
operator, value = value[:operator_length], value[operator_length:]
if key == 'fee_amount':
value = str(Decimal(value)*1000)
query['must'].append({"range": {key: {ops[operator]: value}}})
elif many:
query['must'].append({"terms": {key: value}})
else:
if key == 'fee_amount':
value = str(Decimal(value)*1000)
query['must'].append({"term": {key: {"value": value}}})
elif key == 'not_channel_ids':
for channel_id in value:
query['must_not'].append({"term": {'channel_id.keyword': channel_id}})
query['must_not'].append({"term": {'_id': channel_id}})
elif key == 'channel_ids':
query['must'].append({"terms": {'channel_id.keyword': value}})
elif key == 'claim_ids':
query['must'].append({"terms": {'claim_id.keyword': value}})
elif key == 'media_types':
query['must'].append({"terms": {'media_type.keyword': value}})
elif key == 'stream_types':
query['must'].append({"terms": {'stream_type': [STREAM_TYPES[stype] for stype in value]}})
elif key == 'any_languages':
query['must'].append({"terms": {'languages': clean_tags(value)}})
elif key == 'any_languages':
query['must'].append({"terms": {'languages': value}})
elif key == 'all_languages':
query['must'].extend([{"term": {'languages': tag}} for tag in value])
elif key == 'any_tags':
query['must'].append({"terms": {'tags.keyword': clean_tags(value)}})
elif key == 'all_tags':
query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
elif key == 'not_tags':
query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
elif key == 'not_claim_id':
query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value])
elif key == 'limit_claims_per_channel':
collapse = ('channel_id.keyword', value)
if kwargs.get('has_channel_signature'):
query['must'].append({"exists": {"field": "signature_digest"}})
if 'signature_valid' in kwargs:
query['must'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
elif 'signature_valid' in kwargs:
query.setdefault('should', [])
query["minimum_should_match"] = 1
query['should'].append({"bool": {"must_not": {"exists": {"field": "signature_digest"}}}})
query['should'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
if 'has_source' in kwargs:
query.setdefault('should', [])
query["minimum_should_match"] = 1
is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}}
query['should'].append(
{"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}})
query['should'].append({"bool": {"must_not": [is_stream_or_repost]}})
if kwargs.get('text'):
query['must'].append(
{"simple_query_string":
{"query": kwargs["text"], "fields": [
"claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5"
]}})
query = {
"_source": {"excludes": ["description", "title"]},
'query': {'bool': query},
"sort": [],
}
if "limit" in kwargs:
query["size"] = kwargs["limit"]
if 'offset' in kwargs:
query["from"] = kwargs["offset"]
if 'order_by' in kwargs:
if isinstance(kwargs["order_by"], str):
kwargs["order_by"] = [kwargs["order_by"]]
for value in kwargs['order_by']:
if 'trending_group' in value:
# fixme: trending_mixed is 0 for all records on variable decay, making sort slow.
continue
is_asc = value.startswith('^')
value = value[1:] if is_asc else value
value = REPLACEMENTS.get(value, value)
if value in TEXT_FIELDS:
value += '.keyword'
query['sort'].append({value: "asc" if is_asc else "desc"})
if collapse:
query["collapse"] = {
"field": collapse[0],
"inner_hits": {
"name": collapse[0],
"size": collapse[1],
"sort": query["sort"]
}
}
return query
def expand_result(results):
inner_hits = []
expanded = []
for result in results:
if result.get("inner_hits"):
for _, inner_hit in result["inner_hits"].items():
inner_hits.extend(inner_hit["hits"]["hits"])
continue
result = result['_source']
result['claim_hash'] = unhexlify(result['claim_id'])[::-1]
if result['reposted_claim_id']:
result['reposted_claim_hash'] = unhexlify(result['reposted_claim_id'])[::-1]
else:
result['reposted_claim_hash'] = None
result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None
result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack('<I', result['tx_nout'])
result['tx_hash'] = unhexlify(result['tx_id'])[::-1]
if result['censoring_channel_hash']:
result['censoring_channel_hash'] = unhexlify(result['censoring_channel_hash'])[::-1]
expanded.append(result)
if inner_hits:
return expand_result(inner_hits)
return expanded
class ResultCacheItem:
__slots__ = '_result', 'lock', 'has_result'
def __init__(self):
self.has_result = asyncio.Event()
self.lock = asyncio.Lock()
self._result = None
@property
def result(self) -> str:
return self._result
@result.setter
def result(self, result: str):
self._result = result
if result is not None:
self.has_result.set()
@classmethod
def from_cache(cls, cache_key, cache):
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
return cache_item
+114
View File
@@ -0,0 +1,114 @@
import argparse
import asyncio
import logging
import os
from collections import namedtuple
from multiprocessing import Process
import apsw
from elasticsearch import AsyncElasticsearch
from elasticsearch.helpers import async_bulk
from lbry.wallet.server.env import Env
from lbry.wallet.server.coin import LBC
from lbry.wallet.server.db.elasticsearch.search import extract_doc, SearchIndex
INDEX = 'claims'
async def get_all(db, shard_num, shards_total, limit=0):
logging.info("shard %d starting", shard_num)
def exec_factory(cursor, statement, bindings):
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
cursor.setrowtrace(lambda cursor, row: tpl(*row))
return True
db.setexectrace(exec_factory)
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
for num, claim in enumerate(db.execute(f"""
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
WHERE claim.height % {shards_total} = {shard_num}
ORDER BY claim.height desc
""")):
claim = dict(claim._asdict())
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
claim['censor_type'] = 0
claim['censoring_channel_hash'] = None
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
if num % 10_000 == 0:
logging.info("%d/%d", num, total)
yield extract_doc(claim, INDEX)
if 0 < limit <= num:
break
async def consume(producer):
env = Env(LBC)
logging.info("ES sync host: %s:%i", env.elastic_host, env.elastic_port)
es = AsyncElasticsearch([{'host': env.elastic_host, 'port': env.elastic_port}])
try:
await async_bulk(es, producer, request_timeout=120)
await es.indices.refresh(index=INDEX)
finally:
await es.close()
async def make_es_index():
env = Env(LBC)
index = SearchIndex('', elastic_host=env.elastic_host, elastic_port=env.elastic_port)
try:
return await index.start()
finally:
index.stop()
async def run(args, shard):
def itsbusy(*_):
logging.info("shard %d: db is busy, retry", shard)
return True
db = apsw.Connection(args.db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
db.setbusyhandler(itsbusy)
db.cursor().execute('pragma journal_mode=wal;')
db.cursor().execute('pragma temp_store=memory;')
producer = get_all(db.cursor(), shard, args.clients, limit=args.blocks)
await asyncio.gather(*(consume(producer) for _ in range(min(8, args.clients))))
def __run(args, shard):
asyncio.run(run(args, shard))
def run_elastic_sync():
logging.basicConfig(level=logging.INFO)
logging.getLogger('aiohttp').setLevel(logging.WARNING)
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
logging.info('lbry.server starting')
parser = argparse.ArgumentParser(prog="lbry-hub-elastic-sync")
parser.add_argument("db_path", type=str)
parser.add_argument("-c", "--clients", type=int, default=16)
parser.add_argument("-b", "--blocks", type=int, default=0)
parser.add_argument("-f", "--force", default=False, action='store_true')
args = parser.parse_args()
processes = []
if not args.force and not os.path.exists(args.db_path):
logging.info("DB path doesnt exist")
return
if not args.force and not asyncio.run(make_es_index()):
logging.info("ES is already initialized")
return
for i in range(args.clients):
processes.append(Process(target=__run, args=(args, i)))
processes[-1].start()
for process in processes:
process.join()
process.close()
-52
View File
@@ -1,52 +0,0 @@
from lbry.wallet.database import constraints_to_sql
CREATE_FULL_TEXT_SEARCH = """
create virtual table if not exists search using fts5(
claim_name, channel_name, title, description, author, tags,
content=claim, tokenize=porter
);
"""
FTS_ORDER_BY = "bm25(search, 4.0, 8.0, 1.0, 0.5, 1.0, 0.5)"
def fts_action_sql(claims=None, action='insert'):
select = {
'rowid': "claim.rowid",
'claim_name': "claim.normalized",
'channel_name': "channel.normalized",
'title': "claim.title",
'description': "claim.description",
'author': "claim.author",
'tags': "(select group_concat(tag, ' ') from tag where tag.claim_hash=claim.claim_hash)"
}
if action == 'delete':
select['search'] = '"delete"'
where, values = "", {}
if claims:
where, values = constraints_to_sql({'claim.claim_hash__in': claims})
where = 'WHERE '+where
return f"""
INSERT INTO search ({','.join(select.keys())})
SELECT {','.join(select.values())} FROM claim
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash) {where}
""", values
def update_full_text_search(action, outputs, db, is_first_sync):
if is_first_sync:
return
if not outputs:
return
if action in ("before-delete", "before-update"):
db.execute(*fts_action_sql(outputs, 'delete'))
elif action in ("after-insert", "after-update"):
db.execute(*fts_action_sql(outputs, 'insert'))
else:
raise ValueError(f"Invalid action for updating full text search: '{action}'")
def first_sync_finished(db):
db.execute(*fts_action_sql())
+323 -269
View File
@@ -1,18 +1,17 @@
"""
Delayed AR with variable decay rate.
The spike height function is also simpler.
AR-like trending with a delayed effect and a faster
decay rate for high valued claims.
"""
import copy
import math
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Whale threshold, in LBC (higher -> less DB writing)
WHALE_THRESHOLD = 10000.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
@@ -38,6 +37,7 @@ def install(connection):
Install the trending algorithm.
"""
check_trending_values(connection)
trending_data.initialise(connection.cursor())
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
@@ -46,7 +46,6 @@ def install(connection):
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
@@ -60,19 +59,304 @@ def check_trending_values(connection):
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True, end="")
print("Resetting some columns. This might take a while...", flush=True,
end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def spike_height(x, x_old):
def trending_log(s):
"""
Compute the size of a trending spike (normed - constant units).
Log a string to the log file
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
def trending_unit(height):
"""
Return the trending score unit at a given height.
"""
# Round to the beginning of a SAVE_INTERVAL batch of blocks.
_height = height - (height % SAVE_INTERVAL)
return 1.0/DECAY**(height % RENORM_INTERVAL)
class TrendingDB:
"""
An in-memory database of trending scores
"""
def __init__(self):
self.conn = apsw.Connection(":memory:")
self.cursor = self.conn.cursor()
self.initialised = False
self.write_needed = set()
def execute(self, query, *args, **kwargs):
return self.cursor.execute(query, *args, **kwargs)
def executemany(self, query, *args, **kwargs):
return self.cursor.executemany(query, *args, **kwargs)
def begin(self):
self.execute("BEGIN;")
def commit(self):
self.execute("COMMIT;")
def initialise(self, db):
"""
Pass in claims.db
"""
if self.initialised:
return
trending_log("Initialising trending database...")
# The need for speed
self.execute("PRAGMA JOURNAL_MODE=OFF;")
self.execute("PRAGMA SYNCHRONOUS=0;")
self.begin()
# Create the tables
self.execute("""
CREATE TABLE IF NOT EXISTS claims
(claim_hash BYTES PRIMARY KEY,
lbc REAL NOT NULL DEFAULT 0.0,
trending_score REAL NOT NULL DEFAULT 0.0)
WITHOUT ROWID;""")
self.execute("""
CREATE TABLE IF NOT EXISTS spikes
(id INTEGER PRIMARY KEY,
claim_hash BYTES NOT NULL,
height INTEGER NOT NULL,
mass REAL NOT NULL,
FOREIGN KEY (claim_hash)
REFERENCES claims (claim_hash));""")
# Clear out any existing data
self.execute("DELETE FROM claims;")
self.execute("DELETE FROM spikes;")
# Create indexes
self.execute("CREATE INDEX idx1 ON spikes (claim_hash, height, mass);")
self.execute("CREATE INDEX idx2 ON spikes (claim_hash, height, mass DESC);")
self.execute("CREATE INDEX idx3 on claims (lbc DESC, claim_hash, trending_score);")
# Import data from claims.db
for row in db.execute("""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc,
trending_mixed
FROM claim;
"""):
self.execute("INSERT INTO claims VALUES (?, ?, ?);", row)
self.commit()
self.initialised = True
trending_log("done.\n")
def apply_spikes(self, height):
"""
Apply spikes that are due. This occurs inside a transaction.
"""
spikes = []
unit = trending_unit(height)
for row in self.execute("""
SELECT SUM(mass), claim_hash FROM spikes
WHERE height = ?
GROUP BY claim_hash;
""", (height, )):
spikes.append((row[0]*unit, row[1]))
self.write_needed.add(row[1])
self.executemany("""
UPDATE claims
SET trending_score = (trending_score + ?)
WHERE claim_hash = ?;
""", spikes)
self.execute("DELETE FROM spikes WHERE height = ?;", (height, ))
def decay_whales(self, height):
"""
Occurs inside transaction.
"""
if height % SAVE_INTERVAL != 0:
return
whales = self.execute("""
SELECT trending_score, lbc, claim_hash
FROM claims
WHERE lbc >= ?;
""", (WHALE_THRESHOLD, )).fetchall()
whales2 = []
for whale in whales:
trending, lbc, claim_hash = whale
# Overall multiplication factor for decay rate
# At WHALE_THRESHOLD, this is 1
# At 10*WHALE_THRESHOLD, it is 3
decay_rate_factor = 1.0 + 2.0*math.log10(lbc/WHALE_THRESHOLD)
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# Decay
trending *= factor
whales2.append((trending, claim_hash))
self.write_needed.add(claim_hash)
self.executemany("UPDATE claims SET trending_score=? WHERE claim_hash=?;",
whales2)
def renorm(self, height):
"""
Renormalise trending scores. Occurs inside a transaction.
"""
if height % RENORM_INTERVAL == 0:
threshold = 1.0E-3/DECAY_PER_RENORM
for row in self.execute("""SELECT claim_hash FROM claims
WHERE ABS(trending_score) >= ?;""",
(threshold, )):
self.write_needed.add(row[0])
self.execute("""UPDATE claims SET trending_score = ?*trending_score
WHERE ABS(trending_score) >= ?;""",
(DECAY_PER_RENORM, threshold))
def write_to_claims_db(self, db, height):
"""
Write changed trending scores to claims.db.
"""
if height % SAVE_INTERVAL != 0:
return
rows = self.execute(f"""
SELECT trending_score, claim_hash
FROM claims
WHERE claim_hash IN
({','.join('?' for _ in self.write_needed)});
""", self.write_needed).fetchall()
db.executemany("""UPDATE claim SET trending_mixed = ?
WHERE claim_hash = ?;""", rows)
# Clear list of claims needing to be written to claims.db
self.write_needed = set()
def update(self, db, height, recalculate_claim_hashes):
"""
Update trending scores.
Input is a cursor to claims.db, the block height, and the list of
claims that changed.
"""
assert self.initialised
self.begin()
self.renorm(height)
# Fetch changed/new claims from claims.db
for row in db.execute(f"""
SELECT claim_hash,
1E-8*(amount + support_amount) AS lbc
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
claim_hash, lbc = row
# Insert into trending db if it does not exist
self.execute("""
INSERT INTO claims (claim_hash)
VALUES (?)
ON CONFLICT (claim_hash) DO NOTHING;""",
(claim_hash, ))
# See if it was an LBC change
old = self.execute("SELECT * FROM claims WHERE claim_hash=?;",
(claim_hash, )).fetchone()
lbc_old = old[1]
# Save new LBC value into trending db
self.execute("UPDATE claims SET lbc = ? WHERE claim_hash = ?;",
(lbc, claim_hash))
if lbc > lbc_old:
# Schedule a future spike
delay = min(int((lbc + 1E-8)**0.4), HALF_LIFE)
spike = (claim_hash, height + delay, spike_mass(lbc, lbc_old))
self.execute("""INSERT INTO spikes
(claim_hash, height, mass)
VALUES (?, ?, ?);""", spike)
elif lbc < lbc_old:
# Subtract from future spikes
penalty = spike_mass(lbc_old, lbc)
spikes = self.execute("""
SELECT * FROM spikes
WHERE claim_hash = ?
ORDER BY height ASC, mass DESC;
""", (claim_hash, )).fetchall()
for spike in spikes:
spike_id, mass = spike[0], spike[3]
if mass > penalty:
# The entire penalty merely reduces this spike
self.execute("UPDATE spikes SET mass=? WHERE id=?;",
(mass - penalty, spike_id))
penalty = 0.0
else:
# Removing this spike entirely accounts for some (or
# all) of the penalty, then move on to other spikes
self.execute("DELETE FROM spikes WHERE id=?;",
(spike_id, ))
penalty -= mass
# If penalty remains, that's a negative spike to be applied
# immediately.
if penalty > 0.0:
self.execute("""
INSERT INTO spikes (claim_hash, height, mass)
VALUES (?, ?, ?);""",
(claim_hash, height, -penalty))
self.apply_spikes(height)
self.decay_whales(height)
self.commit()
self.write_to_claims_db(db, height)
# The "global" instance to work with
# pylint: disable=C0103
trending_data = TrendingDB()
def spike_mass(x, x_old):
"""
Compute the mass of a trending spike (normed - constant units).
x_old = old LBC value
x = new LBC value
"""
# Sign of trending spike
@@ -89,155 +373,16 @@ def spike_height(x, x_old):
return sign*mag
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log(f"Skipping trending calculations at block {height}.\n")
return
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
start = time.time()
trending_log(f"Calculating variable_decay trending at block {height}.\n")
trending_data.update(db, height, recalculate_claim_hashes)
end = time.time()
trending_log(f"Trending operations took {end - start} seconds.\n\n")
def test_trending():
"""
@@ -260,12 +405,12 @@ def test_trending():
COMMIT;
""")
# Initialise trending data before anything happens with the claims
trending_data.initialise(db)
# Insert initial states of claims
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
everything = {"huge_whale": 0.01, "medium_whale": 0.01, "small_whale": 0.01,
"huge_whale_botted": 0.01, "minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
@@ -277,13 +422,17 @@ def test_trending():
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
# Process block zero
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
for row in trending_data.execute("""
SELECT claim_hash, trending_score
FROM claims;
"""):
trajectories[row[0]] = [row[1]/trending_unit(height)]
# Main loop
for height in range(1, 1000):
@@ -312,119 +461,24 @@ def test_trending():
# Call run()
run(db, height, height, everything.keys())
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
# Append current trending scores to trajectories
for row in db.execute("""
SELECT claim_hash, trending_mixed
FROM claim;
"""):
trajectories[row[0]].append(row[1]/trending_unit(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trending_data.claims:
for key in trajectories:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
+129 -33
View File
@@ -1,12 +1,12 @@
import os
import apsw
from typing import Union, Tuple, Set, List
from itertools import chain
from decimal import Decimal
from collections import namedtuple
from multiprocessing import Manager
from binascii import unhexlify
from binascii import unhexlify, hexlify
from lbry.wallet.server.leveldb import LevelDB
from lbry.wallet.server.util import class_logger
from lbry.wallet.database import query, constraints_to_sql
@@ -16,11 +16,10 @@ from lbry.schema.mime_types import guess_stream_type
from lbry.wallet import Ledger, RegTestLedger
from lbry.wallet.transaction import Transaction, Output
from lbry.wallet.server.db.canonical import register_canonical_functions
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
from lbry.wallet.server.db.elasticsearch import SearchIndex
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
@@ -56,6 +55,7 @@ class SQLDB:
description text,
claim_type integer,
has_source bool,
reposted integer default 0,
-- streams
@@ -117,6 +117,15 @@ class SQLDB:
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_LANGUAGE_TABLE = """
create table if not exists language (
language text not null,
claim_hash bytes not null,
height integer not null
);
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
"""
CREATE_CLAIMTRIE_TABLE = """
create table if not exists claimtrie (
normalized text primary key,
@@ -126,6 +135,22 @@ class SQLDB:
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
CREATE_CHANGELOG_TRIGGER = """
create table if not exists changelog (
claim_hash bytes primary key
);
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
create trigger if not exists claim_changelog after update on claim
begin
insert or ignore into changelog (claim_hash) values (new.claim_hash);
end;
create trigger if not exists claimtrie_changelog after update on claimtrie
begin
insert or ignore into changelog (claim_hash) values (new.claim_hash);
insert or ignore into changelog (claim_hash) values (old.claim_hash);
end;
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
@@ -174,12 +199,18 @@ class SQLDB:
for tag_value, tag_key in COMMON_TAGS.items()
)
LANGUAGE_INDEXES = '\n'.join(
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
for language in INDEXED_LANGUAGES
)
CREATE_TABLES_QUERY = (
CREATE_CLAIM_TABLE +
CREATE_FULL_TEXT_SEARCH +
CREATE_SUPPORT_TABLE +
CREATE_CLAIMTRIE_TABLE +
CREATE_TAG_TABLE
CREATE_TAG_TABLE +
CREATE_CHANGELOG_TRIGGER +
CREATE_LANGUAGE_TABLE
)
def __init__(
@@ -189,7 +220,6 @@ class SQLDB:
self.db = None
self.logger = class_logger(__name__, self.__class__.__name__)
self.ledger = Ledger if main.coin.NET == 'mainnet' else RegTestLedger
self._fts_synced = False
self.state_manager = None
self.blocked_streams = None
self.blocked_channels = None
@@ -202,6 +232,7 @@ class SQLDB:
unhexlify(channel_id)[::-1] for channel_id in filtering_channels if channel_id
}
self.trending = trending
self.pending_deletes = set()
def open(self):
self.db = apsw.Connection(
@@ -305,7 +336,7 @@ class SQLDB:
self.execute('commit;')
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
claim_hashes, claims, tags = set(), [], {}
claim_hashes, claims, tags, languages = set(), [], {}, {}
for txo in txos:
tx = txo.tx_ref.tx
@@ -316,6 +347,13 @@ class SQLDB:
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
continue
language = 'none'
try:
if txo.claim.is_stream and txo.claim.stream.languages:
language = txo.claim.stream.languages[0].language
except:
pass
claim_hash = txo.claim_hash
claim_hashes.add(claim_hash)
claim_record = {
@@ -333,6 +371,7 @@ class SQLDB:
'author': None,
'duration': None,
'claim_type': None,
'has_source': False,
'stream_type': None,
'media_type': None,
'release_time': None,
@@ -350,6 +389,7 @@ class SQLDB:
if claim.is_stream:
claim_record['claim_type'] = CLAIM_TYPES['stream']
claim_record['has_source'] = claim.stream.has_source
claim_record['media_type'] = claim.stream.source.media_type
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
claim_record['title'] = claim.stream.title
@@ -372,6 +412,10 @@ class SQLDB:
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
elif claim.is_collection:
claim_record['claim_type'] = CLAIM_TYPES['collection']
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
@@ -383,6 +427,10 @@ class SQLDB:
self.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
)
if languages:
self.executemany(
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
)
return claims
@@ -390,14 +438,14 @@ class SQLDB:
claims = self._upsertable_claims(txos, header)
if claims:
self.executemany("""
INSERT OR IGNORE INTO claim (
INSERT OR REPLACE INTO claim (
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
claim_type, media_type, stream_type, timestamp, creation_timestamp,
claim_type, media_type, stream_type, timestamp, creation_timestamp, has_source,
fee_currency, fee_amount, title, description, author, duration, height, reposted_claim_hash,
creation_height, release_time, activation_height, expiration_height, short_url)
VALUES (
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
:claim_type, :media_type, :stream_type, :timestamp, :timestamp,
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :has_source,
:fee_currency, :fee_amount, :title, :description, :author, :duration, :height, :reposted_claim_hash, :height,
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
@@ -415,7 +463,7 @@ class SQLDB:
UPDATE claim SET
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency,
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency, has_source=:has_source,
title=:title, duration=:duration, description=:description, author=:author, reposted_claim_hash=:reposted_claim_hash,
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
WHERE claim_hash=:claim_hash;
@@ -499,6 +547,7 @@ class SQLDB:
WHERE claim_hash = ?
""", targets
)
return {target[0] for target in targets}
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, affected_channels, timer):
if not new_claims and not updated_claims and not spent_claims:
@@ -665,7 +714,7 @@ class SQLDB:
claim_in_channel.channel_hash=claim.claim_hash
)
WHERE claim_hash = ?
""", [(channel_hash,) for channel_hash in all_channel_keys.keys()])
""", [(channel_hash,) for channel_hash in all_channel_keys])
sub_timer.stop()
sub_timer = timer.add_timer('update blocked claims list')
@@ -772,11 +821,56 @@ class SQLDB:
f"SELECT claim_hash, normalized FROM claim WHERE expiration_height = {height}"
)
def enqueue_changes(self):
for claim in self.execute(f"""
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
"""):
claim = claim._asdict()
id_set = set(filter(None, (claim['claim_hash'], claim['channel_hash'], claim['reposted_claim_hash'])))
claim['censor_type'] = 0
claim['censoring_channel_hash'] = None
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
for reason_id in id_set:
if reason_id in self.blocked_streams:
claim['censor_type'] = 2
claim['censoring_channel_hash'] = self.blocked_streams.get(reason_id)
elif reason_id in self.blocked_channels:
claim['censor_type'] = 2
claim['censoring_channel_hash'] = self.blocked_channels.get(reason_id)
elif reason_id in self.filtered_streams:
claim['censor_type'] = 1
claim['censoring_channel_hash'] = self.filtered_streams.get(reason_id)
elif reason_id in self.filtered_channels:
claim['censor_type'] = 1
claim['censoring_channel_hash'] = self.filtered_channels.get(reason_id)
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
yield 'update', claim
def clear_changelog(self):
self.execute("delete from changelog;")
def claim_producer(self):
while self.pending_deletes:
claim_hash = self.pending_deletes.pop()
yield 'delete', hexlify(claim_hash[::-1]).decode()
for claim in self.enqueue_changes():
yield claim
self.clear_changelog()
def advance_txs(self, height, all_txs, header, daemon_height, timer):
insert_claims = []
update_claims = []
update_claim_hashes = set()
delete_claim_hashes = set()
delete_claim_hashes = self.pending_deletes
insert_supports = []
delete_support_txo_hashes = set()
recalculate_claim_hashes = set() # added/deleted supports, added/updated claim
@@ -845,28 +939,17 @@ class SQLDB:
expire_timer.stop()
r = timer.run
r(update_full_text_search, 'before-delete',
delete_claim_hashes, self.db.cursor(), self.main.first_sync)
affected_channels = r(self.delete_claims, delete_claim_hashes)
r(self.delete_supports, delete_support_txo_hashes)
r(self.insert_claims, insert_claims, header)
r(self.calculate_reposts, insert_claims)
r(update_full_text_search, 'after-insert',
[txo.claim_hash for txo in insert_claims], self.db.cursor(), self.main.first_sync)
r(update_full_text_search, 'before-update',
[txo.claim_hash for txo in update_claims], self.db.cursor(), self.main.first_sync)
r(self.update_claims, update_claims, header)
r(update_full_text_search, 'after-update',
[txo.claim_hash for txo in update_claims], self.db.cursor(), self.main.first_sync)
r(self.validate_channel_signatures, height, insert_claims,
update_claims, delete_claim_hashes, affected_channels, forward_timer=True)
r(self.insert_supports, insert_supports)
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
for algorithm in self.trending:
r(algorithm.run, self.db.cursor(), height, daemon_height, recalculate_claim_hashes)
if not self._fts_synced and self.main.first_sync and height == daemon_height:
r(first_sync_finished, self.db.cursor())
self._fts_synced = True
class LBRYLevelDB(LevelDB):
@@ -878,17 +961,30 @@ class LBRYLevelDB(LevelDB):
for algorithm_name in self.env.trending_algorithms:
if algorithm_name in TRENDING_ALGORITHMS:
trending.append(TRENDING_ALGORITHMS[algorithm_name])
self.sql = SQLDB(
self, path,
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
trending
if self.env.es_mode == 'reader':
self.logger.info('Index mode: reader')
self.sql = None
else:
self.logger.info('Index mode: writer. Using SQLite db to sync ES')
self.sql = SQLDB(
self, path,
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
trending
)
# Search index
self.search_index = SearchIndex(
self.env.es_index_prefix, self.env.database_query_timeout, self.env.elastic_host, self.env.elastic_port
)
def close(self):
super().close()
self.sql.close()
if self.sql:
self.sql.close()
async def _open_dbs(self, *args, **kwargs):
await self.search_index.start()
await super()._open_dbs(*args, **kwargs)
self.sql.open()
if self.sql:
self.sql.open()
+9 -2
View File
@@ -33,11 +33,15 @@ class Env:
self.allow_root = self.boolean('ALLOW_ROOT', False)
self.host = self.default('HOST', 'localhost')
self.rpc_host = self.default('RPC_HOST', 'localhost')
self.elastic_host = self.default('ELASTIC_HOST', 'localhost')
self.elastic_port = self.integer('ELASTIC_PORT', 9200)
self.loop_policy = self.set_event_loop_policy()
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.default('DB_ENGINE', 'leveldb')
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
self.trending_algorithms = [
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
]
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
self.track_metrics = self.boolean('TRACK_METRICS', False)
@@ -51,10 +55,13 @@ class Env:
coin_name = self.required('COIN').strip()
network = self.default('NET', 'mainnet').strip()
self.coin = Coin.lookup_coin_class(coin_name, network)
self.es_index_prefix = self.default('ES_INDEX_PREFIX', '')
self.es_mode = self.default('ES_MODE', 'writer')
self.cache_MB = self.integer('CACHE_MB', 1200)
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
# Server stuff
self.tcp_port = self.integer('TCP_PORT', None)
self.udp_port = self.integer('UDP_PORT', self.tcp_port)
self.ssl_port = self.integer('SSL_PORT', None)
if self.ssl_port:
self.ssl_certfile = self.required('SSL_CERTFILE')
@@ -92,7 +99,7 @@ class Env:
self.identities = [identity
for identity in (clearnet_identity, tor_identity)
if identity is not None]
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 250)) / 1000.0
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 3000)) / 1000.0
@classmethod
def default(cls, envvar, default):
+4 -4
View File
@@ -67,17 +67,17 @@ def hash160(x):
return ripemd160(sha256(x))
def hash_to_hex_str(x):
def hash_to_hex_str(x: bytes) -> str:
"""Convert a big-endian binary hash to displayed hex string.
Display form of a binary hash is reversed and converted to hex.
"""
return bytes(reversed(x)).hex()
return x[::-1].hex()
def hex_str_to_hash(x):
def hex_str_to_hash(x: str) -> bytes:
"""Convert a displayed hex string to a binary hash."""
return bytes(reversed(hex_to_bytes(x)))
return hex_to_bytes(x)[::-1]
class Base58Error(Exception):
+14 -14
View File
@@ -173,20 +173,20 @@ class History:
self.logger.info(f'backing up removed {nremoves:,d} history entries')
def get_txnums(self, hashX, limit=1000):
"""Generator that returns an unpruned, sorted list of tx_nums in the
history of a hashX. Includes both spending and receiving
transactions. By default yields at most 1000 entries. Set
limit to None to get them all. """
limit = util.resolve_limit(limit)
for key, hist in self.db.iterator(prefix=hashX):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
if limit == 0:
return
yield tx_num
limit -= 1
# def get_txnums(self, hashX, limit=1000):
# """Generator that returns an unpruned, sorted list of tx_nums in the
# history of a hashX. Includes both spending and receiving
# transactions. By default yields at most 1000 entries. Set
# limit to None to get them all. """
# limit = util.resolve_limit(limit)
# for key, hist in self.db.iterator(prefix=hashX):
# a = array.array('I')
# a.frombytes(hist)
# for tx_num in a:
# if limit == 0:
# return
# yield tx_num
# limit -= 1
#
# History compaction
+188 -89
View File
@@ -12,17 +12,20 @@
import asyncio
import array
import ast
import base64
import os
import time
import zlib
import typing
from typing import Optional, List, Tuple, Iterable
from asyncio import sleep
from bisect import bisect_right
from collections import namedtuple
from functools import partial
from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.utils import LRUCacheWithMetrics
from lbry.wallet.server import util
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.merkle import Merkle, MerkleCache
@@ -35,6 +38,10 @@ UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
HEADER_PREFIX = b'H'
TX_COUNT_PREFIX = b'T'
TX_HASH_PREFIX = b'X'
TX_PREFIX = b'B'
TX_NUM_PREFIX = b'N'
BLOCK_HASH_PREFIX = b'C'
@attr.s(slots=True)
@@ -42,7 +49,8 @@ class FlushData:
height = attr.ib()
tx_count = attr.ib()
headers = attr.ib()
block_tx_hashes = attr.ib()
block_hashes = attr.ib()
block_txs = attr.ib()
# The following are flushed to the UTXO DB if undo_infos is not None
undo_infos = attr.ib()
adds = attr.ib()
@@ -74,6 +82,8 @@ class LevelDB:
self.history = History()
self.utxo_db = None
self.tx_counts = None
self.headers = None
self.encoded_headers = LRUCacheWithMetrics(1 << 21, metric_name='encoded_headers', namespace='wallet_server')
self.last_flush = time.time()
self.logger.info(f'using {self.env.db_engine} for DB backend')
@@ -82,9 +92,11 @@ class LevelDB:
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
self.hashes_db = None
self.headers_db = None
self.tx_count_db = None
self.tx_db = None
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
self.total_transactions = None
async def _read_tx_counts(self):
if self.tx_counts is not None:
@@ -95,7 +107,7 @@ class LevelDB:
def get_counts():
return tuple(
util.unpack_be_uint64(tx_count)
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
)
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
@@ -108,9 +120,32 @@ class LevelDB:
else:
assert self.db_tx_count == 0
async def _read_txids(self):
def get_txids():
return list(self.tx_db.iterator(prefix=TX_HASH_PREFIX, include_key=False))
start = time.perf_counter()
self.logger.info("loading txids")
txids = await asyncio.get_event_loop().run_in_executor(self.executor, get_txids)
assert len(txids) == len(self.tx_counts) == 0 or len(txids) == self.tx_counts[-1]
self.total_transactions = txids
ts = time.perf_counter() - start
self.logger.info("loaded %i txids in %ss", len(self.total_transactions), round(ts, 4))
async def _read_headers(self):
if self.headers is not None:
return
def get_headers():
return list(self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False))
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
self.headers = headers
async def _open_dbs(self, for_sync, compacting):
if self.executor is None:
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
self.executor = ThreadPoolExecutor(1)
coin_path = os.path.join(self.env.db_dir, 'COIN')
if not os.path.isfile(coin_path):
with util.open_file(coin_path, create=True) as f:
@@ -123,17 +158,11 @@ class LevelDB:
self.logger.info('created new headers db')
self.logger.info(f'opened headers DB (for sync: {for_sync})')
assert self.tx_count_db is None
self.tx_count_db = self.db_class('tx_count', for_sync)
if self.tx_count_db.is_new:
self.logger.info('created new tx count db')
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
assert self.hashes_db is None
self.hashes_db = self.db_class('hashes', for_sync)
if self.hashes_db.is_new:
self.logger.info('created new tx hashes db')
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
assert self.tx_db is None
self.tx_db = self.db_class('tx', for_sync)
if self.tx_db.is_new:
self.logger.info('created new tx db')
self.logger.info(f'opened tx DB (for sync: {for_sync})')
assert self.utxo_db is None
# First UTXO DB
@@ -151,13 +180,15 @@ class LevelDB:
# Read TX counts (requires meta directory)
await self._read_tx_counts()
if self.total_transactions is None:
await self._read_txids()
await self._read_headers()
def close(self):
self.utxo_db.close()
self.history.close_db()
self.headers_db.close()
self.tx_count_db.close()
self.hashes_db.close()
self.tx_db.close()
self.executor.shutdown(wait=True)
self.executor = None
@@ -187,12 +218,9 @@ class LevelDB:
if self.headers_db:
self.headers_db.close()
self.headers_db = None
if self.tx_count_db:
self.tx_count_db.close()
self.tx_count_db = None
if self.hashes_db:
self.hashes_db.close()
self.hashes_db = None
if self.tx_db:
self.tx_db.close()
self.tx_db = None
await self._open_dbs(False, False)
self.logger.info("opened for serving")
@@ -217,7 +245,7 @@ class LevelDB:
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.block_txs
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
@@ -275,33 +303,48 @@ class LevelDB:
"""
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
assert len(flush_data.block_txs) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
assert len(
b''.join(hashes for hashes, _ in flush_data.block_txs)
) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
# Write the headers
start_time = time.perf_counter()
with self.headers_db.write_batch() as batch:
batch_put = batch.put
for i, header in enumerate(flush_data.headers):
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
self.headers.append(header)
flush_data.headers.clear()
height_start = self.fs_height + 1
tx_num = prior_tx_count
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
tx_count = self.tx_counts[height_start]
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
tx_num += 1
offset += 32
with self.tx_db.write_batch() as batch:
batch_put = batch.put
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
tx_count = self.tx_counts[height_start]
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
height_start += 1
offset = 0
while offset < len(tx_hashes):
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
tx_num += 1
offset += 32
flush_data.block_txs.clear()
flush_data.block_hashes.clear()
flush_data.block_tx_hashes.clear()
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
flush_data.headers.clear()
elapsed = time.perf_counter() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
@@ -362,7 +405,7 @@ class LevelDB:
def flush_backup(self, flush_data, touched):
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
assert not flush_data.headers
assert not flush_data.block_tx_hashes
assert not flush_data.block_txs
assert flush_data.height < self.db_height
self.history.assert_flushed()
@@ -383,19 +426,31 @@ class LevelDB:
def backup_fs(self, height, tx_count):
"""Back up during a reorg. This just updates our pointers."""
self.fs_height = height
while self.fs_height > height:
self.fs_height -= 1
self.headers.pop()
self.fs_tx_count = tx_count
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(height + 1)
async def raw_header(self, height):
def raw_header(self, height):
"""Return the binary header at the given height."""
header, n = await self.read_headers(height, 1)
header, n = self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
async def read_headers(self, start_height, count):
def encode_headers(self, start_height, count, headers):
key = (start_height, count)
if not self.encoded_headers.get(key):
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode()
if start_height % 1000 != 0:
return headers
self.encoded_headers[key] = headers
return self.encoded_headers.get(key)
def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]:
"""Requires start_height >= 0, count >= 0. Reads as many headers as
are available starting at start_height up to count. This
would be zero if start_height is beyond self.db_height, for
@@ -404,24 +459,15 @@ class LevelDB:
Returns a (binary, n) pair where binary is the concatenated
binary headers, and n is the count of headers returned.
"""
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
def read_headers():
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(
self.headers_db.iterator(
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
include_key=False
)
), disk_count
return b'', 0
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
return b''.join(self.headers[start_height:start_height + disk_count]), disk_count
return b'', 0
def fs_tx_hash(self, tx_num):
"""Return a par (tx_hash, tx_height) for the given tx number.
@@ -429,22 +475,67 @@ class LevelDB:
If the tx_height is not on disk, returns (None, tx_height)."""
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
return tx_hash, tx_height
return None, tx_height
try:
return self.total_transactions[tx_num], tx_height
except IndexError:
self.logger.exception(
"Failed to access a cached transaction, known bug #3142 "
"should be fixed in #3205"
)
return None, tx_height
def _fs_transactions(self, txids: Iterable[str]):
unpack_be_uint64 = util.unpack_be_uint64
tx_counts = self.tx_counts
tx_db_get = self.tx_db.get
tx_cache = self._tx_and_merkle_cache
tx_infos = {}
for tx_hash in txids:
cached_tx = tx_cache.get(tx_hash)
if cached_tx:
tx, merkle = cached_tx
else:
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
tx = None
tx_height = -1
if tx_num is not None:
tx_num = unpack_be_uint64(tx_num)
tx_height = bisect_right(tx_counts, tx_num)
if tx_height < self.db_height:
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
if tx_height == -1:
merkle = {
'block_height': -1
}
else:
tx_pos = tx_num - tx_counts[tx_height - 1]
branch, root = self.merkle.branch_and_root(
self.total_transactions[tx_counts[tx_height - 1]:tx_counts[tx_height]], tx_pos
)
merkle = {
'block_height': tx_height,
'merkle': [
hash_to_hex_str(hash)
for hash in branch
],
'pos': tx_pos
}
if tx_height + 10 < self.db_height:
tx_cache[tx_hash] = tx, merkle
tx_infos[tx_hash] = (None if not tx else tx.hex(), merkle)
return tx_infos
async def fs_transactions(self, txids):
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
async def fs_block_hashes(self, height, count):
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
offset = 0
headers = []
for n in range(count):
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
offset += self.coin.BASIC_HEADER_SIZE
return [self.coin.header_hash(header) for header in headers]
if height + count > len(self.headers):
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
async def limited_history(self, hashX, *, limit=1000):
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
@@ -453,26 +544,33 @@ class LevelDB:
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
"""
def read_history():
hashx_history = []
for key, hist in self.history.db.iterator(prefix=hashX):
db_height = self.db_height
tx_counts = self.tx_counts
cnt = 0
txs = []
for hist in self.history.db.iterator(prefix=hashX, include_key=False):
a = array.array('I')
a.frombytes(hist)
for tx_num in a:
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
tx_hash = None
else:
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
hashx_history.append((tx_hash, tx_height))
if limit and len(hashx_history) >= limit:
return hashx_history
return hashx_history
tx_height = bisect_right(tx_counts, tx_num)
if tx_height > db_height:
return
txs.append((tx_num, tx_height))
cnt += 1
if limit and cnt >= limit:
break
if limit and cnt >= limit:
break
return txs
while True:
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
if all(hash is not None for hash, height in history):
return history
if history is not None:
return [(self.total_transactions[tx_num], tx_height) for (tx_num, tx_height) in history]
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
await sleep(0.25)
@@ -629,13 +727,14 @@ class LevelDB:
utxos = []
utxos_append = utxos.append
s_unpack = unpack
fs_tx_hash = self.fs_tx_hash
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
value, = unpack('<Q', db_value)
tx_hash, height = self.fs_tx_hash(tx_num)
tx_hash, height = fs_tx_hash(tx_num)
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
return utxos
+49 -45
View File
@@ -6,13 +6,11 @@
# and warranty status of this software.
"""Mempool handling."""
import os
import asyncio
import itertools
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from concurrent.futures.thread import ThreadPoolExecutor
from prometheus_client import Histogram
import attr
@@ -74,7 +72,7 @@ class MemPoolAPI(ABC):
"""
@abstractmethod
async def on_mempool(self, touched, height):
async def on_mempool(self, touched, new_touched, height):
"""Called each time the mempool is synchronized. touched is a set of
hashXs touched since the previous call. height is the
daemon's height at the time the mempool was obtained."""
@@ -117,8 +115,8 @@ class MemPool:
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
self.wakeup = asyncio.Event()
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
self.mempool_process_time_metric = mempool_process_time_metric
self.notified_mempool_txs = set()
async def _logging(self, synchronized_event):
"""Print regular logs of mempool stats."""
@@ -135,15 +133,11 @@ class MemPool:
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
try:
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
finally:
self.executor.shutdown(wait=True)
while True:
await synchronized_event.wait()
async with self.lock:
self._update_histogram(100_000)
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
@@ -216,6 +210,15 @@ class MemPool:
return deferred, {prevout: utxo_map[prevout] for prevout in unspent}
async def _mempool_loop(self, synchronized_event):
try:
return await self._refresh_hashes(synchronized_event)
except asyncio.CancelledError:
raise
except Exception as e:
self.logger.exception("MEMPOOL DIED")
raise e
async def _refresh_hashes(self, synchronized_event):
"""Refresh our view of the daemon's mempool."""
while True:
@@ -226,10 +229,15 @@ class MemPool:
continue
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
async with self.lock:
new_hashes = hashes.difference(self.notified_mempool_txs)
touched = await self._process_mempool(hashes)
self.notified_mempool_txs.update(new_hashes)
new_touched = {
touched_hashx for touched_hashx, txs in self.hashXs.items() if txs.intersection(new_hashes)
}
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(touched, height)
await self.api.on_mempool(touched, new_touched, height)
duration = time.perf_counter() - start
self.mempool_process_time_metric.observe(duration)
try:
@@ -243,7 +251,8 @@ class MemPool:
async def _process_mempool(self, all_hashes):
# Re-sync with the new set of hashes
txs = self.txs
hashXs = self.hashXs
hashXs = self.hashXs # hashX: [tx_hash, ...]
touched = set()
# First handle txs that have disappeared
@@ -274,8 +283,8 @@ class MemPool:
# FIXME: this is not particularly efficient
while tx_map and len(tx_map) != prior_count:
prior_count = len(tx_map)
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map,
touched)
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map, touched)
if tx_map:
self.logger.info(f'{len(tx_map)} txs dropped')
@@ -286,30 +295,25 @@ class MemPool:
hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes)
raw_txs = await self.api.raw_transactions(hex_hashes_iter)
def deserialize_txs(): # This function is pure
to_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
to_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
txs = {}
for hash, raw_tx in zip(hashes, raw_txs):
# The daemon may have evicted the tx from its
# mempool or it may have gotten in a block
if not raw_tx:
continue
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
# Convert the inputs and outputs into (hashX, value) pairs
# Drop generation-like inputs from MemPoolTx.prevouts
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
for txin in tx.inputs
if not txin.is_generation())
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
for txout in tx.outputs)
txs[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
0, tx_size)
return txs
# Thread this potentially slow operation so as not to block
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
tx_map = {}
for hash, raw_tx in zip(hashes, raw_txs):
# The daemon may have evicted the tx from its
# mempool or it may have gotten in a block
if not raw_tx:
continue
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
# Convert the inputs and outputs into (hashX, value) pairs
# Drop generation-like inputs from MemPoolTx.prevouts
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
for txin in tx.inputs
if not txin.is_generation())
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
for txout in tx.outputs)
tx_map[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
0, tx_size)
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
@@ -320,7 +324,7 @@ class MemPool:
for prevout in tx.prevouts
if prevout[0] not in all_hashes)
utxos = await self.api.lookup_utxos(prevouts)
utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
utxo_map = dict(zip(prevouts, utxos))
return self._accept_transactions(tx_map, utxo_map, touched)
@@ -331,8 +335,8 @@ class MemPool:
async def keep_synchronized(self, synchronized_event):
"""Keep the mempool synchronized with the daemon."""
await asyncio.wait([
self._refresh_hashes(synchronized_event),
self._refresh_histogram(synchronized_event),
self._mempool_loop(synchronized_event),
# self._refresh_histogram(synchronized_event),
self._logging(synchronized_event)
])
@@ -349,7 +353,7 @@ class MemPool:
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
return value
async def compact_fee_histogram(self):
def compact_fee_histogram(self):
"""Return a compact fee histogram of the current mempool."""
return self.cached_compact_histogram
@@ -366,7 +370,7 @@ class MemPool:
result.update(tx.prevouts)
return result
async def transaction_summaries(self, hashX):
def transaction_summaries(self, hashX):
"""Return a list of MemPoolTxSummary objects for the hashX."""
result = []
for tx_hash in self.hashXs.get(hashX, ()):
+1 -1
View File
@@ -63,7 +63,7 @@ class Merkle:
raise TypeError('index must be an integer')
# This also asserts hashes is not empty
if not 0 <= index < len(hashes):
raise ValueError('index out of range')
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
natural_length = self.branch_length(len(hashes))
if length is None:
length = natural_length
+1 -1
View File
@@ -319,7 +319,7 @@ class PeerManager:
# Check prior header too in case of hard fork.
check_height = min(our_height, their_height)
raw_header = await self.db.raw_header(check_height)
raw_header = self.db.raw_header(check_height)
if ptuple >= (1, 4):
ours = raw_header.hex()
message = 'blockchain.block.header'
+19 -8
View File
@@ -25,9 +25,10 @@ class Notifications:
def __init__(self):
self._touched_mp = {}
self._touched_bp = {}
self.notified_mempool_txs = set()
self._highest_block = -1
async def _maybe_notify(self):
async def _maybe_notify(self, new_touched):
tmp, tbp = self._touched_mp, self._touched_bp
common = set(tmp).intersection(tbp)
if common:
@@ -44,24 +45,24 @@ class Notifications:
del tmp[old]
for old in [h for h in tbp if h <= height]:
touched.update(tbp.pop(old))
await self.notify(height, touched)
await self.notify(height, touched, new_touched)
async def notify(self, height, touched):
async def notify(self, height, touched, new_touched):
pass
async def start(self, height, notify_func):
self._highest_block = height
self.notify = notify_func
await self.notify(height, set())
await self.notify(height, set(), set())
async def on_mempool(self, touched, height):
async def on_mempool(self, touched, new_touched, height):
self._touched_mp[height] = touched
await self._maybe_notify()
await self._maybe_notify(new_touched)
async def on_block(self, touched, height):
self._touched_bp[height] = touched
self._highest_block = height
await self._maybe_notify()
await self._maybe_notify(set())
class Server:
@@ -84,12 +85,16 @@ class Server:
notifications.mempool_hashes = daemon.mempool_hashes
notifications.raw_transactions = daemon.getrawtransactions
notifications.lookup_utxos = db.lookup_utxos
MemPoolAPI.register(Notifications)
self.mempool = mempool = MemPool(env.coin, notifications)
notifications.notified_mempool_txs = self.mempool.notified_mempool_txs
self.session_mgr = env.coin.SESSION_MANAGER(
env, db, bp, daemon, mempool, self.shutdown_event
)
self._indexer_task = None
async def start(self):
env = self.env
@@ -106,11 +111,17 @@ class Server:
self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
return _flag.wait()
await self.start_prometheus()
if self.env.udp_port:
await self.bp.status_server.start(
0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1],
self.env.host, self.env.udp_port
)
await _start_cancellable(self.bp.fetch_and_process_blocks)
await self.db.populate_header_merkle_cache()
await _start_cancellable(self.mempool.keep_synchronized)
await _start_cancellable(self.session_mgr.serve, self.notifications)
await self.start_prometheus()
async def stop(self):
for task in reversed(self.cancellable_tasks):
+177 -320
View File
@@ -3,8 +3,6 @@ import ssl
import math
import time
import json
import zlib
import pylru
import base64
import codecs
import typing
@@ -18,15 +16,16 @@ from collections import defaultdict
from functools import partial
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from elasticsearch import ConnectionTimeout
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.utils import LRUCacheWithMetrics
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.rpc.framing import NewlineFramer
@@ -141,7 +140,11 @@ class SessionManager:
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
labelnames=("method", "version"))
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
@@ -186,7 +189,9 @@ class SessionManager:
self.shutdown_event = shutdown_event
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
self.sessions: typing.Set['SessionBase'] = set()
self.sessions: typing.Dict[int, 'SessionBase'] = {}
self.hashx_subscriptions_by_session: typing.DefaultDict[str, typing.Set[int]] = defaultdict(set)
self.mempool_statuses = {}
self.cur_group = SessionGroup(0)
self.txs_sent = 0
self.start_time = time.time()
@@ -242,11 +247,12 @@ class SessionManager:
async def _manage_servers(self):
paused = False
max_sessions = self.env.max_sessions
low_watermark = max_sessions * 19 // 20
low_watermark = int(max_sessions * 0.95)
while True:
await self.session_event.wait()
self.session_event.clear()
if not paused and len(self.sessions) >= max_sessions:
self.bp.status_server.set_unavailable()
self.logger.info(f'maximum sessions {max_sessions:,d} '
f'reached, stopping new connections until '
f'count drops to {low_watermark:,d}')
@@ -255,6 +261,7 @@ class SessionManager:
# Start listening for incoming connections if paused and
# session count has fallen
if paused and len(self.sessions) <= low_watermark:
self.bp.status_server.set_available()
self.logger.info('resuming listening for incoming connections')
await self._start_external_servers()
paused = False
@@ -272,12 +279,12 @@ class SessionManager:
def _group_map(self):
group_map = defaultdict(list)
for session in self.sessions:
for session in self.sessions.values():
group_map[session.group].append(session)
return group_map
def _sub_count(self) -> int:
return sum(s.sub_count() for s in self.sessions)
return sum(s.sub_count() for s in self.sessions.values())
def _lookup_session(self, session_id):
try:
@@ -285,7 +292,7 @@ class SessionManager:
except Exception:
pass
else:
for session in self.sessions:
for session in self.sessions.values():
if session.session_id == session_id:
return session
return None
@@ -309,7 +316,7 @@ class SessionManager:
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.perf_counter() - session_timeout
stale_sessions = [session for session in self.sessions
stale_sessions = [session for session in self.sessions.values()
if session.last_recv < stale_cutoff]
if stale_sessions:
text = ', '.join(str(session.session_id)
@@ -341,7 +348,7 @@ class SessionManager:
pending_requests = 0
closing = 0
for s in self.sessions:
for s in self.sessions.values():
error_count += s.errors
if s.log_me:
logged += 1
@@ -375,7 +382,7 @@ class SessionManager:
def _session_data(self, for_log):
"""Returned to the RPC 'sessions' call."""
now = time.time()
sessions = sorted(self.sessions, key=lambda s: s.start_time)
sessions = sorted(self.sessions.values(), key=lambda s: s.start_time)
return [(session.session_id,
session.flags(),
session.peer_address_str(for_log=for_log),
@@ -567,6 +574,7 @@ class SessionManager:
await self.start_other()
await self._start_external_servers()
server_listening_event.set()
self.bp.status_server.set_available()
# Peer discovery should start after the external servers
# because we connect to ourself
await asyncio.wait([
@@ -577,9 +585,10 @@ class SessionManager:
])
finally:
await self._close_servers(list(self.servers.keys()))
log.warning("disconnect %i sessions", len(self.sessions))
if self.sessions:
await asyncio.wait([
session.close(force_after=1) for session in self.sessions
session.close(force_after=1) for session in self.sessions.values()
])
await self.stop_other()
@@ -603,7 +612,7 @@ class SessionManager:
async def raw_header(self, height):
"""Return the binary header at the given height."""
try:
return await self.db.raw_header(height)
return self.db.raw_header(height)
except IndexError:
raise RPCError(BAD_REQUEST, f'height {height:,d} '
'out of range') from None
@@ -629,18 +638,42 @@ class SessionManager:
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
return self.history_cache[hashX]
async def _notify_sessions(self, height, touched):
async def _notify_sessions(self, height, touched, new_touched):
"""Notify sessions about height changes and touched addresses."""
height_changed = height != self.notified_height
if height_changed:
await self._refresh_hsub_results(height)
if self.sessions:
await asyncio.wait([
session.notify(touched, height_changed) for session in self.sessions
])
if not self.sessions:
return
if height_changed:
header_tasks = [
session.send_notification('blockchain.headers.subscribe', (self.hsub_results[session.subscribe_headers_raw], ))
for session in self.sessions.values() if session.subscribe_headers
]
if header_tasks:
self.logger.info(f'notify {len(header_tasks)} sessions of new header')
asyncio.create_task(asyncio.wait(header_tasks))
for hashX in touched.intersection(self.mempool_statuses.keys()):
self.mempool_statuses.pop(hashX, None)
touched.intersection_update(self.hashx_subscriptions_by_session.keys())
if touched or (height_changed and self.mempool_statuses):
notified_hashxs = 0
notified_sessions = 0
to_notify = touched if height_changed else new_touched
for hashX in to_notify:
for session_id in self.hashx_subscriptions_by_session[hashX]:
asyncio.create_task(self.sessions[session_id].send_history_notification(hashX))
notified_sessions += 1
notified_hashxs += 1
if notified_sessions:
self.logger.info(f'notified {notified_sessions} sessions/{notified_hashxs:,d} touched addresses')
def add_session(self, session):
self.sessions.add(session)
self.sessions[id(session)] = session
self.session_event.set()
gid = int(session.start_time - self.start_time) // 900
if self.cur_group.gid != gid:
@@ -649,7 +682,13 @@ class SessionManager:
def remove_session(self, session):
"""Remove a session from our sessions list if there."""
self.sessions.remove(session)
session_id = id(session)
for hashX in session.hashX_subs:
sessions = self.hashx_subscriptions_by_session[hashX]
sessions.remove(session_id)
if not sessions:
self.hashx_subscriptions_by_session.pop(hashX)
self.sessions.pop(session_id)
self.session_event.set()
@@ -684,8 +723,6 @@ class SessionBase(RPCSession):
self._receive_message_orig = self.connection.receive_message
self.connection.receive_message = self.receive_message
async def notify(self, touched, height_changed):
pass
def default_framer(self):
return NewlineFramer(self.env.max_receive)
@@ -776,9 +813,6 @@ class LBRYSessionManager(SessionManager):
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
async def process_metrics(self):
while self.running:
@@ -792,22 +826,11 @@ class LBRYSessionManager(SessionManager):
async def start_other(self):
self.running = True
path = os.path.join(self.env.db_dir, 'claims.db')
args = dict(
initializer=reader.initializer,
initargs=(
self.logger, path, self.env.coin.NET, self.env.database_query_timeout,
self.env.track_metrics, (
self.db.sql.blocked_streams, self.db.sql.blocked_channels,
self.db.sql.filtered_streams, self.db.sql.filtered_channels
)
)
)
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
self.query_executor = ThreadPoolExecutor(max_workers=1, **args)
self.query_executor = ThreadPoolExecutor(max_workers=1)
else:
self.query_executor = ProcessPoolExecutor(
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4)
)
if self.websocket is not None:
await self.websocket.start()
@@ -831,6 +854,7 @@ class LBRYElectrumX(SessionBase):
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
version = lbry.__version__
cached_server_features = {}
@classmethod
def initialize_request_handlers(cls):
@@ -859,7 +883,6 @@ class LBRYElectrumX(SessionBase):
'blockchain.transaction.get_height': cls.transaction_get_height,
'blockchain.claimtrie.search': cls.claimtrie_search,
'blockchain.claimtrie.resolve': cls.claimtrie_resolve,
'blockchain.claimtrie.getclaimsbyids': cls.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': cls.get_server_height,
'mempool.get_fee_histogram': cls.mempool_compact_histogram,
'blockchain.block.headers': cls.block_headers,
@@ -877,14 +900,15 @@ class LBRYElectrumX(SessionBase):
super().__init__(*args, **kwargs)
if not LBRYElectrumX.request_handlers:
LBRYElectrumX.initialize_request_handlers()
if not LBRYElectrumX.cached_server_features:
LBRYElectrumX.set_server_features(self.env)
self.subscribe_headers = False
self.subscribe_headers_raw = False
self.connection.max_response_size = self.env.max_send
self.hashX_subs = {}
self.sv_seen = False
self.mempool_statuses = {}
self.protocol_tuple = self.PROTOCOL_MIN
self.protocol_string = None
self.daemon = self.session_mgr.daemon
self.bp: LBRYBlockProcessor = self.session_mgr.bp
self.db: LBRYLevelDB = self.bp.db
@@ -895,10 +919,10 @@ class LBRYElectrumX(SessionBase):
for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)]
@classmethod
def server_features(cls, env):
def set_server_features(cls, env):
"""Return the server features dictionary."""
min_str, max_str = cls.protocol_min_max_strings()
return {
cls.cached_server_features.update({
'hosts': env.hosts_dict(),
'pruning': None,
'server_version': cls.version,
@@ -911,10 +935,10 @@ class LBRYElectrumX(SessionBase):
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
})
async def server_features_async(self):
return self.server_features(self.env)
return self.cached_server_features
@classmethod
def server_version_args(cls):
@@ -927,48 +951,22 @@ class LBRYElectrumX(SessionBase):
def sub_count(self):
return len(self.hashX_subs)
async def notify(self, touched, height_changed):
"""Notify the client about changes to touched addresses (from mempool
updates or new blocks) and height.
"""
if height_changed and self.subscribe_headers:
args = (await self.subscribe_headers_result(), )
if not (await self.send_notification('blockchain.headers.subscribe', args)):
return
async def send_history_notification(alias, hashX):
async def send_history_notification(self, hashX):
start = time.perf_counter()
alias = self.hashX_subs[hashX]
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
start = time.perf_counter()
if len(alias) == 64:
method = 'blockchain.scripthash.subscribe'
else:
method = 'blockchain.address.subscribe'
try:
self.session_mgr.notifications_in_flight_metric.inc()
status = await self.address_status(hashX)
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
start = time.perf_counter()
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
touched = touched.intersection(self.hashX_subs)
if touched or (height_changed and self.mempool_statuses):
notified = set()
mempool_addrs = tuple(self.mempool_statuses.keys())
for hashX in touched:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
for hashX in mempool_addrs:
if hashX not in notified:
alias = self.hashX_subs[hashX]
asyncio.create_task(send_history_notification(alias, hashX))
notified.add(hashX)
if touched:
es = '' if len(touched) == 1 else 'es'
self.logger.info(f'notified {len(notified)} mempool/{len(touched):,d} touched address{es}')
await self.send_notification(method, (alias, status))
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
finally:
self.session_mgr.notifications_in_flight_metric.dec()
def get_metrics_or_placeholder_for_api(self, query_name):
""" Do not hold on to a reference to the metrics
@@ -989,16 +987,6 @@ class LBRYElectrumX(SessionBase):
)
except asyncio.CancelledError:
raise
except reader.SQLiteInterruptedError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_interrupt(start, error.metrics)
self.session_mgr.interrupt_count_metric.inc()
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, error.metrics)
self.session_mgr.db_operational_error_metric.inc()
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except Exception:
log.exception("dear devs, please handle this exception better")
metrics = self.get_metrics_or_placeholder_for_api(query_name)
@@ -1015,37 +1003,35 @@ class LBRYElectrumX(SessionBase):
self.session_mgr.pending_query_metric.dec()
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
async def run_and_cache_query(self, query_name, function, kwargs):
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
return cache_item.result
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
query_name, function, kwargs
)
else:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.cache_response()
return cache_item.result
async def run_and_cache_query(self, query_name, kwargs):
start = time.perf_counter()
if isinstance(kwargs, dict):
kwargs['release_time'] = format_release_time(kwargs.get('release_time'))
try:
self.session_mgr.pending_query_metric.inc()
return await self.db.search_index.session_query(query_name, kwargs)
except ConnectionTimeout:
self.session_mgr.interrupt_count_metric.inc()
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'query timed out')
finally:
self.session_mgr.pending_query_metric.dec()
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
async def mempool_compact_histogram(self):
return self.mempool.compact_fee_histogram()
async def claimtrie_search(self, **kwargs):
if kwargs:
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
return await self.run_and_cache_query('search', kwargs)
async def claimtrie_resolve(self, *urls):
if urls:
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
count = len(urls)
try:
self.session_mgr.urls_to_resolve_count_metric.inc(count)
return await self.run_and_cache_query('resolve', urls)
finally:
self.session_mgr.resolved_url_count_metric.inc(count)
async def get_server_height(self):
return self.bp.height
@@ -1060,67 +1046,6 @@ class LBRYElectrumX(SessionBase):
return -1
return None
async def claimtrie_getclaimsbyids(self, *claim_ids):
claims = await self.batched_formatted_claims_from_daemon(claim_ids)
return dict(zip(claim_ids, claims))
async def batched_formatted_claims_from_daemon(self, claim_ids):
claims = await self.daemon.getclaimsbyids(claim_ids)
result = []
for claim in claims:
if claim and claim.get('value'):
result.append(self.format_claim_from_daemon(claim))
return result
def format_claim_from_daemon(self, claim, name=None):
"""Changes the returned claim data to the format expected by lbry and adds missing fields."""
if not claim:
return {}
# this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
# it will be fixed after the lbrycrd upstream merge to v17 is done
# it originated as a fear of terminals not supporting unicode. alas, they all do
if 'name' in claim:
name = claim['name'].encode('ISO-8859-1').decode()
info = self.db.sql.get_claims(claim_id=claim['claimId'])
if not info:
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
return {}
address = info.address.decode()
# fixme: temporary
#supports = self.format_supports_from_daemon(claim.get('supports', []))
supports = []
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
height = get_from_possible_keys(claim, 'height', 'nHeight')
effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
result = {
"name": name,
"claim_id": claim['claimId'],
"txid": claim['txid'],
"nout": claim['n'],
"amount": amount,
"depth": self.db.db_height - height + 1,
"height": height,
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
"address": address, # from index
"supports": supports,
"effective_amount": effective_amount,
"valid_at_height": valid_at_height
}
if 'claim_sequence' in claim:
# TODO: ensure that lbrycrd #209 fills in this value
result['claim_sequence'] = claim['claim_sequence']
else:
result['claim_sequence'] = -1
if 'normalized_name' in claim:
result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
return result
def assert_tx_hash(self, value):
'''Raise an RPCError if the value is not a valid transaction
hash.'''
@@ -1131,16 +1056,6 @@ class LBRYElectrumX(SessionBase):
pass
raise RPCError(1, f'{value} should be a transaction hash')
def assert_claim_id(self, value):
'''Raise an RPCError if the value is not a valid claim id
hash.'''
try:
if len(util.hex_to_bytes(value)) == 20:
return
except Exception:
pass
raise RPCError(1, f'{value} should be a claim id hash')
async def subscribe_headers_result(self):
"""The result of a header subscription or notification."""
return self.session_mgr.hsub_results[self.subscribe_headers_raw]
@@ -1180,7 +1095,7 @@ class LBRYElectrumX(SessionBase):
# For mempool, height is -1 if it has unconfirmed inputs, otherwise 0
db_history = await self.session_mgr.limited_history(hashX)
mempool = await self.mempool.transaction_summaries(hashX)
mempool = self.mempool.transaction_summaries(hashX)
status = ''.join(f'{hash_to_hex_str(tx_hash)}:'
f'{height:d}:'
@@ -1194,9 +1109,9 @@ class LBRYElectrumX(SessionBase):
status = None
if mempool:
self.mempool_statuses[hashX] = status
self.session_mgr.mempool_statuses[hashX] = status
else:
self.mempool_statuses.pop(hashX, None)
self.session_mgr.mempool_statuses.pop(hashX, None)
return status
async def hashX_listunspent(self, hashX):
@@ -1215,13 +1130,14 @@ class LBRYElectrumX(SessionBase):
async def hashX_subscribe(self, hashX, alias):
self.hashX_subs[hashX] = alias
self.session_mgr.hashx_subscriptions_by_session[hashX].add(id(self))
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
try:
del self.hashX_subs[hashX]
except ValueError:
pass
sessions = self.session_mgr.hashx_subscriptions_by_session[hashX]
sessions.remove(id(self))
if not sessions:
self.hashX_subs.pop(hashX, None)
def address_to_hashX(self, address):
try:
@@ -1243,7 +1159,7 @@ class LBRYElectrumX(SessionBase):
async def address_get_mempool(self, address):
"""Return the mempool transactions touching an address."""
hashX = self.address_to_hashX(address)
return await self.unconfirmed_history(hashX)
return self.unconfirmed_history(hashX)
async def address_listunspent(self, address):
"""Return the list of UTXOs of an address."""
@@ -1256,10 +1172,9 @@ class LBRYElectrumX(SessionBase):
address: the address to subscribe to"""
if len(addresses) > 1000:
raise RPCError(BAD_REQUEST, f'too many addresses in subscription request: {len(addresses)}')
hashXes = [
(self.address_to_hashX(address), address) for address in addresses
return [
await self.hashX_subscribe(self.address_to_hashX(address), address) for address in addresses
]
return await asyncio.gather(*(self.hashX_subscribe(*args) for args in hashXes))
async def address_unsubscribe(self, address):
"""Unsubscribe an address.
@@ -1279,20 +1194,20 @@ class LBRYElectrumX(SessionBase):
hashX = scripthash_to_hashX(scripthash)
return await self.get_balance(hashX)
async def unconfirmed_history(self, hashX):
def unconfirmed_history(self, hashX):
# Note unconfirmed history is unordered in electrum-server
# height is -1 if it has unconfirmed inputs, otherwise 0
return [{'tx_hash': hash_to_hex_str(tx.hash),
'height': -tx.has_unconfirmed_inputs,
'fee': tx.fee}
for tx in await self.mempool.transaction_summaries(hashX)]
for tx in self.mempool.transaction_summaries(hashX)]
async def confirmed_and_unconfirmed_history(self, hashX):
# Note history is ordered but unconfirmed is unordered in e-s
history = await self.session_mgr.limited_history(hashX)
conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height}
for tx_hash, height in history]
return conf + await self.unconfirmed_history(hashX)
return conf + self.unconfirmed_history(hashX)
async def scripthash_get_history(self, scripthash):
"""Return the confirmed and unconfirmed history of a scripthash."""
@@ -1302,7 +1217,7 @@ class LBRYElectrumX(SessionBase):
async def scripthash_get_mempool(self, scripthash):
"""Return the mempool transactions touching a scripthash."""
hashX = scripthash_to_hashX(scripthash)
return await self.unconfirmed_history(hashX)
return self.unconfirmed_history(hashX)
async def scripthash_listunspent(self, scripthash):
"""Return the list of UTXOs of a scripthash."""
@@ -1323,31 +1238,12 @@ class LBRYElectrumX(SessionBase):
f'require header height {height:,d} <= '
f'cp_height {cp_height:,d} <= '
f'chain height {max_height:,d}')
branch, root = await self.db.header_branch_and_root(cp_height + 1,
height)
branch, root = await self.db.header_branch_and_root(cp_height + 1, height)
return {
'branch': [hash_to_hex_str(elt) for elt in branch],
'root': hash_to_hex_str(root),
}
async def block_header(self, height, cp_height=0):
"""Return a raw block header as a hexadecimal string, or as a
dictionary with a merkle proof."""
height = non_negative_integer(height)
cp_height = non_negative_integer(cp_height)
raw_header_hex = (await self.session_mgr.raw_header(height)).hex()
if cp_height == 0:
return raw_header_hex
result = {'header': raw_header_hex}
result.update(await self._merkle_proof(cp_height, height))
return result
async def block_header_13(self, height):
"""Return a raw block header as a hexadecimal string.
height: the header's height"""
return await self.block_header(height)
async def block_headers(self, start_height, count, cp_height=0, b64=False):
"""Return count concatenated block headers as hex for the main chain;
starting at start_height.
@@ -1361,9 +1257,12 @@ class LBRYElectrumX(SessionBase):
max_size = self.MAX_CHUNK_SIZE
count = min(count, max_size)
headers, count = await self.db.read_headers(start_height, count)
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
headers, count = self.db.read_headers(start_height, count)
if b64:
headers = self.db.encode_headers(start_height, count, headers)
else:
headers = headers.hex()
result = {
'base64' if b64 else 'hex': headers,
'count': count,
@@ -1381,7 +1280,7 @@ class LBRYElectrumX(SessionBase):
index = non_negative_integer(index)
size = self.coin.CHUNK_SIZE
start_height = index * size
headers, _ = await self.db.read_headers(start_height, size)
headers, _ = self.db.read_headers(start_height, size)
return headers.hex()
async def block_get_header(self, height):
@@ -1470,7 +1369,8 @@ class LBRYElectrumX(SessionBase):
client_name: a string identifying the client
protocol_version: the protocol version spoken by the client
"""
if self.protocol_string is not None:
return self.version, self.protocol_string
if self.sv_seen and self.protocol_tuple >= (1, 4):
raise RPCError(BAD_REQUEST, f'server.version already sent')
self.sv_seen = True
@@ -1480,8 +1380,7 @@ class LBRYElectrumX(SessionBase):
if self.env.drop_client is not None and \
self.env.drop_client.match(client_name):
self.close_after_send = True
raise RPCError(BAD_REQUEST,
f'unsupported client: {client_name}')
raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}')
if self.client_version != client_name[:17]:
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
self.client_version = client_name[:17]
@@ -1490,19 +1389,16 @@ class LBRYElectrumX(SessionBase):
# Find the highest common protocol version. Disconnect if
# that protocol version in unsupported.
ptuple, client_min = util.protocol_version(
protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
ptuple, client_min = util.protocol_version(protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
if ptuple is None:
# FIXME: this fills the logs
# if client_min > self.PROTOCOL_MIN:
# self.logger.info(f'client requested future protocol version '
# f'{util.version_string(client_min)} '
# f'- is your software out of date?')
self.close_after_send = True
raise RPCError(BAD_REQUEST,
f'unsupported protocol version: {protocol_version}')
ptuple, client_min = util.protocol_version(protocol_version, (1, 1, 0), (1, 4, 0))
if ptuple is None:
self.close_after_send = True
raise RPCError(BAD_REQUEST, f'unsupported protocol version: {protocol_version}')
self.protocol_tuple = ptuple
return self.version, self.protocol_version_string()
self.protocol_string = util.version_string(ptuple)
return self.version, self.protocol_string
async def transaction_broadcast(self, raw_tx):
"""Broadcast a raw transaction to the network.
@@ -1523,27 +1419,23 @@ class LBRYElectrumX(SessionBase):
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
return (await self.transaction_get_batch(tx_hash))[tx_hash]
async def transaction_get_batch(self, *tx_hashes):
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = {}
batch_result = await self.db.fs_transactions(tx_hashes)
needed_merkles = {}
for tx_hash in tx_hashes:
if tx_hash in batch_result and batch_result[tx_hash][0]:
continue
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
merkle = {}
if block_hash:
block = await self.daemon.deserialised_block(block_hash)
height = block['height']
@@ -1552,12 +1444,20 @@ class LBRYElectrumX(SessionBase):
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
merkle["pos"] = pos
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
else:
height = -1
merkle['block_height'] = height
batch_result[tx_hash] = [raw_tx, merkle]
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
if needed_merkles:
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
batch_result[tx_hash] = raw_tx, {
'merkle': self._get_merkle_branch(block_txs, pos),
'pos': pos,
'block_height': block_height
}
await asyncio.sleep(0) # heavy call, give other tasks a chance
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
@@ -1572,19 +1472,6 @@ class LBRYElectrumX(SessionBase):
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
async def _block_hash_and_tx_hashes(self, height):
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
the given height.
block_hash is a hexadecimal string, and tx_hashes is an
ordered list of hexadecimal strings.
"""
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon.deserialised_block(block_hash)
return block_hash, block['tx']
def _get_merkle_branch(self, tx_hashes, tx_pos):
"""Return a merkle branch to a transaction.
@@ -1604,35 +1491,11 @@ class LBRYElectrumX(SessionBase):
height: the height of the block it is in
"""
assert_tx_hash(tx_hash)
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
result = await self.transaction_get_batch(tx_hash)
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
branch = self._get_merkle_branch(tx_hashes, pos)
return {"block_height": height, "merkle": branch, "pos": pos}
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
"""Return the txid and optionally a merkle proof, given
a block height and position in the block.
"""
tx_pos = non_negative_integer(tx_pos)
if merkle not in (True, False):
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
try:
tx_hash = tx_hashes[tx_pos]
except IndexError:
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
f'block {block_hash} at height {height:,d}')
if merkle:
branch = self._get_merkle_branch(tx_hashes, tx_pos)
return {"tx_hash": tx_hash, "merkle": branch}
else:
return tx_hash
f'block at height {height:,d}')
return result[tx_hash][1]
class LocalRPC(SessionBase):
@@ -1647,26 +1510,20 @@ class LocalRPC(SessionBase):
return 'RPC'
class ResultCacheItem:
__slots__ = '_result', 'lock', 'has_result'
def __init__(self):
self.has_result = asyncio.Event()
self.lock = asyncio.Lock()
self._result = None
@property
def result(self) -> str:
return self._result
@result.setter
def result(self, result: str):
self._result = result
if result is not None:
self.has_result.set()
def get_from_possible_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
return dictionary[key]
def format_release_time(release_time):
# round release time to 1000 so it caches better
# also set a default so we dont show claims in the future
def roundup_time(number, factor=360):
return int(1 + int(number / factor)) * factor
if isinstance(release_time, str) and len(release_time) > 0:
time_digits = ''.join(filter(str.isdigit, release_time))
time_prefix = release_time[:-len(time_digits)]
return time_prefix + str(roundup_time(int(time_digits)))
elif isinstance(release_time, int):
return roundup_time(release_time)
+4 -6
View File
@@ -77,18 +77,16 @@ class LevelDB(Storage):
import plyvel
cls.module = plyvel
def open(self, name, create):
mof = 512 if self.for_sync else 128
def open(self, name, create, lru_cache_size=None):
mof = 10000
path = os.path.join(self.db_dir, name)
# Use snappy compression (the default)
self.db = self.module.DB(path, create_if_missing=create,
max_open_files=mof)
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
self.iterator = self.db.iterator
self.write_batch = partial(self.db.write_batch, transaction=True,
sync=True)
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
class RocksDB(Storage):
+199
View File
@@ -0,0 +1,199 @@
import asyncio
import struct
from time import perf_counter
import logging
from typing import Optional, Tuple, NamedTuple
from lbry.utils import LRUCache
# from prometheus_client import Counter
log = logging.getLogger(__name__)
_MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
_PAD_BYTES = b'\x00' * 64
class SPVPing(NamedTuple):
magic: int
protocol_version: int
pad_bytes: bytes
def encode(self):
return struct.pack(b'!lB64s', *self)
@staticmethod
def make(protocol_version=1) -> bytes:
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
@classmethod
def decode(cls, packet: bytes):
decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
if decoded.magic != _MAGIC:
raise ValueError("invalid magic bytes")
return decoded
class SPVPong(NamedTuple):
protocol_version: int
flags: int
height: int
tip: bytes
source_address_raw: bytes
def encode(self):
return struct.pack(b'!BBl32s4s', *self)
@staticmethod
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
@classmethod
def decode(cls, packet: bytes):
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
@property
def available(self) -> bool:
return (self.flags & 0b00000001) > 0
@property
def ip_address(self) -> str:
return ".".join(map(str, self.source_address_raw))
def __repr__(self) -> str:
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
f" height={self.height}, tip={self.tip[::-1].hex()})"
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
PROTOCOL_VERSION = 1
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
super().__init__()
self.transport: Optional[asyncio.transports.DatagramTransport] = None
self._height = height
self._tip = tip
self._flags = 0
self._cached_response = None
self.update_cached_response()
self._throttle = LRUCache(throttle_cache_size)
self._should_log = LRUCache(throttle_cache_size)
self._min_delay = 1 / throttle_reqs_per_sec
def update_cached_response(self):
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
def set_unavailable(self):
self._flags &= 0b11111110
self.update_cached_response()
def set_available(self):
self._flags |= 0b00000001
self.update_cached_response()
def set_height(self, height: int, tip: bytes):
self._height, self._tip = height, tip
self.update_cached_response()
def should_throttle(self, host: str):
now = perf_counter()
last_requested = self._throttle.get(host, default=0)
self._throttle[host] = now
if now - last_requested < self._min_delay:
log_cnt = self._should_log.get(host, default=0) + 1
if log_cnt % 100 == 0:
log.warning("throttle spv status to %s", host)
self._should_log[host] = log_cnt
return True
return False
def make_pong(self, host):
return self._cached_response + bytes(int(b) for b in host.split("."))
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
if self.should_throttle(addr[0]):
return
try:
SPVPing.decode(data)
except (ValueError, struct.error, AttributeError, TypeError):
# log.exception("derp")
return
self.transport.sendto(self.make_pong(addr[0]), addr)
# ping_count_metric.inc()
def connection_made(self, transport) -> None:
self.transport = transport
def connection_lost(self, exc: Optional[Exception]) -> None:
self.transport = None
def close(self):
if self.transport:
self.transport.close()
class StatusServer:
def __init__(self):
self._protocol: Optional[SPVServerStatusProtocol] = None
async def start(self, height: int, tip: bytes, interface: str, port: int):
if self.is_running:
return
loop = asyncio.get_event_loop()
self._protocol = SPVServerStatusProtocol(height, tip)
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
log.info("started udp status server on %s:%i", interface, port)
def stop(self):
if self.is_running:
self._protocol.close()
self._protocol = None
@property
def is_running(self):
return self._protocol is not None
def set_unavailable(self):
if self.is_running:
self._protocol.set_unavailable()
def set_available(self):
if self.is_running:
self._protocol.set_available()
def set_height(self, height: int, tip: bytes):
if self.is_running:
self._protocol.set_height(height, tip)
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
PROTOCOL_VERSION = 1
def __init__(self, responses: asyncio.Queue):
super().__init__()
self.transport: Optional[asyncio.transports.DatagramTransport] = None
self.responses = responses
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
try:
self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
return
def connection_made(self, transport) -> None:
self.transport = transport
def connection_lost(self, exc: Optional[Exception]) -> None:
self.transport = None
log.info("closed udp spv server selection client")
def ping(self, server: Tuple[str, int]):
self.transport.sendto(self._ping_packet, server)
def close(self):
# log.info("close udp client")
if self.transport:
self.transport.close()
+13 -6
View File
@@ -539,6 +539,7 @@ class Transaction:
height: int = -2, position: int = -1, julian_day: int = None) -> None:
self._raw = raw
self._raw_sans_segwit = None
self._raw_outputs = None
self.is_segwit_flag = 0
self.witnesses: List[bytes] = []
self.ref = TXRefMutable(self)
@@ -600,6 +601,7 @@ class Transaction:
def _reset(self):
self._raw = None
self._raw_sans_segwit = None
self._raw_outputs = None
self.ref.reset()
@property
@@ -693,9 +695,7 @@ class Transaction:
stream.write_compact_size(len(self._inputs))
for txin in self._inputs:
txin.serialize_to(stream)
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
self._serialize_outputs(stream)
stream.write_uint32(self.locktime)
return stream.get_bytes()
@@ -709,13 +709,19 @@ class Transaction:
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
else:
txin.serialize_to(stream, b'')
stream.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(stream)
self._serialize_outputs(stream)
stream.write_uint32(self.locktime)
stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL
return stream.get_bytes()
def _serialize_outputs(self, stream):
if self._raw_outputs is None:
self._raw_outputs = BCDataStream()
self._raw_outputs.write_compact_size(len(self._outputs))
for txout in self._outputs:
txout.serialize_to(self._raw_outputs)
stream.write(self._raw_outputs.get_bytes())
def _deserialize(self):
if self._raw is not None:
stream = BCDataStream(self._raw)
@@ -844,6 +850,7 @@ class Transaction:
return hash_type
async def sign(self, funding_accounts: Iterable['Account']):
self._reset()
ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
for i, txi in enumerate(self._inputs):
assert txi.script is not None
+8 -3
View File
@@ -145,14 +145,15 @@ class Wallet:
elif not self.is_locked:
log.warning(
"Disk encryption requested but no password available for encryption. "
"Saving wallet in an unencrypted state."
"Resetting encryption preferences and saving wallet in an unencrypted state."
)
self.preferences[ENCRYPT_ON_DISK] = False
return self.storage.write(self.to_dict())
@property
def hash(self) -> bytes:
h = sha256()
if self.preferences.get(ENCRYPT_ON_DISK, False):
if self.is_encrypted:
assert self.encryption_password is not None, \
"Encryption is enabled but no password is available, cannot generate hash."
h.update(self.encryption_password.encode())
@@ -219,7 +220,11 @@ class Wallet:
@property
def is_encrypted(self) -> bool:
return self.is_locked or self.preferences.get(ENCRYPT_ON_DISK, False)
# either its locked or it was unlocked using a password.
# if its set to encrypt on preferences but isnt encrypted and no password was given so far,
# then its not encrypted
return self.is_locked or (
self.preferences.get(ENCRYPT_ON_DISK, False) and self.encryption_password is not None)
def decrypt(self):
assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first."
+1 -1
View File
@@ -64,7 +64,7 @@ async def checkcontrolling(daemon: Daemon, db: SQLDB):
if __name__ == '__main__':
if not len(sys.argv) == 3:
if len(sys.argv) != 3:
print("usage: <db_file_path> <lbrycrd_url>")
sys.exit(1)
db_path, lbrycrd_url = sys.argv[1:] # pylint: disable=W0632
-177
View File
@@ -1,177 +0,0 @@
import os
import time
import textwrap
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.CRITICAL)
DEFAULT_ANY_TAGS = [
'blockchain',
'news',
'learning',
'technology',
'automotive',
'economics',
'food',
'science',
'art',
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
'mature',
'xxx'
]
ORDER_BY = [
[
"trending_global",
"trending_mixed",
],
[
"release_time"
],
[
"effective_amount"
]
]
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_fee in [False, True]:
for claim_type in [None, 'stream', 'channel']:
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
if claim_type:
kw['claim_type'] = claim_type
if no_fee:
kw['fee_amount'] = 0
args.append(kw)
print(f"-- Trying {len(args)} argument combinations")
return args
def _search(kwargs):
start = time.perf_counter()
error = None
try:
search_to_bytes(kwargs)
except Exception as err:
error = str(err)
return time.perf_counter() - start, kwargs, error
async def search(executor, kwargs):
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
async def main(db_path, max_query_time):
args = dict(initializer=initializer, initargs=(log, db_path, MainNetLedger, 0.25))
workers = max(os.cpu_count(), 4)
log.info(f"using {workers} reader processes")
query_executor = ProcessPoolExecutor(workers, **args)
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
query_times = [
{
'sql': interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints)),
'duration': ts,
'error': error
}
for ts, constraints, error in results
]
errored = [query_info for query_info in query_times if query_info['error']]
errors = {str(query_info['error']): [] for query_info in errored}
for error in errored:
errors[str(error['error'])].append(error['sql'])
slow = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] > (max_query_time / 2.0)
]
fast = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] <= (max_query_time / 2.0)
]
print(f"-- {len(fast)} queries were fast")
slow.sort(key=lambda query_info: query_info['duration'], reverse=True)
print(f"-- Failing queries:")
for error in errors:
print(f"-- Failure: \"{error}\"")
for failing_query in errors[error]:
print(f"{textwrap.dedent(failing_query)};\n")
print()
print(f"-- Slow queries:")
for slow_query in slow:
print(f"-- Query took {slow_query['duration']}\n{textwrap.dedent(slow_query['sql'])};\n")
finally:
query_executor.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.25, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
asyncio.run(main(db_path, max_query_time))
+1
View File
@@ -1,6 +1,7 @@
import curses
import time
import asyncio
import lbry.wallet
from lbry.conf import Config
from lbry.extras.daemon.client import daemon_rpc
+29 -2
View File
@@ -13,6 +13,7 @@ from lbry.extras.cli import set_kwargs, get_argument_parser
from lbry.extras.daemon.daemon import (
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
)
from tests.integration.other.test_comment_commands import MockedCommentServer
from lbry.extras.daemon.json_response_encoder import (
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
encode_wallet_doc
@@ -25,7 +26,8 @@ RETURN_DOCS = {
'File': encode_file_doc(),
'Transaction': encode_tx_doc(),
'Output': encode_txo_doc(),
'Address': 'an address in base58'
'Address': 'an address in base58',
'Dict': 'glorious data in dictionary',
}
@@ -65,6 +67,10 @@ class Examples(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon.conf.comment_server = 'http://localhost:2903/api'
self.comment_server = MockedCommentServer(2903)
await self.comment_server.start()
self.addCleanup(self.comment_server.stop)
self.recorder = ExampleRecorder(self)
async def play(self):
@@ -336,7 +342,8 @@ class Examples(CommandTestCase):
'Use the parent_id param to make replies',
'comment', 'create',
'--comment="I have photographic evidence confirming Sasquatch exists"',
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}',
f'--claim_id={stream_id}'
)
await r(
@@ -358,6 +365,26 @@ class Examples(CommandTestCase):
await self.daemon.jsonrpc_comment_abandon(reply['comment_id'])
# collections
collection = await r(
'Create a collection of one stream',
'collection', 'create',
'--name=tom', '--bid=1.0',
f'--channel_id={channel_id}',
f'--claims={stream_id}'
)
await self.on_transaction_dict(collection)
await self.generate(1)
await self.on_transaction_dict(collection)
await r(
'List collections',
'collection', 'list',
'--resolve', '--resolve_claims=1',
)
# files
file_list_result = (await r(
-62
View File
@@ -1,62 +0,0 @@
import uvloop, asyncio, time, sys, logging
from concurrent.futures import ProcessPoolExecutor
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
db_path = '../../../lbryconf/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
async def run_times(executor, iterations, show=True):
start = time.perf_counter()
timings = await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(
executor, reader.search_to_bytes, {
'no_totals': True,
'offset': 0,
'limit': 20,
'any_tags': [
'ufos', 'city fix'
],
'not_tags': [
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query'][0]['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t}ms')
print(' ...\n' if len(timings) > 10 else '\n')
async def main():
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
uvloop.install()
asyncio.run(main())
+5 -3
View File
@@ -28,13 +28,14 @@ setup(
entry_points={
'console_scripts': [
'lbrynet=lbry.extras.cli:main',
'torba-server=lbry.wallet.server.cli:main',
'lbry-hub=lbry.wallet.server.cli:main',
'orchstr8=lbry.wallet.orchstr8.cli:main',
'lbry-hub-elastic-sync=lbry.wallet.server.db.elasticsearch.sync:run_elastic_sync'
],
},
install_requires=[
'aiohttp==3.5.4',
'aioupnp==0.0.17',
'aioupnp==0.0.18',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
@@ -53,7 +54,8 @@ setup(
'coincurve==11.0.0',
'pbkdf2==1.3',
'attrs==18.2.0',
'pylru==1.1.0'
'pylru==1.1.0',
'elasticsearch==7.10.1'
] + PLYVEL,
classifiers=[
'Framework :: AsyncIO',
@@ -9,17 +9,32 @@ class BlockchainReorganizationTests(CommandTestCase):
VERBOSITY = logging.WARN
async def assertBlockHash(self, height):
self.assertEqual(
(await self.ledger.headers.hash(height)).decode(),
await self.blockchain.get_block_hash(height)
)
bp = self.conductor.spv_node.server.bp
def get_txids():
return [
bp.db.fs_tx_hash(tx_num)[0][::-1].hex()
for tx_num in range(bp.db.tx_counts[height - 1], bp.db.tx_counts[height])
]
block_hash = await self.blockchain.get_block_hash(height)
self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode())
self.assertEqual(block_hash, (await bp.db.fs_block_hashes(height, 1))[0][::-1].hex())
txids = await asyncio.get_event_loop().run_in_executor(bp.db.executor, get_txids)
txs = await bp.db.fs_transactions(txids)
block_txs = (await bp.daemon.deserialised_block(block_hash))['tx']
self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions')
self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order')
async def test_reorg(self):
bp = self.conductor.spv_node.server.bp
bp.reorg_count_metric.set(0)
# invalidate current block, move forward 2
self.assertEqual(self.ledger.headers.height, 206)
await self.assertBlockHash(206)
height = 206
self.assertEqual(self.ledger.headers.height, height)
await self.assertBlockHash(height)
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.generate(2)
await self.ledger.on_header.where(lambda e: e.height == 207)
@@ -37,6 +52,11 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.assertBlockHash(207)
await self.assertBlockHash(208)
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
await self.blockchain.generate(3)
await self.ledger.on_header.where(lambda e: e.height == 211)
await self.assertBlockHash(209)
await self.assertBlockHash(210)
await self.assertBlockHash(211)
async def test_reorg_change_claim_height(self):
# sanity check
@@ -51,6 +71,8 @@ class BlockchainReorganizationTests(CommandTestCase):
# create a claim and verify it's returned by claim_search
self.assertEqual(self.ledger.headers.height, 207)
await self.assertBlockHash(207)
broadcast_tx = await self.daemon.jsonrpc_stream_create(
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
@@ -58,6 +80,8 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.generate(1)
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
self.assertEqual(self.ledger.headers.height, 208)
await self.assertBlockHash(208)
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
txo = txos[0]
@@ -75,29 +99,21 @@ class BlockchainReorganizationTests(CommandTestCase):
await self.blockchain.clear_mempool()
await self.blockchain.generate(2)
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
# verify the claim was dropped from block 208 as returned by lbrycrdd
reorg_block_hash = await self.blockchain.get_block_hash(208)
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
block_207 = await self.blockchain.get_block(reorg_block_hash)
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
# verify the dropped claim is no longer returned by claim search
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertListEqual(txos, [])
# verify the claim published a block earlier wasn't also reverted
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
self.assertEqual(1, len(txos))
self.assertEqual(207, txos[0].tx_ref.height)
# broadcast the claim in a different block
new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode())
self.assertEqual(broadcast_tx.id, new_txid)
@@ -3,12 +3,15 @@ import tempfile
import logging
import asyncio
from binascii import unhexlify
from unittest import skip
from urllib.request import urlopen
from lbry.error import InsufficientFundsError
from lbry.extras.daemon.comment_client import verify
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
from lbry.testcase import CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.transaction import Transaction
from lbry.wallet.util import satoshis_to_coins as lbc
@@ -71,9 +74,22 @@ class ClaimSearchCommand(ClaimTestCase):
for claim, result in zip(claims, results):
self.assertEqual(
(claim['txid'], self.get_claim_id(claim)),
(result['txid'], result['claim_id'])
(result['txid'], result['claim_id']),
f"(expected {claim['outputs'][0]['name']}) != (got {result['name']})"
)
async def assertListsClaims(self, claims, **kwargs):
kwargs.setdefault('order_by', 'height')
results = await self.claim_list(**kwargs)
self.assertEqual(len(claims), len(results))
for claim, result in zip(claims, results):
self.assertEqual(
(claim['txid'], self.get_claim_id(claim)),
(result['txid'], result['claim_id']),
f"(expected {claim['outputs'][0]['name']}) != (got {result['name']})"
)
@skip("doesnt happen on ES...?")
async def test_disconnect_on_memory_error(self):
claim_ids = [
'0000000000000000000000000000000000000000',
@@ -89,10 +105,22 @@ class ClaimSearchCommand(ClaimTestCase):
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23829
] * 33829
with self.assertRaises(ConnectionResetError):
await self.claim_search(claim_ids=claim_ids)
async def test_claim_search_as_reader_server(self):
node2 = SPVNode(self.conductor.spv_module, node_number=2)
current_prefix = self.conductor.spv_node.server.bp.env.es_index_prefix
await node2.start(self.blockchain, extraconf={'ES_MODE': 'reader', 'ES_INDEX_PREFIX': current_prefix})
self.addCleanup(node2.stop)
self.ledger.network.config['default_servers'] = [(node2.hostname, node2.port)]
await self.ledger.stop()
await self.ledger.start()
channel2 = await self.channel_create('@abc', '0.1', allow_duplicate_name=True)
await asyncio.sleep(1) # fixme: find a way to block on the writer
await self.assertFindsClaims([channel2], name='@abc')
async def test_basic_claim_search(self):
await self.create_channel()
channel_txo = self.channel['outputs'][0]
@@ -133,6 +161,7 @@ class ClaimSearchCommand(ClaimTestCase):
claims = [three, two, signed]
await self.assertFindsClaims(claims, channel_ids=[self.channel_id])
await self.assertFindsClaims(claims, channel=f"@abc#{self.channel_id}")
await self.assertFindsClaims([], channel=f"@inexistent")
await self.assertFindsClaims([three, two, signed2, signed], channel_ids=[channel_id2, self.channel_id])
await self.channel_abandon(claim_id=self.channel_id)
await self.assertFindsClaims([], channel=f"@abc#{self.channel_id}", valid_channel_signature=True)
@@ -156,6 +185,23 @@ class ClaimSearchCommand(ClaimTestCase):
# abandoned stream won't show up for streams in channel search
await self.stream_abandon(txid=signed2['txid'], nout=0)
await self.assertFindsClaims([], channel_ids=[channel_id2])
# resolve by claim ids
await self.assertFindsClaims([three, two], claim_ids=[self.get_claim_id(three), self.get_claim_id(two)])
await self.assertFindsClaims([three], claim_id=self.get_claim_id(three))
await self.assertFindsClaims([three], claim_id=self.get_claim_id(three), text='*')
async def test_source_filter(self):
channel = await self.channel_create('@abc')
no_source = await self.stream_create('no-source', data=None)
normal = await self.stream_create('normal', data=b'normal')
normal_repost = await self.stream_repost(self.get_claim_id(normal), 'normal-repost')
no_source_repost = await self.stream_repost(self.get_claim_id(no_source), 'no-source-repost')
await self.assertFindsClaims([no_source_repost, no_source, channel], has_no_source=True)
await self.assertListsClaims([no_source, channel], has_no_source=True)
await self.assertFindsClaims([normal_repost, normal, channel], has_source=True)
await self.assertListsClaims([no_source_repost, normal_repost, normal], has_source=True)
await self.assertFindsClaims([no_source_repost, normal_repost, normal, no_source, channel])
await self.assertListsClaims([no_source_repost, normal_repost, normal, no_source, channel])
async def test_pagination(self):
await self.create_channel()
@@ -262,6 +308,19 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc')
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es')
await self.assertFindsClaims([claim4, claim3], any_languages=['en'])
await self.assertFindsClaims([claim2, claim1], any_languages=['none'])
await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_languages=['none', 'en'])
await self.assertFindsClaims([claim5], any_languages=['es'])
await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es'])
await self.assertFindsClaims([], fee_currency='foo')
async def test_search_by_channel(self):
@@ -319,6 +378,57 @@ class ClaimSearchCommand(ClaimTestCase):
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
async def test_limit_claims_per_channel(self):
match = self.assertFindsClaims
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
claim1 = await self.stream_create('claim1')
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
claim3 = await self.stream_create('claim3', channel_id=chan1_id)
claim4 = await self.stream_create('claim4', channel_id=chan1_id)
claim5 = await self.stream_create('claim5', channel_id=chan2_id)
claim6 = await self.stream_create('claim6', channel_id=chan2_id)
await match(
[claim6, claim5, claim4, claim3, claim1],
limit_claims_per_channel=2, claim_type='stream'
)
await match(
[claim6, claim5, claim4, claim3, claim2, claim1],
limit_claims_per_channel=3, claim_type='stream'
)
async def test_limit_claims_per_channel_across_sorted_pages(self):
await self.generate(10)
match = self.assertFindsClaims
channel_id = self.get_claim_id(await self.channel_create('@chan0'))
claims = []
first = await self.stream_create('claim0', channel_id=channel_id)
second = await self.stream_create('claim1', channel_id=channel_id)
for i in range(2, 10):
some_chan = self.get_claim_id(await self.channel_create(f'@chan{i}', bid='0.001'))
claims.append(await self.stream_create(f'claim{i}', bid='0.001', channel_id=some_chan))
last = await self.stream_create('claim10', channel_id=channel_id)
await match(
[first, second, claims[0], claims[1]], page_size=4,
limit_claims_per_channel=3, claim_type='stream', order_by=['^height']
)
# second goes out
await match(
[first, claims[0], claims[1], claims[2]], page_size=4,
limit_claims_per_channel=1, claim_type='stream', order_by=['^height']
)
# second appears, from replacement queue
await match(
[second, claims[3], claims[4], claims[5]], page_size=4, page=2,
limit_claims_per_channel=1, claim_type='stream', order_by=['^height']
)
# last is unaffected, as the limit applies per page
await match(
[claims[6], claims[7], last], page_size=4, page=3,
limit_claims_per_channel=1, claim_type='stream', order_by=['^height']
)
async def test_claim_type_and_media_type_search(self):
# create an invalid/unknown claim
address = await self.account.receiving.get_or_create_usable_address()
@@ -332,6 +442,7 @@ class ClaimSearchCommand(ClaimTestCase):
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
channel = await self.channel_create()
unknown = self.sout(tx)
@@ -339,6 +450,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
await self.assertFindsClaims([collection], claim_type='collection')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
@@ -389,10 +501,11 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([claim2], text='autobiography')
await self.assertFindsClaims([claim3], text='history')
await self.assertFindsClaims([claim4], text='conspiracy')
await self.assertFindsClaims([], text='conspiracy AND history')
await self.assertFindsClaims([claim4, claim3], text='conspiracy OR history')
await self.assertFindsClaims([claim1, claim4, claim2, claim3], text='documentary')
await self.assertFindsClaims([claim4, claim1, claim2, claim3], text='satoshi')
await self.assertFindsClaims([], text='conspiracy+history')
await self.assertFindsClaims([claim4, claim3], text='conspiracy|history')
await self.assertFindsClaims([claim1, claim4, claim2, claim3], text='documentary', order_by=[])
# todo: check why claim1 and claim2 order changed. used to be ...claim1, claim2...
await self.assertFindsClaims([claim4, claim2, claim1, claim3], text='satoshi', order_by=[])
claim2 = await self.stream_update(
self.get_claim_id(claim2), clear_tags=True, tags=['cloud'],
@@ -428,6 +541,47 @@ class TransactionCommands(ClaimTestCase):
class TransactionOutputCommands(ClaimTestCase):
async def test_txo_list_resolve_supports(self):
channel = self.get_claim_id(await self.channel_create('@identity'))
stream = self.get_claim_id(await self.stream_create())
support = await self.support_create(stream, channel_id=channel)
r, = await self.txo_list(type='support')
self.assertEqual(r['txid'], support['txid'])
self.assertNotIn('name', r['signing_channel'])
r, = await self.txo_list(type='support', resolve=True)
self.assertIn('name', r['signing_channel'])
self.assertEqual(r['signing_channel']['name'], '@identity')
async def test_txo_list_by_channel_filtering(self):
channel_foo = self.get_claim_id(await self.channel_create('@foo'))
channel_bar = self.get_claim_id(await self.channel_create('@bar'))
stream_a = self.get_claim_id(await self.stream_create('a', channel_id=channel_foo))
stream_b = self.get_claim_id(await self.stream_create('b', channel_id=channel_foo))
stream_c = self.get_claim_id(await self.stream_create('c', channel_id=channel_bar))
stream_d = self.get_claim_id(await self.stream_create('d'))
support_c = await self.support_create(stream_c, '0.3', channel_id=channel_foo)
support_d = await self.support_create(stream_d, '0.3', channel_id=channel_bar)
r = await self.txo_list(type='stream')
self.assertEqual({stream_a, stream_b, stream_c, stream_d}, {c['claim_id'] for c in r})
r = await self.txo_list(type='stream', channel_id=channel_foo)
self.assertEqual({stream_a, stream_b}, {c['claim_id'] for c in r})
r = await self.txo_list(type='support', channel_id=channel_foo)
self.assertEqual({support_c['txid']}, {s['txid'] for s in r})
r = await self.txo_list(type='support', channel_id=channel_bar)
self.assertEqual({support_d['txid']}, {s['txid'] for s in r})
r = await self.txo_list(type='stream', channel_id=[channel_foo, channel_bar])
self.assertEqual({stream_a, stream_b, stream_c}, {c['claim_id'] for c in r})
r = await self.txo_list(type='stream', not_channel_id=channel_foo)
self.assertEqual({stream_c, stream_d}, {c['claim_id'] for c in r})
r = await self.txo_list(type='stream', not_channel_id=[channel_foo, channel_bar])
self.assertEqual({stream_d}, {c['claim_id'] for c in r})
async def test_txo_list_and_sum_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
@@ -830,6 +984,22 @@ class ClaimCommands(ClaimTestCase):
self.assertEqual('0.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
async def stream_update_and_wait(self, claim_id, **kwargs):
tx = await self.daemon.jsonrpc_stream_update(claim_id, **kwargs)
await self.ledger.wait(tx)
async def test_claim_list_pending_edits_ordering(self):
stream5_id = self.get_claim_id(await self.stream_create('five'))
stream4_id = self.get_claim_id(await self.stream_create('four'))
stream3_id = self.get_claim_id(await self.stream_create('three'))
stream2_id = self.get_claim_id(await self.stream_create('two'))
stream1_id = self.get_claim_id(await self.stream_create('one'))
await self.assertClaimList([stream1_id, stream2_id, stream3_id, stream4_id, stream5_id])
await self.stream_update_and_wait(stream4_id, title='foo')
await self.assertClaimList([stream4_id, stream1_id, stream2_id, stream3_id, stream5_id])
await self.stream_update_and_wait(stream3_id, title='foo')
await self.assertClaimList([stream4_id, stream3_id, stream1_id, stream2_id, stream5_id])
class ChannelCommands(CommandTestCase):
@@ -972,6 +1142,18 @@ class ChannelCommands(CommandTestCase):
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2)
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1)
async def test_sign_hex_encoded_data(self):
data_to_sign = "CAFEBABE"
# claim new name
await self.channel_create('@someotherchan')
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
await self.confirm_tx(channel_tx.id)
channel = channel_tx.outputs[0]
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
async def test_channel_export_import_before_sending_channel(self):
# export
tx = await self.channel_create('@foo', '1.0')
@@ -1250,6 +1432,11 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# same search, but details omitted by 'no_totals'
last_result = result
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content', no_totals=True))
self.assertEqual(result['items'], last_result['items'])
# search inside channel containing filtered content
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
filtered = result['blocked']
@@ -1259,6 +1446,11 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(1, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# same search, but details omitted by 'no_totals'
last_result = result
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', no_totals=True))
self.assertEqual(result['items'], last_result['items'])
# content was filtered by not_tag before censoring
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
self.assertEqual(0, len(result['items']))
@@ -1312,6 +1504,13 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(3, filtered['channels'][0]['blocked'])
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
# same search, but details omitted by 'no_totals'
last_result = result
result = await self.out(
self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height'], no_totals=True)
)
self.assertEqual(result['items'], last_result['items'])
# filtered channel should still resolve
result = await self.resolve('lbry://@bad_channel')
self.assertEqual(bad_channel_id, result['claim_id'])
@@ -1756,9 +1955,6 @@ class StreamCommands(ClaimTestCase):
with self.assertRaisesRegex(Exception, "'bid' is a required argument for new publishes."):
await self.daemon.jsonrpc_publish('foo')
with self.assertRaisesRegex(Exception, "'file_path' is a required argument for new publishes."):
await self.daemon.jsonrpc_publish('foo', bid='1.0')
# successfully create stream
with tempfile.NamedTemporaryFile() as file:
file.write(b'hi')
@@ -1801,6 +1997,41 @@ class StreamCommands(ClaimTestCase):
self.assertEqual(claim['value']['languages'], ['uk-UA'])
self.assertEqual(claim['value']['tags'], ['anime'])
# publish a stream with no source
tx5 = await self.publish(
'future-release', bid='0.1', languages='uk-UA', tags=['Anime', 'anime ']
)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
claim = await self.resolve('lbry://future-release')
self.assertEqual(claim['txid'], tx5['outputs'][0]['txid'])
self.assertNotIn('signing_channel', claim)
self.assertEqual(claim['value']['languages'], ['uk-UA'])
self.assertEqual(claim['value']['tags'], ['anime'])
self.assertNotIn('source', claim['value'])
# change metadata before the release
await self.publish(
'future-release', bid='0.1', tags=['Anime', 'anime ', 'psy-trance'], title='Psy will be over 9000!!!'
)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
claim = await self.resolve('lbry://future-release')
self.assertEqual(claim['value']['tags'], ['anime', 'psy-trance'])
self.assertEqual(claim['value']['title'], 'Psy will be over 9000!!!')
self.assertNotIn('source', claim['value'])
# update the stream to have a source
tx6 = await self.publish(
'future-release', sd_hash='beef', file_hash='beef',
file_name='blah.mp3', tags=['something-else']
)
claim = await self.resolve('lbry://future-release')
self.assertEqual(claim['txid'], tx6['outputs'][0]['txid'])
self.assertEqual(claim['value']['tags'], ['something-else'])
self.assertEqual(claim['value']['source']['sd_hash'], 'beef')
self.assertEqual(claim['value']['source']['hash'], 'beef')
self.assertEqual(claim['value']['source']['name'], 'blah.mp3')
self.assertEqual(claim['value']['source']['media_type'], 'audio/mpeg')
class SupportCommands(CommandTestCase):
@@ -1888,10 +2119,14 @@ class SupportCommands(CommandTestCase):
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
async def test_signed_supports(self):
async def test_signed_supports_with_no_change_txo_regression(self):
# reproduces a bug where transactions did not get properly signed
# if there was no change and just a single output
# lbrycrd returned 'the transaction was rejected by network rules.'
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
tx = await self.support_create(stream_id, '0.3', channel_id=channel_id)
tx = await self.support_create(stream_id, '7.967598', channel_id=channel_id)
self.assertEqual(len(tx['outputs']), 1) # must be one to reproduce bug
self.assertTrue(tx['outputs'][0]['is_channel_signature_valid'])
@@ -1937,6 +2172,10 @@ class CollectionCommands(CommandTestCase):
await self.collection_update(claim_id, clear_claims=True, claims=claim_ids[:2])
collections = await self.out(self.daemon.jsonrpc_collection_list())
self.assertEquals(len(collections['items']), 2)
self.assertNotIn('canonical_url', collections['items'][0])
resolved_collections = await self.out(self.daemon.jsonrpc_collection_list(resolve=True))
self.assertIn('canonical_url', resolved_collections['items'][0])
await self.collection_abandon(claim_id)
self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 1)
+73 -36
View File
@@ -4,9 +4,12 @@ import lbry
from unittest.mock import Mock
from lbry.wallet.network import Network
from lbry.wallet.orchstr8 import Conductor
from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.rpc import RPCSession
from lbry.wallet.server.udp import StatusServer
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
from lbry.conf import Config
class NetworkTests(IntegrationTestCase):
@@ -32,18 +35,17 @@ class NetworkTests(IntegrationTestCase):
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
# await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
self.conductor.blockchain_node,
extraconf={
'DESCRIPTION': 'Fastest server in the west.',
'PAYMENT_ADDRESS': payment_address,
'DONATION_ADDRESS': donation_address,
'DAILY_FEE': '42'
}
)
await self.ledger.network.on_connected.first
self.conductor.spv_node.server.env.payment_address = payment_address
self.conductor.spv_node.server.env.donation_address = donation_address
self.conductor.spv_node.server.env.description = 'Fastest server in the west.'
self.conductor.spv_node.server.env.daily_fee = '42'
from lbry.wallet.server.session import LBRYElectrumX
LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env)
# await self.ledger.network.on_connected.first
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
@@ -65,22 +67,20 @@ class ReconnectTests(IntegrationTestCase):
async def test_multiple_servers(self):
# we have a secondary node that connects later, so
node2 = SPVNode(self.conductor.spv_module, node_number=2)
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
await asyncio.wait_for(self.ledger.stop(), timeout=1)
await asyncio.wait_for(self.ledger.start(), timeout=1)
self.ledger.network.session_pool.new_connection_event.clear()
await node2.start(self.blockchain)
# this is only to speed up the test as retrying would take 4+ seconds
for session in self.ledger.network.session_pool.sessions:
session.trigger_urgent_reconnect.set()
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
self.ledger.network.config['default_servers'].reverse()
self.assertEqual(50002, self.ledger.network.client.server[1])
await self.ledger.stop()
await self.ledger.start()
self.assertTrue(self.ledger.network.is_connected)
switch_event = self.ledger.network.on_connected.first
self.assertEqual(50003, self.ledger.network.client.server[1])
await node2.stop(True)
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
with self.assertRaises(asyncio.TimeoutError):
await asyncio.wait_for(switch_event, timeout=1)
self.assertFalse(self.ledger.network.is_connected)
await self.ledger.resolve([], ['derp'])
self.assertEqual(50002, self.ledger.network.client.server[1])
async def test_direct_sync(self):
await self.ledger.stop()
@@ -98,10 +98,13 @@ class ReconnectTests(IntegrationTestCase):
async def test_connection_drop_still_receives_events_after_reconnected(self):
address1 = await self.account.receiving.get_or_create_usable_address()
# disconnect and send a new tx, should reconnect and get it
self.ledger.network.client.connection_lost(Exception())
self.ledger.network.client.transport.close()
self.assertFalse(self.ledger.network.is_connected)
await self.ledger.resolve([], 'derp')
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
# await self.ledger.resolve([], 'derp')
# self.assertTrue(self.ledger.network.is_connected)
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
await self.blockchain.generate(1)
await self.on_transaction_id(sendtxid) # confirmed
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
@@ -123,7 +126,7 @@ class ReconnectTests(IntegrationTestCase):
await self.blockchain.generate(1)
# (this is just so the test doesn't hang forever if it doesn't reconnect)
if not self.ledger.network.is_connected:
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0)
# omg, the burned cable still works! torba is fire proof!
await self.ledger.network.get_transaction(sendtxid)
@@ -136,15 +139,47 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_online_but_still_unavailable(self):
# Edge case. See issue #2445 for context
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
for session in self.ledger.network.session_pool.sessions:
session.response_time = None
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
async def test_timeout_and_concurrency_propagated_from_config(self):
conf = Config()
self.assertEqual(self.ledger.network.client.timeout, 30)
self.assertEqual(self.ledger.network.client.concurrency, 32)
conf.hub_timeout = 123.0
conf.concurrent_hub_requests = 42
conf.lbryum_servers = self.ledger.config['default_servers']
conf.wallet_dir = self.ledger.config['data_path']
self.manager.config = conf
await self.manager.reset()
self.assertEqual(self.ledger.network.client.timeout, 123)
self.assertEqual(self.ledger.network.client.concurrency, 42)
# async def test_online_but_still_unavailable(self):
# # Edge case. See issue #2445 for context
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
# for session in self.ledger.network.session_pool.sessions:
# session.response_time = None
# self.assertIsNone(self.ledger.network.session_pool.fastest_session)
class UDPServerFailDiscoveryTest(AsyncioTestCase):
async def test_wallet_connects_despite_lack_of_udp(self):
conductor = Conductor()
conductor.spv_node.udp_port = '0'
await conductor.start_blockchain()
self.addCleanup(conductor.stop_blockchain)
await conductor.start_spv()
self.addCleanup(conductor.stop_spv)
self.assertFalse(conductor.spv_node.server.bp.status_server.is_running)
await asyncio.wait_for(conductor.start_wallet(), timeout=5)
self.addCleanup(conductor.stop_wallet)
self.assertTrue(conductor.wallet_node.ledger.network.is_connected)
class ServerPickingTestCase(AsyncioTestCase):
async def _make_udp_server(self, port):
s = StatusServer()
await s.start(0, b'\x00' * 32, '127.0.0.1', port)
self.addCleanup(s.stop)
async def _make_fake_server(self, latency=1.0, port=1):
# local fake server with artificial latency
@@ -156,6 +191,7 @@ class ServerPickingTestCase(AsyncioTestCase):
return {'height': 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
await self._make_udp_server(port)
return '127.0.0.1', port
async def _make_bad_server(self, port=42420):
@@ -164,9 +200,10 @@ class ServerPickingTestCase(AsyncioTestCase):
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
await self._make_udp_server(port)
return '127.0.0.1', port
async def test_pick_fastest(self):
async def _test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
# fast but unhealthy, should be discarded
@@ -182,8 +219,8 @@ class ServerPickingTestCase(AsyncioTestCase):
network = Network(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
await network.start()
await asyncio.wait_for(network.on_connected.first, timeout=10)
self.assertTrue(network.is_connected)
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
@@ -11,7 +11,9 @@ class PurchaseCommandTests(CommandTestCase):
await super().asyncSetUp()
self.merchant_address = await self.blockchain.get_raw_change_address()
async def priced_stream(self, name='stream', price: Optional[str] = '2.0', currency='LBC') -> Transaction:
async def priced_stream(
self, name='stream', price: Optional[str] = '0.2', currency='LBC', mine=False
) -> Transaction:
kwargs = {}
if price and currency:
kwargs = {
@@ -19,6 +21,8 @@ class PurchaseCommandTests(CommandTestCase):
'fee_currency': currency,
'fee_address': self.merchant_address
}
if not mine:
kwargs['claim_address'] = self.merchant_address
file_path = self.create_upload_file(data=b'high value content')
tx = await self.daemon.jsonrpc_stream_create(
name, '0.01', file_path=file_path, **kwargs
@@ -117,6 +121,13 @@ class PurchaseCommandTests(CommandTestCase):
abc_stream = await self.priced_stream('abc')
await self.assertStreamPurchased(abc_stream, lambda: self.daemon.jsonrpc_purchase_create(url='lbry://abc'))
# purchase without valid exchange rate fails
erm = self.daemon.component_manager.get_component('exchange_rate_manager')
for feed in erm.market_feeds:
feed.last_check -= 10_000
with self.assertRaisesRegex(Exception, "Unable to convert 50 from USD to LBC"):
await self.daemon.jsonrpc_purchase_create(claim_id, allow_duplicate_purchase=True)
async def test_purchase_and_transaction_list(self):
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
self.assertItemCount(await self.daemon.jsonrpc_transaction_list(), 1)
@@ -173,7 +184,13 @@ class PurchaseCommandTests(CommandTestCase):
await self.generate(1)
await self.ledger.wait(purchase)
await self.assertBalance(self.account, '10.987893')
# confirm that available and reserved take into account purchase received
self.assertEqual(await self.account.get_detailed_balance(), {
'total': 1099789300,
'available': 1098789300,
'reserved': 1000000,
'reserved_subtotals': {'claims': 1000000, 'supports': 0, 'tips': 0}
})
self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 2)
spend = await self.daemon.jsonrpc_wallet_send('10.5', address2)
@@ -182,3 +199,12 @@ class PurchaseCommandTests(CommandTestCase):
await self.ledger.wait(spend)
await self.assertBalance(self.account, '0.487695')
self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1)
async def test_owner_not_required_purchase_own_content(self):
await self.priced_stream(mine=True)
# check that `get` doesn't purchase own claim
balance = await self.account.get_balance()
response = await self.daemon.jsonrpc_get('lbry://stream')
self.assertIsNone(response.content_fee)
self.assertEqual(await self.account.get_balance(), balance)
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
@@ -117,11 +117,6 @@ class ResolveCommand(BaseResolveTestCase):
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo:1', claim_id1)
await self.assertResolvesToClaimId('foo:2', claim_id2)
await self.assertResolvesToClaimId('foo:3', claim_id3)
await self.assertResolvesToClaimId('foo:4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
@@ -138,8 +138,8 @@ class BasicTransactionTests(IntegrationTestCase):
# evil trick: mempool is unsorted on real life, but same order between python instances. reproduce it
original_summary = self.conductor.spv_node.server.mempool.transaction_summaries
async def random_summary(*args, **kwargs):
summary = await original_summary(*args, **kwargs)
def random_summary(*args, **kwargs):
summary = original_summary(*args, **kwargs)
if summary and len(summary) > 2:
ordered = summary.copy()
while summary == ordered:
@@ -10,7 +10,7 @@ from lbry.wallet.dewies import dict_values_to_lbc
class WalletCommands(CommandTestCase):
async def test_wallet_create_and_add_subscribe(self):
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions))
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions.values()))
self.assertEqual(len(session.hashX_subs), 27)
wallet = await self.daemon.jsonrpc_wallet_create('foo', create_account=True, single_key=True)
self.assertEqual(len(session.hashX_subs), 28)
@@ -43,12 +43,14 @@ class WalletCommands(CommandTestCase):
)
async def test_wallet_reconnect(self):
status = await self.daemon.jsonrpc_status()
self.assertEqual(len(status['wallet']['servers']), 1)
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
await self.conductor.spv_node.stop(True)
self.conductor.spv_node.port = 54320
await self.conductor.spv_node.start(self.conductor.blockchain_node)
status = await self.daemon.jsonrpc_status()
self.assertEqual(len(status['wallet']['servers']), 1)
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
self.assertEqual(len(status['wallet']['servers']), 0)
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
await self.daemon.jsonrpc_wallet_reconnect()
status = await self.daemon.jsonrpc_status()
@@ -74,7 +76,7 @@ class WalletCommands(CommandTestCase):
}
self.assertIsNone(ledger._balance_cache.get(self.account.id))
query_count += 6
query_count += 2
self.assertEqual(await wallet_balance(), expected)
self.assertEqual(self.ledger.db.db.query_count, query_count)
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0')
@@ -98,7 +100,7 @@ class WalletCommands(CommandTestCase):
# on_transaction event reset balance cache
query_count = self.ledger.db.db.query_count
self.assertEqual(await wallet_balance(), expected)
query_count += 3 # only one of the accounts changed
query_count += 1 # only one of the accounts changed
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '9.979893')
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0')
self.assertEqual(self.ledger.db.db.query_count, query_count)
@@ -4,6 +4,7 @@ import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.wallet.server.session import LBRYElectrumX
from lbry.testcase import IntegrationTestCase, CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
@@ -46,7 +47,7 @@ class TestSessions(IntegrationTestCase):
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
async def _test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
wallet_pay_service.payment_period = 1
# only starts with a positive max key fee
@@ -68,21 +69,21 @@ class TestUsagePayment(CommandTestCase):
self.addCleanup(node.stop)
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
LBRYElectrumX.set_server_features(node.server.env)
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
node.server.env.daily_fee = "1.0"
node.server.env.payment_address = address
LBRYElectrumX.set_server_features(node.server.env)
# self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
@@ -9,6 +9,10 @@ from lbry.wallet import Transaction
class FileCommands(CommandTestCase):
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self.skip_libtorrent = False
async def initialize_torrent(self, tx_to_update=None):
if not hasattr(self, 'seeder_session'):
self.seeder_session = TorrentSession(self.loop, None)
@@ -217,6 +221,20 @@ class FileCommands(CommandTestCase):
file_list = (await self.daemon.jsonrpc_file_list())['items']
self.assertEqual(file_list[0].stream_claim_info.claim.stream.description, claim.stream.description)
async def test_sourceless_content(self):
# claim has no source, then it has one
tx = await self.stream_create('foo', '0.01', data=None)
claim_id = self.get_claim_id(tx)
await self.daemon.jsonrpc_file_delete(claim_name='foo')
response = await self.out(self.daemon.jsonrpc_get('lbry://foo'))
self.assertIn('error', response)
self.assertIn('nothing to download', response['error'])
# source is set (there isn't a way to clear the source field, so we stop here for now)
await self.stream_update(claim_id, data=b'surpriiiiiiiise')
response = await self.out(self.daemon.jsonrpc_get('lbry://foo'))
self.assertNotIn('error', response)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
async def test_file_list_paginated_output(self):
await self.create_streams_in_range(0, 20)
@@ -354,7 +372,8 @@ class FileCommands(CommandTestCase):
# FAIL: beyond available balance
await self.stream_create(
'expensive', '0.01', data=b'pay me if you can',
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
fee_currency='LBC', fee_amount='11.0',
fee_address=target_address, claim_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
@@ -364,7 +383,8 @@ class FileCommands(CommandTestCase):
# FAIL: beyond maximum key fee
await self.stream_create(
'maxkey', '0.01', data=b'no pay me, no',
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
fee_currency='LBC', fee_amount='111.0',
fee_address=target_address, claim_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
@@ -376,7 +396,8 @@ class FileCommands(CommandTestCase):
# PASS: purchase is successful
await self.stream_create(
'icanpay', '0.01', data=b'I got the power!',
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
fee_currency='LBC', fee_amount='1.0',
fee_address=target_address, claim_address=target_address
)
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
await self.assertBalance(self.account, '9.925679')
@@ -427,7 +448,8 @@ class FileCommands(CommandTestCase):
await self.stream_create(
'somename', '0.5', data=b'Yes, please',
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
fee_currency='LBC', fee_amount='1.0',
fee_address=target_address, claim_address=target_address
)
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
# Assert the fee and bid are subtracted
+5 -3
View File
@@ -7,7 +7,8 @@ from lbry.extras import cli
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
@@ -22,7 +23,8 @@ class CLIIntegrationTest(AsyncioTestCase):
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)
@@ -34,4 +36,4 @@ class CLIIntegrationTest(AsyncioTestCase):
with contextlib.redirect_stdout(actual_output):
cli.main(["--api", "localhost:5299", "status"])
actual_output = actual_output.getvalue()
self.assertIn("connection_status", actual_output)
self.assertIn("is_running", actual_output)
+332 -54
View File
@@ -1,8 +1,8 @@
import re
import time
import typing
from math import ceil
import secrets
from aiohttp import web
@@ -30,6 +30,17 @@ class MockedCommentServer:
'timestamp': None,
'channel_url': None,
'is_hidden': False,
'is_pinned': False,
}
REACT_SCHEMA = {
'react_id': None,
'comment_id': None,
'reaction_type': None,
'timestamp': None,
'signature': None,
'channel_id': None,
'channel_name': None
}
def __init__(self, port=2903):
@@ -39,7 +50,9 @@ class MockedCommentServer:
self.runner = None
self.server = None
self.comments = []
self.comment_id = 0
self.reacts = {}
self.index = 0
self.react_id = 0
@classmethod
def _create_comment(cls, **kwargs):
@@ -47,15 +60,22 @@ class MockedCommentServer:
schema.update(**kwargs)
return schema
@classmethod
def _react(cls, **kwargs):
schema = cls.REACT_SCHEMA.copy()
schema.update(**kwargs)
return schema
def create_comment(self, claim_id=None, parent_id=None, channel_name=None, channel_id=None, **kwargs):
comment_id = self.comment_id
comment_id = secrets.token_hex(64)
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
if parent_id:
claim_id = self.comments[self.get_comment_id(parent_id)]['claim_id']
parent_comment = list(filter(lambda c: c['comment_id'] == parent_id, self.comments))[0]
claim_id = parent_comment['claim_id']
comment = self._create_comment(
comment_id=str(comment_id),
comment_id=comment_id,
channel_name=channel_name,
channel_id=channel_id,
channel_url=channel_url,
@@ -65,15 +85,14 @@ class MockedCommentServer:
**kwargs
)
self.comments.append(comment)
self.comment_id += 1
return self.clean(comment)
def abandon_comment(self, comment_id: int, channel_id: str, **kwargs):
def abandon_comment(self, comment_id: str, channel_id: str, **kwargs):
deleted = False
comment_id = self.get_comment_id(comment_id)
index = self.get_index_for_comment_id(comment_id)
try:
if self.comments[comment_id]['channel_id'] == channel_id:
self.comments.pop(comment_id)
if index >= 0:
self.comments.pop(index)
deleted = True
finally:
return {
@@ -82,29 +101,43 @@ class MockedCommentServer:
}
}
def edit_comment(self, comment_id: typing.Union[str, int], comment: str, channel_id: str,
def edit_comment(self, comment_id: str, comment: str, channel_id: str,
channel_name: str, signature: str, signing_ts: str) -> dict:
edited = False
if self.credentials_are_valid(channel_id, channel_name, signature, signing_ts) \
and self.is_valid_body(comment):
comment_id = self.get_comment_id(comment_id)
if self.comments[comment_id]['channel_id'] == channel_id:
self.comments[comment_id].update({
index = self.get_index_for_comment_id(comment_id)
if self.comments[index]['channel_id'] == channel_id:
self.comments[index].update({
'comment': comment,
'signature': signature,
'signing_ts': signing_ts
})
edited = True
return self.comments[comment_id] if edited else None
return self.comments[index] if edited else None
def hide_comment(self, comment_id: typing.Union[int, str], signing_ts: str, signature: str):
comment_id = self.get_comment_id(comment_id)
def hide_comment(self, comment_id: str, signing_ts: str, signature: str):
if self.is_signable(signature, signing_ts):
self.comments[comment_id]['is_hidden'] = True
self.comments[self.get_index_for_comment_id(comment_id)]['is_hidden'] = True
return True
return False
def pin_comment(
self,
comment_id: str,
channel_name: str, channel_id: str, remove: bool,
signing_ts: str, signature: str
):
index = self.get_index_for_comment_id(comment_id)
if self.is_signable(signature, signing_ts):
if remove:
self.comments[index]['is_pinned'] = False
else:
self.comments[index]['is_pinned'] = True
return self.comments[index]
return False
def hide_comments(self, pieces: list):
hidden = []
for p in pieces:
@@ -117,38 +150,29 @@ class MockedCommentServer:
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
def get_claim_comments(self, claim_id, hidden=False, visible=False, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
if hidden ^ visible:
comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(comments)/page_size),
'total_items': len(comments),
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], self.comments)))
}
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(select_comments) / page_size),
'total_items': len(select_comments),
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[self.get_comment_id(comment_id)]
def get_comment_channel_by_id(self, comment_id: str, **kwargs):
comment = self.comments[self.get_index_for_comment_id(comment_id)]
return {
'channel_id': comment['channel_id'],
'channel_name': comment['channel_name'],
}
def get_comments_by_id(self, comment_ids: list):
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
comments = [self.comments[self.get_index_for_comment_id(cid)] for cid in comment_ids]
return {
'page': 1,
'page_size': len(comment_ids),
@@ -157,15 +181,99 @@ class MockedCommentServer:
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
def react(
self,
comment_ids=None,
channel_name=None,
channel_id=None,
remove=None,
clear_types=None,
type=None,
signing_ts=None,
**kwargs
):
comment_batch = comment_ids.split(',')
for item in comment_batch:
react_id = self.react_id
c_id = item.strip()
reacts_for_comment_id = [r for r in list(self.reacts.values()) if r['comment_id'] == c_id]
channels_reacts_for_comment_id = [r for r in reacts_for_comment_id if r['channel_id'] == channel_id]
if remove:
matching_react = None
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == type:
matching_react = reaction
break
if matching_react:
self.reacts.pop(int(matching_react['react_id']), None)
else:
if clear_types:
for r_type in clear_types.split(','):
for reaction in channels_reacts_for_comment_id:
if reaction['reaction_type'] == r_type:
x = self.reacts.pop(int(reaction['react_id']), None)
react = self._react(
react_id=str(react_id),
comment_id=str(c_id),
channel_name=channel_name,
channel_id=channel_id,
reaction_type=type,
timestamp=str(int(time.time())),
**kwargs
)
self.reacts[self.react_id] = react
self.react_id += 1
self.clean(react)
return True
def list_reacts(self, comment_ids, channel_id, channel_name, types=None, **kwargs):
all_types = list(set([r['reaction_type'] for r in list(self.reacts.values())]))
comment_id_list = comment_ids.split(',')
# _reacts: {'a1': {'like': 0, 'dislike': 0}, 'a2': {'like': 0, 'dislike': 0}}
own_reacts = {}
other_reacts = {}
# for each comment_id
# add comment_id: {} to own_reacts and other_reacts
# for each react in own_reacts
# for each react in other_reacts
for cid in comment_id_list:
own_reacts[cid] = {}
other_reacts[cid] = {}
for r_type in all_types:
own_reacts[cid][r_type] = 0
other_reacts[cid][r_type] = 0
# process that comment ids reactions for own and other categories
reacts_for_comment = list(filter(lambda c: c['comment_id'] == cid, list(self.reacts.values())))
if types:
reacts_for_comment = list(filter(lambda c: c['reaction_type'] in types.split(','), reacts_for_comment))
own_reacts_for_comment = list(filter(lambda c: c['channel_id'] == channel_id, reacts_for_comment))
other_reacts_for_comment = list(filter(lambda c: c['channel_id'] != channel_id, reacts_for_comment))
if own_reacts_for_comment:
for react in own_reacts_for_comment:
own_reacts[cid][react['reaction_type']] += 1
if other_reacts_for_comment:
for react in other_reacts_for_comment:
other_reacts[cid][react['reaction_type']] += 1
return {
'my_reactions': own_reacts,
'others_reactions': other_reacts,
}
methods = {
'get_claim_comments': get_claim_comments,
'comment.List': get_claim_comments,
'get_comments_by_id': get_comments_by_id,
'create_comment': create_comment,
'abandon_comment': abandon_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
'get_claim_hidden_comments': get_claim_hidden_comments,
'hide_comments': hide_comments,
'edit_comment': edit_comment,
'comment.Create': create_comment,
'comment.Abandon': abandon_comment,
'comment.GetChannelFromCommentID': get_comment_channel_by_id,
'comment.Hide': hide_comments,
'comment.Pin': pin_comment,
'comment.Edit': edit_comment,
'reaction.React': react,
'reaction.List': list_reacts,
}
def process_json(self, body) -> dict:
@@ -223,17 +331,18 @@ class MockedCommentServer:
return 0 < len(comment) <= 2000
def is_valid_comment_id(self, comment_id: typing.Union[int, str]) -> bool:
if isinstance(comment_id, str) and comment_id.isalnum():
comment_id = int(comment_id)
if isinstance(comment_id, int):
return 0 <= comment_id < len(self.comments)
if isinstance(comment_id, str):
return True
return False
def get_comment_id(self, cid: typing.Union[int, str, any]) -> int:
if not self.is_valid_comment_id(cid):
raise ValueError('Comment ID is Invalid')
return cid if isinstance(cid, int) else int(cid)
def get_comment_for_id(self, cid: str) -> dict:
return list(filter(lambda c: c['comment_id'] == cid, self.comments))[0]
def get_index_for_comment_id(self, value: str):
for i, dic in enumerate(self.comments):
if dic['comment_id'] == value:
return i
return -1
@staticmethod
def claim_id_is_valid(claim_id: str) -> bool:
@@ -276,7 +385,6 @@ class MockedCommentServer:
(parent_id is None or self.is_valid_comment_id(parent_id))
class CommentCommands(CommandTestCase):
async def asyncSetUp(self):
@@ -304,7 +412,7 @@ class CommentCommands(CommandTestCase):
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
await self.daemon.jsonrpc_comment_create(
comment2 = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_name=channel2['name'],
comment='Let\'s all go to Margaritaville',
@@ -427,6 +535,56 @@ class CommentCommands(CommandTestCase):
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test05_comment_pin(self):
# wherein a bee makes a sick burn on moth's channel and moth pins it
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
moth_stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
moth_claim_id = moth_stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=moth_claim_id,
channel_id=bee['claim_id']
)
self.assertFalse(comment1['is_pinned'])
comment2 = await self.daemon.jsonrpc_comment_create(
comment='sick burn, brah',
claim_id=moth_claim_id,
channel_id=moth_id,
)
self.assertFalse(comment2['is_pinned'])
comments = await self.daemon.jsonrpc_comment_list(moth_claim_id)
comments_items = comments['items']
self.assertIn('is_pinned', comments_items[0])
self.assertFalse(comments_items[0]['is_pinned'])
pinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name']
)
self.assertTrue(pinned['is_pinned'])
comments_after_pinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_pin = comments_after_pinning['items']
self.assertTrue(items_after_pin[1]['is_pinned'])
unpinned = await self.daemon.jsonrpc_comment_pin(
comment_id=comment1['comment_id'],
channel_id=moth['claim_id'],
channel_name=moth['name'],
remove=True
)
self.assertFalse(unpinned['is_pinned'])
comments_after_unpinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
items_after_unpin = comments_after_unpinning['items']
self.assertFalse(items_after_unpin[1]['is_pinned'])
async def test06_comment_list(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
@@ -450,7 +608,6 @@ class CommentCommands(CommandTestCase):
channel_id=bee['claim_id']
)
all_comments = [other_comment, owner_comment, hidden_comment]
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
@@ -532,3 +689,124 @@ class CommentCommands(CommandTestCase):
comment='If you see it and you mean then you know you have to go',
comment_id=original_cid
)
async def test06_reactions(self):
# wherein a bee insulted by a rude moth accidentally
# 1) likes an insult
# 2) clicks dislike instead, testing "clear_types"
# 3) then clicks dislike again, testing "remove"
# calls from the point of view of the moth
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
claim_id = stream['outputs'][0]['claim_id']
first_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth['claim_id'],
)
second_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
first_comment_id = first_comment['comment_id']
second_comment_id = second_comment['comment_id']
comment_list = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertEqual(
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
set(comment_list)
)
self.assertEqual(comment_list['total_items'], 2)
bee_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment_id,
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='like',
)
moth_like_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_type='like',
)
reactions = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1}}, 'others_reactions': {'0': {'like': 1}}}
self.assertEqual(reactions['my_reactions'][first_comment_id]['like'], 1)
self.assertEqual(reactions['others_reactions'][first_comment_id]['like'], 1)
bee_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment_id,
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='dislike',
clear_types='like',
)
reactions_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
# {'my_reactions': {'0': {'like': 1, 'dislike', 0}}, 'others_reactions': {'0': {'like': 0, 'dislike': 1}}}
self.assertEqual(reactions_after_bee_dislikes['my_reactions'][first_comment_id]['like'], 1)
self.assertEqual(reactions_after_bee_dislikes['my_reactions'][first_comment_id]['dislike'], 0)
self.assertEqual(reactions_after_bee_dislikes['others_reactions'][first_comment_id]['dislike'], 1)
self.assertEqual(reactions_after_bee_dislikes['others_reactions'][first_comment_id]['like'], 0)
only_likes_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
react_types='like',
)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions'][first_comment_id]['like'], 1)
self.assertEqual(only_likes_after_bee_dislikes['my_reactions'][first_comment_id]['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions'][first_comment_id]['dislike'], 0)
self.assertEqual(only_likes_after_bee_dislikes['others_reactions'][first_comment_id]['like'], 0)
bee_un_dislike_reaction = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment_id,
channel_id=bee['claim_id'],
channel_name=bee['name'],
remove=True,
react_type='dislike',
)
reactions_after_bee_absconds = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_bee_absconds['my_reactions'][first_comment_id]['like'], 1)
self.assertNotIn('dislike', reactions_after_bee_absconds['my_reactions'][first_comment_id])
self.assertEqual(reactions_after_bee_absconds['others_reactions'][first_comment_id]['like'], 0)
self.assertNotIn('dislike', reactions_after_bee_absconds['others_reactions'][first_comment_id])
bee_reacts_to_both_comments = await self.daemon.jsonrpc_comment_react(
comment_ids=first_comment_id + ',' + second_comment_id,
channel_id=bee['claim_id'],
channel_name=bee['name'],
react_type='frozen_tom',
)
reactions_after_double_frozen_tom = await self.daemon.jsonrpc_comment_react_list(
comment_ids=first_comment_id + ',' + second_comment_id,
channel_id=moth['claim_id'],
channel_name=moth['name'],
)
self.assertEqual(reactions_after_double_frozen_tom['my_reactions'][first_comment_id]['like'], 1)
self.assertNotIn('dislike', reactions_after_double_frozen_tom['my_reactions'][first_comment_id])
self.assertEqual(reactions_after_double_frozen_tom['others_reactions'][first_comment_id]['frozen_tom'], 1)
self.assertEqual(reactions_after_double_frozen_tom['others_reactions'][second_comment_id]['frozen_tom'], 1)
@@ -4,9 +4,7 @@ from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateM
class TestExchangeRateManager(AsyncioTestCase):
async def test_exchange_rate_manager(self):
# TODO: re-enable cryptonator.com
manager = ExchangeRateManager(FEEDS)
manager.start()
self.addCleanup(manager.stop)
@@ -17,6 +15,10 @@ class TestExchangeRateManager(AsyncioTestCase):
for feed in manager.market_feeds:
self.assertTrue(feed.is_online)
self.assertIsInstance(feed.rate, ExchangeRate)
lbc = manager.convert_currency('USD', 'LBC', Decimal('0.01'))
self.assertGreaterEqual(lbc, 0.1)
# print(f'{feed.name} - {feed.market} - {feed.rate.spot}')
lbc = manager.convert_currency('USD', 'LBC', Decimal('1.0'))
self.assertGreaterEqual(lbc, 2.0)
self.assertLessEqual(lbc, 10.0)
lbc = manager.convert_currency('BTC', 'LBC', Decimal('0.01'))
self.assertGreaterEqual(lbc, 1_000)
self.assertLessEqual(lbc, 4_000)
@@ -36,12 +36,14 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
fixed_peers=[])
self.server_config.transaction_cache_size = 10000
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir,
wallet=self.client_wallet_dir, fixed_peers=[])
self.client_config.transaction_cache_size = 10000
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
+5 -5
View File
@@ -37,8 +37,8 @@ class TestSigningComments(AsyncioTestCase):
rswanson = await get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
sign_comment(comment_body, rswanson, sign_comment_id=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, sign_comment_id=True))
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
@@ -53,7 +53,7 @@ class TestSigningComments(AsyncioTestCase):
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
sign_comment(fake_abandon_signal, jeanralphio, sign_comment_id=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, sign_comment_id=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, sign_comment_id=True))
@@ -1,7 +1,9 @@
import binascii
import unittest
from lbry.dht.error import DecodeError
from lbry.dht.serialization.bencoding import _bencode
from lbry.dht.serialization.datagram import RequestDatagram, ResponseDatagram, decode_datagram, ErrorDatagram
from lbry.dht.serialization.datagram import _decode_datagram
from lbry.dht.serialization.datagram import REQUEST_TYPE, RESPONSE_TYPE, ERROR_TYPE
from lbry.dht.serialization.datagram import make_compact_address, decode_compact_address
@@ -139,6 +141,38 @@ class TestDatagram(unittest.TestCase):
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_str_or_int_keys(self):
datagram = decode_datagram(_bencode({
b'0': 0,
b'1': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
b'2': b'111111111111111111111111111111111111111111111111',
b'3': b'ping',
b'4': [{b'protocolVersion': 1}],
b'5': b'should not error'
}))
self.assertEqual(datagram.packet_type, REQUEST_TYPE)
self.assertEqual(b'ping', datagram.method)
def test_mixed_str_or_int_keys(self):
# datagram, _ = _bencode({
# b'0': 0,
# 1: b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
# b'2': b'111111111111111111111111111111111111111111111111',
# 3: b'ping',
# b'4': [{b'protocolVersion': 1}],
# b'5': b'should not error'
# }))
encoded = binascii.unhexlify(b"64313a3069306569316532303a0abcb5269d6cfc1e87a08e920bf39fe9df8e92fc313a3234383a313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131693365343a70696e67313a346c6431353a70726f746f636f6c56657273696f6e6931656565313a3531363a73686f756c64206e6f74206572726f7265")
self.assertDictEqual(
{
'packet_type': 0,
'rpc_id': b'\n\xbc\xb5&\x9dl\xfc\x1e\x87\xa0\x8e\x92\x0b\xf3\x9f\xe9\xdf\x8e\x92\xfc',
'node_id': b'111111111111111111111111111111111111111111111111',
'method': b'ping',
'args': [{b'protocolVersion': 1}]
}, _decode_datagram(encoded)[0]
)
class TestCompactAddress(unittest.TestCase):
def test_encode_decode(self, address='1.2.3.4', port=4444, node_id=b'1' * 48):
+1
View File
@@ -11,6 +11,7 @@ from lbry.extras.daemon.storage import SQLiteStorage
class TestNodePingQueueDiscover(AsyncioTestCase):
TIMEOUT = None # not supported as it advances time
async def test_ping_queue_discover(self):
loop = asyncio.get_event_loop()
loop.set_debug(False)
@@ -1,11 +1,19 @@
import unittest
from aiohttp import ClientSession
from aiohttp.test_utils import make_mocked_request as request
from aiohttp.web import HTTPForbidden
from lbry.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.extras.daemon.security import is_request_allowed as allowed, ensure_request_allowed as ensure
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
class TestAllowedOrigin(unittest.TestCase):
@@ -51,3 +59,38 @@ class TestAllowedOrigin(unittest.TestCase):
ensure(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)
self.assertIn("'hackers.com' are not allowed", log.output[0])
self.assertIn("'allowed_origin' limits requests to: 'localhost'", log.output[0])
class TestAccessHeaders(AsyncioTestCase):
async def asyncSetUp(self):
conf = Config(allowed_origin='localhost')
conf.data_dir = '/tmp'
conf.share_usage_data = False
conf.api = 'localhost:5299'
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)
await self.daemon.start()
self.addCleanup(self.daemon.stop)
async def test_headers(self):
async with ClientSession() as session:
# OPTIONS
async with session.options('http://localhost:5299') as resp:
self.assertEqual(resp.headers['Access-Control-Allow-Origin'], 'localhost')
self.assertEqual(resp.headers['Access-Control-Allow-Methods'], 'localhost')
self.assertEqual(resp.headers['Access-Control-Allow-Headers'], 'localhost')
# GET
status = {'method': 'status', 'params': []}
async with session.get('http://localhost:5299/lbryapi', json=status) as resp:
self.assertEqual(resp.headers['Access-Control-Allow-Origin'], 'localhost')
self.assertEqual(resp.headers['Access-Control-Allow-Methods'], 'localhost')
self.assertEqual(resp.headers['Access-Control-Allow-Headers'], 'localhost')
@@ -4,9 +4,9 @@ from time import time
from lbry.schema.claim import Claim
from lbry.extras.daemon.exchange_rate_manager import (
ExchangeRate, ExchangeRateManager, CurrencyConversionError,
LBRYFeed, LBRYBTCFeed,
CryptonatorFeed, CryptonatorBTCFeed,
BittrexFeed,
CryptonatorUSDFeed, CryptonatorBTCFeed,
BittrexUSDFeed, BittrexBTCFeed,
CoinExBTCFeed
)
from lbry.testcase import AsyncioTestCase, FakeExchangeRateManager, get_fake_exchange_rate_manager
from lbry.error import InvalidExchangeRateResponseError
@@ -26,59 +26,21 @@ class ExchangeRateTests(AsyncioTestCase):
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
manager = get_fake_exchange_rate_manager()
result = manager.convert_currency(fee.currency, "LBC", fee.amount)
self.assertEqual(60.0, result)
self.assertEqual(20.0, result)
def test_missing_feed(self):
fee = Claim().stream.fee
fee.usd = Decimal(1.0)
fee.address = "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
manager = FakeExchangeRateManager([LBRYFeed()], {'BTCLBC': 1.0})
manager = FakeExchangeRateManager([BittrexBTCFeed()], {'BTCLBC': 1.0})
with self.assertRaises(CurrencyConversionError):
manager.convert_currency(fee.currency, "LBC", fee.amount)
def test_lbry_feed_response(self):
feed = LBRYFeed()
out = feed.get_rate_from_response({
'data': {
'fresh': 0, 'lbc_usd': 0.05863062523378918,
'lbc_btc': 5.065289549855739e-05, 'btc_usd': 1157.498
},
'success': True,
'error': None
})
self.assertEqual(1.0 / 5.065289549855739e-05, out)
with self.assertRaises(InvalidExchangeRateResponseError):
feed.get_rate_from_response({})
with self.assertRaises(InvalidExchangeRateResponseError):
feed.get_rate_from_response({
"success": True,
"result": []
})
def test_lbry_btc_feed_response(self):
feed = LBRYBTCFeed()
out = feed.get_rate_from_response({
'data': {
'fresh': 0, 'lbc_usd': 0.05863062523378918,
'lbc_btc': 5.065289549855739e-05, 'btc_usd': 1157.498
},
'success': True,
'error': None
})
self.assertEqual(1.0 / 1157.498, out)
with self.assertRaises(InvalidExchangeRateResponseError):
feed.get_rate_from_response({})
with self.assertRaises(InvalidExchangeRateResponseError):
feed.get_rate_from_response({
"success": True,
"result": {}
})
def test_cryptonator_lbc_feed_response(self):
feed = CryptonatorFeed()
feed = CryptonatorUSDFeed()
out = feed.get_rate_from_response({
'ticker': {
'base': 'BTC', 'target': 'LBC', 'price': 23657.44026496,
'base': 'USD', 'target': 'LBC', 'price': 23657.44026496,
'volume': '', 'change': -5.59806916,
},
'timestamp': 1507470422,
@@ -115,22 +77,14 @@ class ExchangeRateTests(AsyncioTestCase):
})
def test_bittrex_feed_response(self):
feed = BittrexFeed()
feed = BittrexBTCFeed()
out = feed.get_rate_from_response({
"success": True,
"message": "",
"result": [
{
'Id': 6902471, 'TimeStamp': '2017-02-27T23:41:52.213', 'Quantity': 56.12611239,
"Price": 0.00001621, "Total": 0.00090980, "FillType": "PARTIAL_FILL", "OrderType": "SELL"
},
{
"Id": 6902403, "TimeStamp": "2017-02-27t23:31:40.463", "Quantity": 430.99988180,
"Price": 0.00001592, "Total": 0.00686151, "FillType": "PARTIAL_FILL", "OrderType": "SELL"
}
]
"symbol": "LBC-BTC",
"lastTradeRate": "0.00000323",
"bidRate": "0.00000322",
"askRate": "0.00000327"
})
self.assertEqual(1.0 / ((0.00090980+0.00686151) / (56.12611239+430.99988180)), out)
self.assertEqual(1.0 / 0.00000323, out)
with self.assertRaises(InvalidExchangeRateResponseError):
feed.get_rate_from_response({})
with self.assertRaises(InvalidExchangeRateResponseError):
@@ -140,7 +94,7 @@ class ExchangeRateTests(AsyncioTestCase):
})
class BadMarketFeed(LBRYFeed):
class BadMarketFeed(BittrexUSDFeed):
def get_response(self):
raise InvalidExchangeRateResponseError(self.name, 'bad stuff')
@@ -153,3 +107,15 @@ class ExchangeRateManagerTests(AsyncioTestCase):
manager.start()
await asyncio.sleep(1)
self.addCleanup(manager.stop)
async def test_median_rate_used(self):
manager = ExchangeRateManager([BittrexBTCFeed, CryptonatorBTCFeed, CoinExBTCFeed])
for feed in manager.market_feeds:
feed.last_check = time()
bittrex, cryptonator, coinex = manager.market_feeds
bittrex.rate = ExchangeRate(bittrex.market, 1.0, time())
cryptonator.rate = ExchangeRate(cryptonator.market, 2.0, time())
coinex.rate = ExchangeRate(coinex.market, 3.0, time())
self.assertEqual(14.0, manager.convert_currency("BTC", "LBC", Decimal(7.0)))
cryptonator.rate.spot = 4.0
self.assertEqual(21.0, manager.convert_currency("BTC", "LBC", Decimal(7.0)))
+16 -15
View File
@@ -9,14 +9,15 @@ claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
class TestURLParsing(unittest.TestCase):
segments = 'stream', 'channel'
fields = 'name', 'claim_id', 'sequence', 'amount_order'
fields = 'name', 'claim_id', 'amount_order'
def _assert_url(self, url_string, **kwargs):
def _assert_url(self, url_string, strictly=True, **kwargs):
url = URL.parse(url_string)
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
if strictly:
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
present = {}
for key in kwargs:
for segment_name in self.segments:
@@ -42,19 +43,22 @@ class TestURLParsing(unittest.TestCase):
url = self._assert_url
# stream
url('test', stream_name='test')
url('test:1', stream_name='test', stream_sequence='1')
url('test*1', stream_name='test*1')
url('test$1', stream_name='test', stream_amount_order='1')
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id)
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id, strictly=False)
url(f'test:{claim_id}', stream_name='test', stream_claim_id=claim_id)
# channel
url('@test', channel_name='@test')
url('@test:1', channel_name='@test', channel_sequence='1')
url('@test$1', channel_name='@test', channel_amount_order='1')
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id)
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id, strictly=False)
url(f'@test:{claim_id}', channel_name='@test', channel_claim_id=claim_id)
# channel/stream
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff', strictly=False)
url(f'lbry://@test:{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# combined legacy and new
url('@test:1/stuff#2', channel_claim_id='1', stream_claim_id='2', channel_name='@test', stream_name='stuff', strictly=False)
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\uD799')
@@ -104,10 +108,8 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://test@")
fail("lbry://tes@t")
fail(f"lbry://test:1#{claim_id}")
fail("lbry://test:0")
fail("lbry://test$0")
fail("lbry://test/path")
fail("lbry://@test1:1ab/fakepath")
fail("lbry://test:1:1:1")
fail("whatever/lbry://test")
fail("lbry://lbry://test")
@@ -115,5 +117,4 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://abc:0x123")
fail("lbry://abc:0x123/page")
fail("lbry://@test1#ABCDEF/fakepath")
fail("test:0001")
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
+8 -4
View File
@@ -40,7 +40,7 @@ class TestReflector(AsyncioTestCase):
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream_manager.config.reflect_streams = False
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self, response_chunk_size):
@@ -95,6 +95,7 @@ class TestReflector(AsyncioTestCase):
# this used to raise (and then propagate) a CancelledError
self.assertListEqual(await reflect_task, [])
self.assertFalse(self.stream.is_fully_reflected)
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
async def test_result_from_disconnect_after_sd_transfer(self):
stop = asyncio.Event()
@@ -112,7 +113,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -134,7 +136,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await not_incoming.wait()
stop.set()
self.assertListEqual(await reflect_task, [self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash])
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash], sent)
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
self.assertFalse(self.stream.is_fully_reflected)
@@ -181,7 +184,8 @@ class TestReflector(AsyncioTestCase):
await incoming.wait()
await self.stream_manager.delete(self.stream, delete_file=True)
# this used to raise OSError when it can't read the deleted blob for the upload
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
sent = await reflect_task
self.assertListEqual([self.stream.sd_hash], sent)
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
self.assertFalse(
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
+3 -1
View File
@@ -98,12 +98,14 @@ async def get_mock_wallet(sd_hash, storage, wallet_dir, balance=10.0, fee=None):
wallet = Wallet()
ledger = Ledger({
'db': Database(os.path.join(wallet_dir, 'blockchain.db')),
'headers': FakeHeaders(514082)
'headers': FakeHeaders(514082),
'tx_cache_size': 10000
})
await ledger.db.open()
wallet.generate_account(ledger)
manager = WalletManager()
manager.config = Config()
manager.config.transaction_cache_size = 10000
manager.wallets.append(wallet)
manager.ledgers[Ledger] = ledger
manager.ledger.network.client = ClientSession(
@@ -18,8 +18,7 @@ from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs, Censor
from lbry.wallet import Ledger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
from .full_text_search import FTS_ORDER_BY
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
class SQLiteOperationalError(apsw.Error):
@@ -39,7 +38,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
'tx_position', 'channel_join', 'reposted',
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
@@ -53,7 +52,7 @@ SEARCH_PARAMS = {
'any_locations', 'all_locations', 'not_locations',
'any_languages', 'all_languages', 'not_languages',
'is_controlling', 'limit', 'offset', 'order_by',
'no_totals',
'no_totals', 'has_source'
} | INTEGER_PARAMS
@@ -94,10 +93,10 @@ class ReaderState:
self.db.setprogresshandler(interruptor, 100)
def get_resolve_censor(self) -> Censor:
return Censor(self.blocked_streams, self.blocked_channels)
return Censor(Censor.RESOLVE)
def get_search_censor(self) -> Censor:
return Censor(self.filtered_streams, self.filtered_channels)
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
return Censor(Censor.SEARCH)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
@@ -283,7 +282,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = {
unhexlify(cid)[::-1] for cid in channel_ids
unhexlify(cid)[::-1] for cid in channel_ids if cid
}
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
@@ -342,12 +341,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
if 'text' in constraints:
constraints["search"] = constraints.pop("text")
constraints["order_by"] = FTS_ORDER_BY
select = f"SELECT {cols} FROM search JOIN claim ON (search.rowid=claim.rowid)"
else:
select = f"SELECT {cols} FROM claim"
select = f"SELECT {cols} FROM claim"
if not for_count:
select += " LEFT JOIN claimtrie USING (claim_hash)"
return query(select, **constraints)
@@ -372,7 +366,7 @@ def count_claims(**constraints) -> int:
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = select_claims(Censor(), 'count(*) as row_count', for_count=True, **constraints)
count = select_claims(Censor(Censor.SEARCH), 'count(*) as row_count', for_count=True, **constraints)
return count[0]['row_count']
@@ -421,12 +415,13 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
total = None
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
if not constraints.pop('no_totals', False):
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
context = ctx.get()
search_censor = context.get_search_censor()
search_censor = context.get_search_censor(limit_claims_per_channel)
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -535,6 +530,18 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
AND tag IN ({values})
)
"""
elif attr == 'language':
indexed_languages = any_items & set(INDEXED_LANGUAGES)
if indexed_languages:
any_items -= indexed_languages
for language in indexed_languages:
any_queries[f'#_any_common_languages_{language}'] = f"""
EXISTS(
SELECT 1 FROM language INDEXED BY language_{language}_idx
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
AND language = '{language}'
)
"""
if any_items:
+7 -2
View File
@@ -8,12 +8,16 @@ from typing import List, Tuple
from lbry.wallet.constants import COIN, NULL_HASH32
from lbry.schema.claim import Claim
from lbry.schema.result import Censor
from lbry.wallet.server.db import reader, writer
from lbry.wallet.server.db import writer
from lbry.wallet.server.coin import LBCRegTest
from lbry.wallet.server.db.trending import zscore
from lbry.wallet.server.db.canonical import FindShortestID
from lbry.wallet.server.block_processor import Timer
from lbry.wallet.transaction import Transaction, Input, Output
try:
import reader
except:
from . import reader
def get_output(amount=COIN, pubkey_hash=NULL_HASH32):
@@ -31,7 +35,7 @@ def get_tx():
def search(**constraints) -> List:
return reader.search_claims(Censor(), **constraints)
return reader.search_claims(Censor(Censor.SEARCH), **constraints)
def censored_search(**constraints) -> Tuple[List, Censor]:
@@ -553,6 +557,7 @@ class TestTrending(TestSQLDB):
self.advance(zscore.TRENDING_WINDOW * 2, [self.get_support(problematic, 500000000)])
@unittest.skip("filtering/blocking is applied during ES sync, this needs to be ported to integration test")
class TestContentBlocking(TestSQLDB):
def test_blocking_and_filtering(self):
+2 -4
View File
@@ -40,7 +40,7 @@ class MockNetwork:
merkle = await self.get_merkle(tx_hash, known_height)
return tx, merkle
async def get_transaction_batch(self, txids):
async def get_transaction_batch(self, txids, restricted):
return {
txid: await self.get_transaction_and_merkle(txid)
for txid in txids
@@ -126,9 +126,7 @@ class TestSynchronization(LedgerTestCase):
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
self.assertEqual(0, len(self.ledger._tx_cache))
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])
+42
View File
@@ -74,6 +74,48 @@ class TestWalletCreation(AsyncioTestCase):
decrypted = Wallet.unpack('password', encrypted)
self.assertEqual(decrypted['accounts'][0]['name'], 'An Account')
def test_no_password_but_encryption_preferred(self):
wallet_dict = {
'version': 1,
'name': 'Main Wallet',
'preferences': {
"encrypt-on-disk": {
"ts": 1571762543.351794,
"value": True
},
},
'accounts': [
{
'certificates': {},
'name': 'An Account',
'ledger': 'lbc_mainnet',
'modified_on': 123,
'seed':
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
"h absent",
'encrypted': False,
'private_key':
'xprv9s21ZrQH143K42ovpZygnjfHdAqSd9jo7zceDfPRogM7bkkoNVv7'
'DRNLEoB8HoirMgH969NrgL8jNzLEegqFzPRWM37GXd4uE8uuRkx4LAe',
'public_key':
'xpub661MyMwAqRbcGWtPvbWh9sc2BCfw2cTeVDYF23o3N1t6UZ5wv3EMm'
'Dgp66FxHuDtWdft3B5eL5xQtyzAtkdmhhC95gjRjLzSTdkho95asu9',
'address_generator': {
'name': 'deterministic-chain',
'receiving': {'gap': 17, 'maximum_uses_per_address': 3},
'change': {'gap': 10, 'maximum_uses_per_address': 3}
}
}
]
}
storage = WalletStorage(default=wallet_dict)
wallet = Wallet.from_storage(storage, self.manager)
self.assertEqual(
hexlify(wallet.hash), b'8cc6341885e6ad46f72a17364c65f8441f09e79996c55202196b399c75f8d751'
)
self.assertFalse(wallet.is_encrypted)
def test_read_write(self):
manager = WalletManager()
config = {'data_path': '/tmp/wallet'}
Loaded 100 of 101 files, more files were not shown because too many files have changed in this diff. Show more