Compare commits

...
315 Commits
Author SHA1 Message Date
Lex Berezhny 48502961cf v0.75.0 2020-06-03 14:44:02 -04:00
Lex Berezhny 3c8bec61d3 Merge pull request #2966 from lbryio/check-origin
add `allowed_origin` to config, by default no longer allow any requests which pass Origin in header
2020-06-03 14:39:57 -04:00
Lex Berezhny 7296c7df1a Origin: null no longer allowed 2020-06-03 14:19:16 -04:00
Lex Berezhny f3ee6603de improve allowed_origin request handling 2020-06-03 13:55:20 -04:00
Lex Berezhny ee0aabda1d backwards compatible allowed_origin, default browsers not allowed 2020-06-03 13:28:32 -04:00
Jack Robison 08d37a4b0f add allowed_origin to config
-raise 403 error if a request doesn't have a matching origin
2020-06-03 12:55:24 -04:00
Jack Robison f975ea99cb Merge branch 'endes123321-master' 2020-06-02 15:59:53 -04:00
Jack Robison f030d41dc7 add test_is_valid_ipv4 2020-06-02 15:56:57 -04:00
Jack Robison 8d079bfcd1 Merge branch 'master' into master 2020-06-02 15:52:04 -04:00
Lex Berezhny d39b8654a6 Merge pull request #2968 from lbryio/transcode-adjust
more aggressive video transcoding to reduce file sizes
2020-06-01 13:56:33 -04:00
Thomas Zarebczan ce7816a968 more aggressive video transdoing
Have noticed the defaults aren't aggressive enough to stream smoothly (yet). Downgrade max rate to 5500K, higher crf = smaller file size for now.
2020-06-01 13:11:56 -04:00
Jack Robison 2ee572e68f Merge pull request #2967 from lbryio/transaction-cache-size
add `transaction_cache_size` to config
2020-05-27 00:57:48 -04:00
Jack Robison 4bbd850898 fix uncaught ValueError in hashX_unsubscribe 2020-05-25 10:25:04 -04:00
Jack Robison 34eae6e608 fix wallet server prometheus bucket sizes 2020-05-25 10:24:31 -04:00
Jack Robison 6a0302fec6 fix uncaught dht DecodeError 2020-05-25 10:23:11 -04:00
Jack Robison c94cc293c2 fix uncaught errors in test_component_manager 2020-05-25 10:21:36 -04:00
Jack Robison cae7792a1e add transaction_cache_size to config 2020-05-25 10:16:18 -04:00
Lex Berezhny 7f6b2fe4f1 v0.74.0 2020-05-18 16:56:58 -04:00
Lex Berezhny eba430bbc0 Merge pull request #2957 from lbryio/file-list-download-path
added `download_path` as a filter for `file_list` command
2020-05-18 14:15:44 -04:00
Akinwale Ariwodola 01280c8d04 update docstring for download_path 2020-05-18 18:52:13 +01:00
Lex Berezhny ec141ac9c9 Merge pull request #2960 from lbryio/leaky_ship
fixed bug with leaky information between outputs
2020-05-18 13:21:08 -04:00
Lex Berezhny 590c892a6a re-set channel on txo 2020-05-18 12:27:22 -04:00
Lex Berezhny ff8a50c366 fixed bug with leaky information between outputs 2020-05-18 11:16:01 -04:00
Lex Berezhny 46ef6c8ab7 Merge pull request #2947 from thebubbleindex/patch-1
properly handle integer environment values, fixes issue with specifying ports via environment vars
2020-05-18 11:11:10 -04:00
Lex Berezhny b09eabc478 minor simplifcation 2020-05-18 08:53:34 -04:00
thebubbleindex e49fcea6e3 fix issue with specifying ports via env vars
make sure tcp and udp port for dht are int type
2020-05-18 08:53:07 -04:00
Akinwale Ariwodola 68ed9f4ffc add download_path property to managed_stream 2020-05-17 12:12:31 +01:00
Akinwale Ariwodola af94687d45 add download_path as a filter field for file_list 2020-05-17 10:32:26 +01:00
Jack Robison 26964ecf0f fix download_blob_from_peer.py 2020-05-13 09:24:35 -04:00
Jack Robison 77d19af359 v0.73.1 2020-05-13 09:21:48 -04:00
Jack Robison 120b82f243 Merge pull request #2952 from lbryio/newwait
fix case where publish hangs until a new block
2020-05-12 23:10:03 -04:00
Victor Shyba a0fea30a11 make wait check every second instead of once 2020-05-12 19:44:02 -04:00
Jack Robison 8bb3e0a64d Merge pull request #2955 from lbryio/fix-set-node
fix node not being set on the downloader in some cases
2020-05-12 18:49:03 -04:00
Jack Robison bbded12923 fix node not being set on the downloader in some cases 2020-05-12 17:50:20 -04:00
Lex Berezhny e8ba5d7606 v0.73.0 2020-05-11 19:47:08 -04:00
Jack Robison af66b31a44 Merge pull request #2954 from lbryio/file-list-reflector-args
add `uploading_to_reflector`,  `is_fully_reflected`, `completed`, and `status` filter arguments to `file_list`, update to match all filters rather than any filters
2020-05-11 19:45:51 -04:00
Jack Robison b000a40f28 add completed filter arg to file_list 2020-05-11 19:22:53 -04:00
Jack Robison 3c85322523 add status arg to file_list cli 2020-05-11 19:16:08 -04:00
Jack Robison a469b8bc04 return streams matching all file_list filters rather than those matching any
-fix filter fields when using sets
2020-05-11 19:11:41 -04:00
Jack Robison 78b8261a3a cancel pending reflector request when connection is lost
-add 180s timeout
2020-05-11 16:08:48 -04:00
Jack Robison f20ca70c01 add uploading_to_reflector and is_fully_reflected filter arguments to file_list 2020-05-11 15:48:34 -04:00
Jack Robison 4e4148fc1c Merge pull request #2953 from lbryio/ecdsa-executor
fix channel key generation blocking globally
2020-05-11 15:16:17 -04:00
Jack Robison c22482f907 channel private key generation in a thread pool 2020-05-11 14:54:31 -04:00
Jack Robison 870e139fce Merge pull request #2951 from lbryio/add-fixed-peers-conf
add `fixed_peers` setting to config
2020-05-11 14:54:01 -04:00
Jack Robison 4d58648c02 update default fixed peer to cdn.reflector.lbry.com 2020-05-11 14:52:31 -04:00
Jack Robison ebbb182537 fix test 2020-05-11 14:06:23 -04:00
Jack Robison 1cd5377b45 split fixed peer setting out from reflector_servers 2020-05-11 13:43:13 -04:00
Lex Berezhny 1d1f0527ee Merge pull request #2948 from lpessin/patch-1
fix duplicate line on api doc
2020-05-07 17:59:57 -04:00
Jack Robison 37a5f77415 Merge pull request #2930 from lbryio/source_manager_ 2020-05-07 16:10:01 -04:00
Luiz ced3c7efe4 fix duplicate line on api doc
delete line 3616 (duplicate line 3618)
2020-05-07 13:37:46 -03:00
Victor Shyba c3b8f366ed fixes from review 2020-05-07 04:04:55 -03:00
Victor Shyba de78876b1a fix test purchase 2020-05-07 04:04:55 -03:00
Victor Shyba 64c25b049c fixup get_filtered from rebase 2020-05-07 04:04:55 -03:00
Victor Shyba 8a4fe4f3ad lbry-libtorrent is now on pypi 2020-05-07 04:04:55 -03:00
Victor Shyba 8811b8c1fd use ubuntu 18 on gitlab, temporarly 2020-05-07 04:04:55 -03:00
Victor Shyba 190b01fdf9 calculate total bytes outside of dict 2020-05-07 04:04:55 -03:00
Victor Shyba f145d08c10 tell progress, stop trying to read first piece 2020-05-07 04:04:55 -03:00
Victor Shyba 53382b7e15 wait started event 2020-05-07 04:04:55 -03:00
Victor Shyba 6ad0242617 find and show largest file 2020-05-07 04:04:55 -03:00
Victor Shyba a7c2408c0a fix and test delete with torrents 2020-05-07 04:04:55 -03:00
Victor Shyba ce1eabaed6 fix moving to a new btih 2020-05-07 04:04:55 -03:00
Victor Shyba f602541ede fix not knowing a torrent exists 2020-05-07 04:04:55 -03:00
Victor Shyba 3f718e6efc gitlab: use ubuntu on datanetwork tests 2020-05-07 04:04:54 -03:00
Victor Shyba ce7a985df6 add boost on gitlab, fix failing test, add libtorrent to linux build 2020-05-07 04:04:54 -03:00
Victor Shyba 6d83f7e7bd correct wheel with boost 1.65 2020-05-07 04:04:54 -03:00
Victor Shyba b73c00943c linting and minor refactor 2020-05-07 04:04:54 -03:00
Victor Shyba abaac8ef48 fixes from rebase, install libtorrent from s3 2020-05-07 04:04:54 -03:00
Victor Shyba a2f8e7068e pylint 2020-05-07 04:04:54 -03:00
Victor Shyba 4d47873219 working file list after torrent get 2020-05-07 04:04:54 -03:00
Victor Shyba cf985486e5 torrent test and misc fixes 2020-05-07 04:04:54 -03:00
Victor Shyba b930c3fc93 fix torrent and stream manager reference leftovers 2020-05-07 04:04:54 -03:00
Victor Shyba dd26a96828 adds more torrent parts 2020-05-07 04:04:54 -03:00
Victor Shyba 6865ddfc12 torrent manager and torrent source 2020-05-07 04:04:54 -03:00
Victor Shyba e888e69d4d fix unit tests 2020-05-07 04:04:54 -03:00
Victor Shyba 2089059792 pylint 2020-05-07 04:04:54 -03:00
Victor Shyba 27739e0364 fix save from resolve 2020-05-07 04:04:54 -03:00
Victor Shyba 698ee271d6 stream manager component becomes file manager component 2020-05-07 04:04:54 -03:00
Victor Shyba 543c75b293 wip 2020-05-07 04:04:54 -03:00
Victor Shyba b09c46f6f7 add torrent component 2020-05-07 04:04:54 -03:00
Jack Robison f2cc19e6aa add lbry.torrent 2020-05-07 04:04:54 -03:00
Jack Robison 814a0a123f file manager refactor 2020-05-07 04:04:54 -03:00
Jack Robison 179383540f ManagedDownloadSource and SourceManager refactor 2020-05-07 04:04:54 -03:00
Lex Berezhny 8be1c8310d v0.72.0 2020-05-04 13:53:13 -04:00
Jack Robison ef02d776ca Merge pull request #2937 from lbryio/daemon-prometheus
fix database lockup and add prometheus db metrics
2020-05-04 13:51:12 -04:00
Jack Robison 750ff448ad comments 2020-05-04 13:47:37 -04:00
Jack Robison e3abab6d4d pylint 2020-05-04 12:15:48 -04:00
Jack Robison d3ffae72fb buckets 2020-05-02 22:30:25 -04:00
Jack Robison 87f751188e cancelled and failed api request metrics 2020-05-02 21:58:41 -04:00
Jack Robison 3469abaefd write lock metrics 2020-05-02 21:23:17 -04:00
Jack Robison 797364ee5c refactor prometheus metrics 2020-05-02 15:01:07 -04:00
Jack Robison 36c05fc4b9 move wallet server prometheus
-only run wallet server metrics for the wallet server
2020-05-01 12:54:35 -04:00
Jack Robison 79624febc0 prevent pileup of writes blocking reads 2020-05-01 12:48:41 -04:00
Jack Robison 0a9d4de126 include write lock in try/finally 2020-05-01 11:40:57 -04:00
Jack Robison b6f6994db4 Merge pull request #2934 from lbryio/fix-disconnect-on-cancelled-request
don't close the wallet server connection upon a cancelled request
2020-04-30 12:22:46 -04:00
Jack Robison ff7bed720a don't close the connection upon a cancelled request 2020-04-29 12:32:31 -04:00
Lex Berezhny 00c0f48b02 v0.71.0 2020-04-27 12:21:04 -04:00
Lex Berezhny 94c45cf2a0 Merge pull request #2922 from lbryio/estimate_on_demand
estimate block timestamps on client only when necessary (header hasn't been downloaded yet)
2020-04-27 12:17:54 -04:00
Victor Shyba 58f77b2a1c load/dump header file using executor 2020-04-27 10:34:49 -04:00
Victor Shyba 7170e69b22 test fixes + leave tx plot always on estimations 2020-04-27 10:34:49 -04:00
Victor Shyba 239ee2437c estimate only whats not downloaded 2020-04-27 10:34:49 -04:00
Victor Shyba ced368db31 hold headers file in memory during runtime 2020-04-27 10:34:49 -04:00
Lex Berezhny 6a991e5c15 Merge pull request #2914 from StripedMonkey/SM-Config-patch
use home directory if xdg directory not found (Linux)
2020-04-27 10:33:57 -04:00
Lex Berezhny 523d22262b pin pylint to 2.4.4 2020-04-27 10:18:39 -04:00
Noah 08197a327e fix Missing xdg download location
Fixes  an error in detection xdg config locations when XDG_DOWNLOAD_DIR is not present in `user-dirs.dirs`
2020-04-27 10:11:16 -04:00
Jack Robison b7cb2a7aa5 Merge pull request #2926 from lbryio/blocking-file-io
make file i/o in stream and blob creation and reads non-blocking
2020-04-24 11:07:24 -04:00
Jack Robison decc5c74ef don't block when reading a file when creating a stream 2020-04-23 00:35:08 -04:00
Jack Robison fbe0f886b6 non blocking blob creation 2020-04-23 00:35:08 -04:00
Jack Robison 7e23d6e2ef Merge pull request #2925 from lbryio/fix-reposts-tag
fix: reposts being returned for single tags
2020-04-23 00:29:04 -04:00
Tom 49458d1085 fix: reposts being returned for single tags 2020-04-22 23:16:30 -04:00
Lex Berezhny 289c12bd9a Merge pull request #2924 from lbryio/claim_list_is_spent_fix
fix for `claim_list` incorrectly handling `--is_spent` flag
2020-04-22 11:34:16 -04:00
Lex Berezhny 9a6326b027 fix for claim_list incorrectly handling --is_spent flag 2020-04-22 10:36:09 -04:00
Lex Berezhny 51f573f1ea v0.70.0 2020-04-20 13:37:40 -04:00
Jack Robison c8b7cd8862 Merge pull request #2920 from lbryio/uploading-to-reflector-status
add `uploading_to_reflector` to `file_list` results
2020-04-20 13:35:02 -04:00
Jack Robison 21c112d059 lbrycrd url 2020-04-20 12:16:31 -04:00
Jack Robison 9432e1b5b2 add uploading_to_reflector to file_list results 2020-04-20 11:57:09 -04:00
endes 8269d2f83c better lint style 2020-04-19 20:20:19 +01:00
endes 5a4b6be974 fix 2020-04-19 20:14:21 +01:00
endes123321 35e8ce60a9 Merge branch 'master' into master 2020-04-19 19:54:24 +01:00
endes 8b6dd9f603 better lint style 2020-04-19 19:51:03 +01:00
Lex Berezhny 084f0ebdab v0.69.1 2020-04-17 12:55:07 -04:00
Lex Berezhny 58640c1521 Merge pull request #2917 from lbryio/faster-repost-search-query
improve how claim search query handles reposts
2020-04-17 12:53:50 -04:00
Jack Robison 7ffdfd12f8 faster not tags 2020-04-17 10:47:01 -04:00
Jack Robison cb9a30f285 faster query 2020-04-17 10:47:01 -04:00
Lex Berezhny e48bef809f Merge pull request #2919 from lbryio/exclude_internal_transfers_fix
fix issue with `--exclude_internal_transfers` where it was filtering out sent payments
2020-04-16 20:26:37 -04:00
Lex Berezhny f5d7570102 fix issue with --exclude_internal_transfers where it was filtering out sent payments 2020-04-16 17:55:49 -04:00
Lex Berezhny a600c60cf8 v0.69.0 2020-04-13 15:36:27 -04:00
Jack Robison 695eabd026 Merge pull request #2911 from lbryio/file_list_claim_ids
support `claim_id`, `channel_claim_id`, and `outpoint` args given as lists of values to match in `file_list`
2020-04-13 14:58:13 -04:00
Jack Robison e81b51a647 support claim_id, channel_claim_id, and outpoint args in file_list being lists 2020-04-13 13:19:25 -04:00
Jack Robison b7e95ff090 Merge pull request #2912 from lbryio/reflector-progress
add `reflector_progress` to `file_list` results
2020-04-13 13:00:59 -04:00
Jack Robison 3ca41be686 add reflector_progress to file_list results 2020-04-13 12:08:22 -04:00
Lex Berezhny 3152046173 Merge pull request #2910 from osilkin98/comment-schema-update
Comment_hide update
2020-04-13 10:34:30 -04:00
Lex Berezhny 3a98dc8a95 run apt-get update before installing 2020-04-13 10:14:52 -04:00
Lex Berezhny d737b28916 trying actions/checkout v2 2020-04-13 09:57:59 -04:00
Oleg Silkin 97c0dac876 linter 2020-04-07 19:28:26 -04:00
Oleg Silkin 006494b1fa hide_comments now returns lists for both hidden and visible comments 2020-04-07 19:17:27 -04:00
Alex Grintsvayg 149d343201 drop a few unused conf vars 2020-04-07 16:11:23 -04:00
Lex Berezhny 496cc79ba8 v0.68.0 2020-04-06 12:55:21 -04:00
Jack Robison 3591768745 Merge pull request #2908 from lbryio/batched-resolve
Fix connection to wallet server breaking upon giant resolve requests
2020-04-06 12:52:51 -04:00
Jack Robison d615f6761a automatically batch large resolve requests 2020-04-06 12:03:33 -04:00
Jack Robison 5b29894048 add reset clients counter to prometheus 2020-04-06 12:03:33 -04:00
Lex Berezhny d17f0cfa40 Merge pull request #2909 from lbryio/none_for_unconfirmed_time
do not estimate timestamp for unconfirmed txs
2020-04-06 11:41:17 -04:00
Victor Shyba b2f70c7120 return none for unconfirmed time estimation 2020-04-06 06:03:35 -03:00
Jack Robison 238707bd93 Merge pull request #2897 from lbryio/reorg-claims
handle claim reorgs in the wallet server
2020-04-05 17:48:56 -04:00
Jack Robison 87cdf1e3a0 improve test_reorg_change_claim_height 2020-04-05 16:58:36 -04:00
Jack Robison 57fd47022e test_reorg_change_claim_height 2020-04-03 13:39:44 -04:00
Jack Robison 5eafd3bf6b feedback 2020-04-03 13:39:44 -04:00
Jack Robison 640b5b0ea9 delete_claims_above_height with thread lock 2020-04-03 13:39:44 -04:00
Jack Robison e4fb2f4680 test_reorg_dropping_claim 2020-04-03 13:39:44 -04:00
Jack Robison a4909f54e4 test reorg count metric 2020-04-03 13:39:44 -04:00
Jack Robison f7065c6f0c add reorg count metric to prometheus 2020-04-03 13:39:44 -04:00
Jack Robison 64f7f837e7 delete claims above reorg height from the database 2020-04-03 13:39:44 -04:00
Jack Robison 130d36acd9 Merge pull request #2905 from lbryio/testnet-headers
fix header checkpoints on testnet
2020-04-03 13:38:47 -04:00
Jack Robison e94c28cfa2 fix header checkpoints on testnet 2020-04-03 12:21:17 -04:00
Jack Robison a697dec30c Merge pull request #2902 from lbryio/bump_checkpoints
bump checkpoints
2020-04-03 12:20:38 -04:00
Victor Shyba bac09e9b9f bump checkpoints 2020-04-03 12:26:52 -03:00
Lex Berezhny b179f8f86a Merge pull request #2901 from lbryio/txo_is_spent_filter
replaced `--unspent` with `--is_not_spent` on several commands and added `--is_spent` to many claim/txo `*_list` commands, `support_list` got three new flags `--sent --received --staked`
2020-04-02 22:24:47 -04:00
Lex Berezhny 962d04ae17 fix txo_spend 2020-04-01 21:03:56 -04:00
Lex Berezhny f28e3bfe37 lint 2020-04-01 20:53:09 -04:00
Lex Berezhny 6474c86d32 cleaned up *_list commands 2020-04-01 20:44:34 -04:00
Lex Berezhny 769ea8cdfe added --is_spent filter to txo list/sum commands 2020-03-31 23:08:51 -04:00
Lex Berezhny 052e77dd5a v0.67.2 2020-03-31 17:47:19 -04:00
Lex Berezhny 1a4b0cb12a Merge pull request #2900 from lbryio/wallet_status_does_not_fail
`wallet_status` no longer fails if wallet component has not started
2020-03-31 17:45:51 -04:00
Lex Berezhny f9aa95c987 default to None for all values in wallet_status when wallet_manager not started yet 2020-03-31 17:22:13 -04:00
Lex Berezhny a5d06fb4a4 wallet_status no longer fails if wallet component has not started 2020-03-31 16:20:13 -04:00
Lex Berezhny d5e5d90bdc v0.67.1 2020-03-31 12:38:38 -04:00
Lex Berezhny e62678e4e6 Merge pull request #2898 from lbryio/duration_is_int
`ffmpeg` file analysis returns duration as integer now
2020-03-31 12:06:44 -04:00
Lex Berezhny 558ac24f7e fix test 2020-03-31 11:29:58 -04:00
Lex Berezhny 6f22f6a59f use ceil() on duration float() instead of int() directly 2020-03-31 10:57:37 -04:00
Lex Berezhny 5ec74f8abe ffmpeg file analysis returns duration as integer now 2020-03-31 10:28:04 -04:00
Lex Berezhny 16d7547e03 changed default txo_spend batch_size default to 500 from 1000 2020-03-30 22:48:05 -04:00
Lex Berezhny 767112dcda updated txo_spend docs 2020-03-30 21:45:58 -04:00
Lex Berezhny 81e23d1d8c v0.67.0 2020-03-30 19:43:45 -04:00
Lex Berezhny ace90a4354 Merge pull request #2890 from lbryio/faster-status
improve speed of `status`
2020-03-30 19:42:57 -04:00
Jack Robison 267e7096cc add test for rechecking ffmpeg installation 2020-03-30 19:11:04 -04:00
Jack Robison e7cded7511 check ffmpeg/ffmprobe paths in a thread 2020-03-30 19:11:04 -04:00
Jack Robison 33fbd715c0 don't block status on connectivity check 2020-03-30 19:11:04 -04:00
Jack Robison 25ba5b867c dont recheck ffmpeg installation in status 2020-03-30 19:11:04 -04:00
Lex Berezhny c8f431447c Merge pull request #2885 from lbryio/repair_tip_on_open
repair headers tip on open and let new headers come in during background fetching
2020-03-30 19:03:09 -04:00
Victor Shyba 1b83a1d09a test and fix verifying from middle 2020-03-30 19:42:27 -03:00
Victor Shyba d2fb7a7151 lock only when fetching, giving a chance for tip updates 2020-03-30 19:42:27 -03:00
Victor Shyba 7ad3447598 repair tip on open 2020-03-30 19:42:27 -03:00
Lex Berezhny 5e5bc8e705 Merge pull request #2887 from lbryio/txo_spend
added `txo_spend` command to support liquidating large number of txos (eg. tips) by batching them across several transactions
2020-03-30 18:41:05 -04:00
Lex Berezhny 886d1e8a19 added --include_full_tx option to txo_list 2020-03-30 18:15:13 -04:00
Lex Berezhny 6de7a035fa added preview/blocking back into doc string 2020-03-30 17:47:38 -04:00
Lex Berezhny 48d2497eb2 added txo_spend command to support liquidating large number of txos (eg. tips) 2020-03-30 17:47:38 -04:00
Lex Berezhny 6494754ab9 Merge pull request #2896 from lbryio/first_on_ready
fix for startup issues when connecting to wallet servers
2020-03-30 17:41:14 -04:00
Lex Berezhny a8153627c6 move on_read.first to earlier 2020-03-30 17:02:08 -04:00
Lex Berezhny ed38966edb add test to verify we listen to on_ready before it actually triggers 2020-03-30 15:26:04 -04:00
Lex Berezhny 85551d1e54 remove no-op test, segwit is always on now 2020-03-30 15:17:25 -04:00
Lex Berezhny ca31363180 listen for on_read.first before it is triggered 2020-03-30 14:53:52 -04:00
Lex Berezhny a8f11eb3c3 Merge pull request #2893 from lbryio/fix_float_timestamp
fix estimated timestamp to return integer instead of float
2020-03-30 10:58:32 -04:00
Lex Berezhny 151805121c increase wallet server payment test timeout 2020-03-29 20:00:23 -04:00
Lex Berezhny 0422d2a021 estimated timestamp should be integer 2020-03-29 19:39:37 -04:00
endes123321 8586762dde fixed is_valid_public_ipv4 2020-03-29 16:12:36 +01:00
Lex Berezhny ffdf70257b Merge pull request #2888 from lbryio/m4a-content-type
add m4a content type
2020-03-27 17:54:45 -04:00
Thomas Zarebczan 6079d60aa1 add m4a content type 2020-03-26 23:08:52 -04:00
Jack Robison d46d012f8c Revert "delete live bittrex test"
This reverts commit f70343bb
2020-03-26 20:31:39 -04:00
Jack Robison baf926b360 v0.66.0 2020-03-26 17:59:02 -04:00
Jack Robison 82339869dc Revert "v0.66.0"
This reverts commit 4ab56ae3
2020-03-26 17:50:44 -04:00
Jack Robison f70343bb63 delete live bittrex test 2020-03-26 17:50:31 -04:00
Jack Robison 4ab56ae3d1 v0.66.0 2020-03-26 16:49:54 -04:00
Jack Robison 39b4031684 Merge pull request #2880 from lbryio/handle_mkv_files
Handle MKV files
2020-03-26 16:45:20 -04:00
Brannon King 71f8965393 re-use ffprobe info in stream_type
avoid duplicate args


handle review comment
2020-03-26 16:20:12 -04:00
Brannon King 66857e72a4 ensure only webm-type MKV files 2020-03-26 16:20:12 -04:00
Jack Robison 15eb5d47eb Merge pull request #2881 from lbryio/wallet_sync_status
add `wallet_syncing` to `status`, which is set to true when processing transaction updates
2020-03-26 15:28:20 -04:00
Jack Robison 86242139da update tests 2020-03-26 13:56:51 -04:00
Victor Shyba 460bdc4148 move wallet_syncing to wallet status is_syncing 2020-03-26 11:43:48 -04:00
Victor Shyba 1052126522 add wallet_syncing status 2020-03-26 11:43:48 -04:00
Jack Robison fdb42ac876 Merge pull request #2852 from lbryio/faster-transaction-sync
More efficient syncing of wallet transactions
2020-03-26 11:43:20 -04:00
Jack Robison 6486f986e8 bump min required server version 2020-03-26 11:41:49 -04:00
Jack Robison bc4075e2ed create_task instead of ensure_future 2020-03-26 11:41:49 -04:00
Jack Robison 11525c7d0d fix caching transaction_show 2020-03-26 11:41:49 -04:00
Jack Robison bb12ae9ce6 fix duplicate get_merkle requests 2020-03-26 11:41:49 -04:00
Jack Robison 66b4ad0c44 improve log 2020-03-26 11:41:49 -04:00
Jack Robison 4bff2d718e clean up test 2020-03-26 11:41:49 -04:00
Jack Robison af7a7b4dc6 fix maybe_migrate_certificates blocking the api while populating channel signing keys 2020-03-26 11:41:49 -04:00
Jack Robison ec20d9a2a8 faster wallet sync with get_transaction_and_merkle 2020-03-26 11:41:49 -04:00
Lex Berezhny b0aee3d335 Merge pull request #2884 from lbryio/txo_plot
added `txo_plot` command to allow plotting txo sums over time
2020-03-26 10:14:02 -04:00
Lex Berezhny 5e2ddbfd86 txo_plot returns lbc instead of dewies 2020-03-26 01:13:09 -04:00
Lex Berezhny f086ebbb8e lint 2020-03-26 00:42:11 -04:00
Lex Berezhny e5bf6a5bfc added txo_plot command to allow plotting txo sums over time 2020-03-26 00:37:13 -04:00
Lex Berezhny 76fa86d54b v0.65.0 2020-03-23 14:56:45 -04:00
Jack Robison 60d89506a5 Merge pull request #2877 from lbryio/on_ready_fixup
wait for tx sync to finish but not header sync
2020-03-23 13:46:47 -04:00
Victor Shyba 2893f1eb9e add other taskgroup for non-start-important tasks 2020-03-23 12:36:05 -03:00
Victor Shyba a08cbf412d on_ready.wait so we start after tx syncs 2020-03-23 12:26:07 -03:00
Lex Berezhny 0bd65356f9 increase timeout in usage payment test 2020-03-23 11:06:29 -04:00
Lex Berezhny 2e2b39455f Merge pull request #2875 from lbryio/1080p
default to 1080P for anything higher than 1080P
2020-03-23 10:44:18 -04:00
Lex Berezhny 1f9fbe34e4 Merge pull request #2863 from lbryio/async_headers
fetch headers on demand
2020-03-23 10:41:31 -04:00
Thomas Zarebczan 506d7ae50e default to 1080P for anything higher than 1080P
Previously would do 1440p
2020-03-23 10:20:13 -04:00
Victor Shyba 342cb00625 less concurrent repeated header checks 2020-03-23 01:19:34 -03:00
Victor Shyba 952fc01efd add script that generates checkpoints 2020-03-23 00:05:36 -03:00
Victor Shyba 3eebe301fe move checkpoints out of folder into file 2020-03-22 23:45:14 -03:00
Victor Shyba 19c0a81c42 fix bad usages of hash and some tests 2020-03-22 23:31:47 -03:00
Victor Shyba 9fc7f9904b fix tests, delete old code 2020-03-22 23:31:47 -03:00
Victor Shyba b04a516063 better locking, stop corrupting headers, fix some tests 2020-03-22 23:31:47 -03:00
Victor Shyba 241e946d91 first experimental version 2020-03-22 23:31:45 -03:00
Victor Shyba e45375dc26 more async parts 2020-03-22 23:30:11 -03:00
Victor Shyba ec8e243323 estimate timestamps instead of using block headers 2020-03-22 23:30:11 -03:00
Victor Shyba af0e9368d4 headers get now async 2020-03-22 23:30:11 -03:00
Lex Berezhny 285483b81a Merge pull request #2872 from lbryio/claim_list_include_received_tips
added `--include_received_tips` to `claim_list` and `txo_list` commands
2020-03-22 18:00:35 -04:00
Lex Berezhny 73a91d5569 added --include_received_tips to claim_list/txo_list 2020-03-22 17:22:15 -04:00
Lex Berezhny e24c78be32 Merge pull request #2871 from lbryio/wallet_txo_data_leak
fix to prevent transaction cache from leaking some information about outputs between unrelated wallets
2020-03-22 13:16:03 -04:00
Lex Berezhny b11184de68 fix purchase_create 2020-03-22 12:23:21 -04:00
Lex Berezhny 61d02fc5d7 fix to prevent transaction cache from leaking some information about outputs between unrelated wallets 2020-03-22 11:45:40 -04:00
Lex Berezhny f0217f6821 Merge pull request #2870 from lbryio/resolve_includes
added new flags to `resolve` (and a few to `claim_search`) commands: `--include_purchase_receipt`, `--include_is_my_output`, `--include_sent_supports`, `--include_sent_tips` and `--include_received_tips`
2020-03-22 02:08:28 -04:00
Lex Berezhny 09644914a6 added --include_is_my_output to claim_search 2020-03-22 01:51:09 -04:00
Lex Berezhny b5c24d6a48 fixing unit tests 2020-03-22 01:29:26 -04:00
Lex Berezhny bdd2ac2c25 renamed some flags 2020-03-22 01:13:26 -04:00
Lex Berezhny 76376f0d33 lint 2020-03-22 00:24:38 -04:00
Lex Berezhny 9749da46ae added flags to resolve: --include_purchase_receipt, --include_is_my_output, --include_my_supports, --include_my_tips 2020-03-21 23:44:57 -04:00
Lex Berezhny 1c05295e89 updated doc 2020-03-21 20:15:21 -04:00
Lex Berezhny 4d0f28215a added release-text-lines: to release script to support multi-line release notes 2020-03-21 20:09:39 -04:00
Lex Berezhny 0bba72bc5a Merge pull request #2862 from lbryio/txo_list_support_sent_outputs
`txo_list` adds many new ownership filters such as `--is_my_input`, `--is_my_output`, etc and some metadata filters such as `--channel_id`, `--reposted_claim_id`, etc and a new command `txo_sum` which takes the same arguments as `txo_list` and produces sum of outputs
2020-03-21 19:48:34 -04:00
Lex Berezhny 87089b8e83 fix support test 2020-03-21 18:48:06 -04:00
Lex Berezhny 6a58148a89 added support for --order_by=none 2020-03-21 18:16:25 -04:00
Lex Berezhny 15091052be added --no_totals to txo_list 2020-03-21 18:06:05 -04:00
Lex Berezhny 5cd7e9a9b8 increment scema version and force specific indexes to be used for get_txos() 2020-03-21 15:08:14 -04:00
Lex Berezhny 7cb530c334 added --channel_id and --order_by to txo_list 2020-03-20 23:19:26 -04:00
Lex Berezhny 5e0324cc91 added --reposted_claim_id to txo_list 2020-03-20 20:22:57 -04:00
Lex Berezhny 6293e227ea added txo_sum command 2020-03-20 19:08:47 -04:00
Lex Berezhny d6d83a5c76 integration test fix 2020-03-20 19:08:47 -04:00
Lex Berezhny 93fc883b90 fixing unit tests 2020-03-20 19:08:47 -04:00
Lex Berezhny dd21803598 working --is_my_input_or_output --is_my_input --is_my_output etc 2020-03-20 19:08:47 -04:00
Lex Berezhny af2f2282c2 txo_list returns txo funded by my account but sent to external address 2020-03-20 19:08:47 -04:00
Jack Robison 869a76c9bb Merge pull request #2851 from lbryio/transaction-info
Add `blockchain.transaction.info` and `blockchain.transaction.get_batch` to the wallet server for more efficient transaction fetching
2020-03-20 10:25:45 -04:00
Jack Robison 44b2964a6a add blockchain.transaction.get_batch 2020-03-19 19:20:30 -04:00
Jack Robison 8fddb57e0a reduce block lru cache sizes 2020-03-19 19:20:30 -04:00
Jack Robison c7a5a0cab0 add blockchain.transaction.info 2020-03-19 19:20:30 -04:00
Jack Robison c13aab3ffc Merge pull request #2823 from lbryio/multiple-db-readers
Use multiple processes for querying the db for api calls
2020-03-19 19:19:53 -04:00
Jack Robison 95a9a76598 windows 2020-03-19 19:19:02 -04:00
Jack Robison 923cfa3d50 another 2020-03-19 19:19:02 -04:00
Jack Robison 2c489168c2 feedback 2020-03-19 19:19:02 -04:00
Jack Robison f3292b4d34 feedback 2020-03-19 19:19:02 -04:00
Jack Robison 9769829b72 fix android executor 2020-03-19 19:19:02 -04:00
Jack Robison 36243d15cc fix peer_list 2020-03-19 19:19:02 -04:00
Jack Robison f4645f570c more read only calls 2020-03-19 19:19:02 -04:00
Jack Robison 61603ccfce write lock 2020-03-19 19:19:02 -04:00
Jack Robison a26cfc639c WAL 2020-03-19 19:19:02 -04:00
Jack Robison 90602931d8 multiple readers for transaction_list 2020-03-19 19:19:02 -04:00
Jack Robison d1b330028c multiple db reader processeses 2020-03-19 19:19:02 -04:00
Jack Robison 7a6b1930bf rename executor -> writer_executor 2020-03-19 19:19:02 -04:00
Lex Berezhny c271361552 Merge pull request #2865 from lbryio/lbrycrd-1744
upgrade lbrycrd to v0.17.4.4
2020-03-19 17:54:26 -04:00
Lex Berezhny 17789bc814 Merge pull request #2866 from lbryio/dont_shlex_windows
fixed issue with shell parsing on Windows
2020-03-19 17:53:47 -04:00
Brannon King d7c16e161a fixed issue with shell parsing on Windows 2020-03-19 13:11:57 -06:00
Lex Berezhny e71c17d7e7 upgrade lbrycrd to v0.17.4.4 2020-03-19 12:54:19 -04:00
Lex Berezhny 17d87eb157 Merge pull request #2861 from lbryio/ffmpeg_fixes
transcode in stream_update, re-enable transcode on Windows
2020-03-19 12:20:47 -04:00
Brannon King bf11bcc084 fixed transcoding tests for non-async execute
status ordering broke path check
2020-03-18 12:43:27 -06:00
Brannon King bd291109df addressing code review comments 2020-03-18 12:00:52 -06:00
Brannon King ac89ba9b8d don't require ProactorEventLoop on Windows
fix linter errors
2020-03-18 12:00:52 -06:00
Brannon King 5ab634e375 support search path for ffmpeg 2020-03-18 12:00:52 -06:00
Brannon King bb1978d976 ffmpeg now invoked via stream update 2020-03-18 12:00:52 -06:00
Lex Berezhny bea94ce8ac Merge pull request #2854 from lbryio/tracemalloc_api
Adds tracemalloc api for memory troubleshooting
2020-03-16 22:15:41 -04:00
Victor Shyba 9561f93594 disable tracemalloc after test finishes so it doesnt slow down everything else 2020-03-16 11:29:16 -03:00
Victor Shyba d44d5c3304 enable/disable instead of set 2020-03-16 06:40:22 -03:00
Victor Shyba ec541e2057 lint and improve tests 2020-03-11 19:56:37 -03:00
Victor Shyba 8169bf6b97 top objects api 2020-03-11 19:56:37 -03:00
Victor Shyba 56c8ad1221 start/stop tracemalloc over api 2020-03-11 19:56:37 -03:00
Lex Berezhny 6814f2e38c Merge pull request #2857 from eggplantbren/master
Trending algorithm with time delay and variable decay rate
2020-03-11 18:34:52 -04:00
Brendon J. Brewer 5f043b9a78 variable decay 2020-03-12 10:33:15 +13:00
Lex Berezhny c9092cd1c7 v0.64.0 2020-03-09 13:32:19 -04:00
Lex Berezhny 86cc65d894 Merge pull request #2848 from lbryio/claim_transaction_list_filtering
added new `txo_list` command with filtering for `--claim_id`, claim `--name` and `--is_received`/`--is_not_received`, also commands `claim_list`/`stream_list`/`channel_list`/`support_list` are based on `txo_list` and thus support most of the new filters
2020-03-08 23:33:38 -04:00
Lex Berezhny 8ef2647fa9 is_received 2020-03-08 23:11:03 -04:00
Lex Berezhny 3ff9e99416 added txo_list command 2020-03-08 16:22:49 -04:00
Lex Berezhny 1731046011 Merge pull request #2846 from lbryio/txo_is_spent
added `is_spent` attribute to transaction outputs
2020-03-06 21:10:30 -05:00
Lex Berezhny c3d96184b6 fix test 2020-03-06 20:35:26 -05:00
Lex Berezhny c255c606a7 added is_spent attribute to transaction outputs 2020-03-06 20:12:38 -05:00
Jack Robison cc69faa1fd v0.63.0 2020-03-06 15:58:34 -05:00
Jack Robison 0f53cd86c8 Merge branch 'debug_find_ffmpeg' 2020-03-06 15:54:24 -05:00
Brannon King 212c8f188d only change LD vars if running from bundle
missed a word


moved is_running_from_bundle
2020-03-06 15:53:09 -05:00
Brannon King dec248adec repair env modified by pyinstaller
see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
2020-03-06 15:53:09 -05:00
Brannon King 98e6a066f4 add unit test and debug output 2020-03-06 15:53:09 -05:00
Jack Robison c90db54a3d Revert "v0.63.0"
This reverts commit 927d1569
2020-03-06 15:51:42 -05:00
Jack Robison 927d156933 v0.63.0 2020-03-06 14:49:53 -05:00
Jack Robison c71f91073f Merge pull request #2845 from lbryio/temp-disable-windows-ffmpeg
fix DHT setup and disable ffmpeg on windows temporarily
2020-03-06 14:20:18 -05:00
Jack Robison ab28387692 remove ProactorEventLoop and disable ffmpeg on windows temporarily 2020-03-06 13:45:34 -05:00
98 changed files with 6760 additions and 1769 deletions

No files matched your search

+7 -5
View File
@@ -6,7 +6,7 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -17,7 +17,7 @@ jobs:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -37,12 +37,14 @@ jobs:
- blockchain
- other
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: sudo apt install -y --no-install-recommends ffmpeg
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
@@ -57,7 +59,7 @@ jobs:
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
+2 -1
View File
@@ -41,7 +41,7 @@ test:datanetwork-integration:
stage: test
script:
- pip install tox-travis
- tox -e datanetwork
- tox -e datanetwork --recreate
test:blockchain-integration:
stage: test
@@ -94,6 +94,7 @@ build:linux:
- apt-get update
- apt-get install -y --no-install-recommends python3.7-dev
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
- pip install lbry-libtorrent
build:mac:
extends: .build
+2 -1
View File
@@ -1,6 +1,7 @@
.PHONY: install tools lint test idea
install:
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
--global-option=fetch \
--global-option=--version --global-option=3.30.1 --global-option=--all \
@@ -8,7 +9,7 @@ install:
pip install -e .
tools:
pip install mypy==0.701
pip install mypy==0.701 pylint==2.4.4
pip install coverage astroid pylint
lint:
+783 -172
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.62.0"
__version__ = "0.75.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+23 -9
View File
@@ -110,7 +110,7 @@ class AbstractBlob:
if reader in self.readers:
self.readers.remove(reader)
def _write_blob(self, blob_bytes: bytes):
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
raise NotImplementedError()
def set_length(self, length) -> None:
@@ -198,11 +198,17 @@ class AbstractBlob:
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
def update_events(_):
self.verified.set()
self.writing.clear()
if self.is_writeable():
self.writing.set()
task = self._write_blob(verified_bytes)
task.add_done_callback(update_events)
if self.blob_completed_callback:
self.blob_completed_callback(self)
task.add_done_callback(lambda _: self.blob_completed_callback(self))
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
async def write():
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
return self.loop.create_task(write())
def delete(self):
if self._verified_bytes:
@@ -319,8 +327,14 @@ class BlobFile(AbstractBlob):
handle.close()
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def _write_blob():
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
async def write_blob():
await self.loop.run_in_executor(None, _write_blob)
return self.loop.create_task(write_blob())
def delete(self):
if os.path.isfile(self.file_path):
+2
View File
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
# wait for the io to finish
await self.blob.verified.wait()
log.info("%s at %fMB/s", msg,
round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0, 2))
+34 -15
View File
@@ -277,14 +277,23 @@ class Strings(ListSetting):
class EnvironmentAccess:
PREFIX = 'LBRY_'
def __init__(self, environ: dict):
self.environ = environ
def __init__(self, config: 'BaseConfig', environ: dict):
self.configuration = config
self.data = {}
if environ:
self.load(environ)
def load(self, environ):
for setting in self.configuration.get_settings():
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
self.data[setting.name] = setting.deserialize(value)
def __contains__(self, item: str):
return f'{self.PREFIX}{item.upper()}' in self.environ
return item in self.data
def __getitem__(self, item: str):
return self.environ[f'{self.PREFIX}{item.upper()}']
return self.data[item]
class ArgumentAccess:
@@ -443,7 +452,7 @@ class BaseConfig:
self.arguments = ArgumentAccess(self, args)
def set_environment(self, environ=None):
self.environment = EnvironmentAccess(environ or os.environ)
self.environment = EnvironmentAccess(self, environ or os.environ)
def set_persisted(self, config_file_path=None):
if config_file_path is None:
@@ -464,15 +473,17 @@ class BaseConfig:
class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
'', previous_names=['ffmpeg_folder'])
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
r'-maxrate 8400K -bufsize 5000K')
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
r'-maxrate 5500K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k',
'aac -b:a 160k')
@@ -575,9 +586,14 @@ class Config(CLIConfig):
)
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
('reflector.lbry.com', 5566)
])
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
('cdn.reflector.lbry.com', 5567)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv11.lbry.com', 50001),
('spv12.lbry.com', 50001),
@@ -600,8 +616,6 @@ class Config(CLIConfig):
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
@@ -611,6 +625,9 @@ class Config(CLIConfig):
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
allowed_origin = String(
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
"all hosts; default is to only allow API requests with no `Origin` value.", "")
# media server
streaming_server = String('Host name and port to serve streaming media over range requests',
@@ -622,6 +639,7 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
transaction_cache_size = Integer("Transaction cache size", 100_000)
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
@@ -691,9 +709,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
if down_dir:
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
download_dir = re.sub('\"', '', down_dir)
except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
+8 -6
View File
@@ -33,12 +33,14 @@ def is_valid_public_ipv4(address, allow_localhost: bool = False):
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
parsed_ip.is_reserved,
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except ipaddress.AddressValueError:
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
+2 -1
View File
@@ -10,6 +10,7 @@ from asyncio.protocols import DatagramProtocol
from asyncio.transports import DatagramTransport
from lbry.dht import constants
from lbry.dht.serialization.bencoding import DecodeError
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
from lbry.dht.error import RemoteException, TransportNotConnected
@@ -554,7 +555,7 @@ class KademliaProtocol(DatagramProtocol):
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
try:
message = decode_datagram(datagram)
except (ValueError, TypeError):
except (ValueError, TypeError, DecodeError):
self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
-4
View File
@@ -3,7 +3,6 @@ import sys
import shutil
import signal
import pathlib
import platform
import json
import asyncio
import argparse
@@ -262,9 +261,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop()
if args.verbose is not None:
loop.set_debug(True)
+1 -1
View File
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
"node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.reflector_servers) > 0,
"use_fixed_peers": len(conf.fixed_peers) > 0,
"fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count,
+5 -2
View File
@@ -158,11 +158,14 @@ class ComponentManager:
for component in self.components
}
def get_component(self, component_name):
def get_actual_component(self, component_name):
for component in self.components:
if component.component_name == component_name:
return component.component
return component
raise NameError(component_name)
def get_component(self, component_name):
return self.get_actual_component(component_name).component
def has_component(self, component_name):
return any(component for component in self.components if component_name == component.component_name)
+57 -14
View File
@@ -17,11 +17,17 @@ from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager
from lbry.blob_exchange.server import BlobServer
from lbry.stream.stream_manager import StreamManager
from lbry.file.file_manager import FileManager
from lbry.extras.daemon.component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.torrent.torrent_manager import TorrentManager
from lbry.wallet import WalletManager
from lbry.wallet.usage_payment import WalletServerPayer
try:
from lbry.torrent.session import TorrentSession
except ImportError:
TorrentSession = None
log = logging.getLogger(__name__)
@@ -33,10 +39,11 @@ WALLET_COMPONENT = "wallet"
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_MANAGER_COMPONENT = "stream_manager"
FILE_MANAGER_COMPONENT = "file_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
LIBTORRENT_COMPONENT = "libtorrent_component"
class DatabaseComponent(Component):
@@ -143,7 +150,7 @@ class WalletComponent(Component):
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
else:
progress = 100
best_hash = self.wallet_manager.get_best_blockhash()
best_hash = await self.wallet_manager.get_best_blockhash()
result.update({
'headers_synchronization_progress': progress,
'blocks': max(local_height, 0),
@@ -319,23 +326,23 @@ class HashAnnouncerComponent(Component):
}
class StreamManagerComponent(Component):
component_name = STREAM_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.stream_manager: typing.Optional[StreamManager] = None
self.file_manager: typing.Optional[FileManager] = None
@property
def component(self) -> typing.Optional[StreamManager]:
return self.stream_manager
def component(self) -> typing.Optional[FileManager]:
return self.file_manager
async def get_status(self):
if not self.stream_manager:
if not self.file_manager:
return
return {
'managed_files': len(self.stream_manager.streams),
'managed_files': len(self.file_manager.get_filtered()),
}
async def start(self):
@@ -344,16 +351,52 @@ class StreamManagerComponent(Component):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
node = self.component_manager.get_component(DHT_COMPONENT) \
if self.component_manager.has_component(DHT_COMPONENT) else None
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
self.file_manager = FileManager(
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
)
await self.stream_manager.start()
self.file_manager.source_managers['stream'] = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node,
)
if TorrentSession:
self.file_manager.source_managers['torrent'] = TorrentManager(
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
)
await self.file_manager.start()
log.info('Done setting up file manager')
async def stop(self):
self.stream_manager.stop()
self.file_manager.stop()
class TorrentComponent(Component):
component_name = LIBTORRENT_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.torrent_session = None
@property
def component(self) -> typing.Optional[TorrentSession]:
return self.torrent_session
async def get_status(self):
if not self.torrent_session:
return
return {
'running': True, # TODO: what to return here?
}
async def start(self):
if TorrentSession:
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
await self.torrent_session.bind() # TODO: specify host/port
async def stop(self):
if self.torrent_session:
await self.torrent_session.pause()
class PeerProtocolServerComponent(Component):
File diff suppressed because it is too large. Load diff
+76 -26
View File
@@ -7,6 +7,7 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError
from lbry.schema.claim import Claim
from lbry.torrent.torrent_manager import TorrentSource
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
from lbry.wallet.bip32 import PubKey
from lbry.wallet.dewies import dewies_to_lbc
@@ -25,6 +26,8 @@ def encode_txo_doc():
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_received': "true if txo was sent from external account to this account",
'is_spent': "true if txo is spent",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'purchase'",
'name': "when type is 'claim' or 'support', this is the claim name",
@@ -106,7 +109,9 @@ def encode_file_doc():
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
}
@@ -122,7 +127,7 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_account(obj)
if isinstance(obj, Wallet):
return self.encode_wallet(obj)
if isinstance(obj, ManagedStream):
if isinstance(obj, (ManagedStream, TorrentSource)):
return self.encode_file(obj)
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
@@ -164,12 +169,22 @@ class JSONResponseEncoder(JSONEncoder):
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger) if txo.has_address else None,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.is_spent is not None:
output['is_spent'] = txo.is_spent
if txo.is_my_output is not None:
output['is_my_output'] = txo.is_my_output
if txo.is_my_input is not None:
output['is_my_input'] = txo.is_my_input
if txo.sent_supports is not None:
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
if txo.sent_tips is not None:
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
if txo.received_tips is not None:
output['received_tips'] = dewies_to_lbc(txo.received_tips)
if txo.is_internal_transfer is not None:
output['is_internal_transfer'] = txo.is_internal_transfer
if txo.script.is_claim_name:
output['type'] = 'claim'
@@ -232,7 +247,7 @@ class JSONResponseEncoder(JSONEncoder):
if isinstance(value, int):
meta[key] = dewies_to_lbc(value)
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
return meta
def encode_input(self, txi):
@@ -259,26 +274,32 @@ class JSONResponseEncoder(JSONEncoder):
output_exists = managed_stream.output_file_exists
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
return {
'streaming_url': managed_stream.stream_url,
is_stream = hasattr(managed_stream, 'stream_hash')
if is_stream:
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
else:
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
result = {
'streaming_url': None,
'completed': managed_stream.completed,
'file_name': managed_stream.file_name if output_exists else None,
'download_directory': managed_stream.download_directory if output_exists else None,
'download_path': managed_stream.full_path if output_exists else None,
'file_name': None,
'download_directory': None,
'download_path': None,
'points_paid': 0.0,
'stopped': not managed_stream.running,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.descriptor.stream_name,
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
'stream_hash': None,
'stream_name': None,
'suggested_file_name': None,
'sd_hash': None,
'mime_type': None,
'key': None,
'total_bytes_lower_bound': total_bytes_lower_bound,
'total_bytes': total_bytes,
'written_bytes': managed_stream.written_bytes,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'blobs_completed': None,
'blobs_in_stream': None,
'blobs_remaining': None,
'status': managed_stream.status,
'claim_id': managed_stream.claim_id,
'txid': managed_stream.txid,
@@ -294,9 +315,38 @@ class JSONResponseEncoder(JSONEncoder):
'added_on': managed_stream.added_on,
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
'is_fully_reflected': managed_stream.is_fully_reflected
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': False,
'reflector_progress': False,
'uploading_to_reflector': False
}
if is_stream:
result.update({
'streaming_url': managed_stream.stream_url,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.descriptor.stream_name,
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
})
else:
result.update({
'streaming_url': f'file://{managed_stream.full_path}',
})
if output_exists:
result.update({
'file_name': managed_stream.file_name,
'download_directory': managed_stream.download_directory,
'download_path': managed_stream.full_path,
})
return result
def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict()
+31
View File
@@ -0,0 +1,31 @@
import logging
from aiohttp import web
log = logging.getLogger(__name__)
def ensure_request_allowed(request, conf):
if is_request_allowed(request, conf):
return
if conf.allowed_origin:
log.warning(
"API requests with Origin '%s' are not allowed, "
"configuration 'allowed_origin' limits requests to: '%s'",
request.headers.get('Origin'), conf.allowed_origin
)
else:
log.warning(
"API requests with Origin '%s' are not allowed, "
"update configuration 'allowed_origin' to enable this origin.",
request.headers.get('Origin')
)
raise web.HTTPForbidden()
def is_request_allowed(request, conf) -> bool:
origin = request.headers.get('Origin')
return (
origin is None or
origin == conf.allowed_origin or
conf.allowed_origin == '*'
)
+33 -2
View File
@@ -9,7 +9,7 @@ from typing import Optional
from lbry.wallet import SQLiteMixin
from lbry.conf import Config
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbry.wallet.transaction import Transaction
from lbry.wallet.transaction import Transaction, Output
from lbry.schema.claim import Claim
from lbry.dht.constants import DATA_EXPIRATION
from lbry.blob.blob_info import BlobInfo
@@ -727,6 +727,19 @@ class SQLiteStorage(SQLiteMixin):
if claim_id_to_supports:
await self.save_supports(claim_id_to_supports)
def save_claim_from_output(self, ledger, *outputs: Output):
return self.save_claims([{
"claim_id": output.claim_id,
"name": output.claim_name,
"amount": dewies_to_lbc(output.amount),
"address": output.get_address(ledger),
"txid": output.tx_ref.id,
"nout": output.position,
"value": output.claim,
"height": output.tx_ref.height,
"claim_sequence": -1,
} for output in outputs])
def save_claims_for_resolve(self, claim_infos):
to_save = {}
for info in claim_infos:
@@ -740,7 +753,8 @@ class SQLiteStorage(SQLiteMixin):
return self.save_claims(to_save.values())
@staticmethod
def _save_content_claim(transaction, claim_outpoint, stream_hash):
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
assert stream_hash or bt_infohash
# get the claim id and serialized metadata
claim_info = transaction.execute(
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
@@ -788,6 +802,19 @@ class SQLiteStorage(SQLiteMixin):
if stream_hash in self.content_claim_callbacks:
await self.content_claim_callbacks[stream_hash]()
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
def _save_torrent(transaction):
transaction.execute(
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
).fetchall()
transaction.execute(
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
).fetchall()
await self.db.run(_save_torrent)
# update corresponding ManagedEncryptedFileDownloader object
if bt_infohash in self.content_claim_callbacks:
await self.content_claim_callbacks[bt_infohash]()
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
claim = None
@@ -799,6 +826,10 @@ class SQLiteStorage(SQLiteMixin):
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
return claim
async def get_content_claim_for_torrent(self, bt_infohash):
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
return claims[bt_infohash].as_dict() if claims else None
# # # # # # # # # reflector functions # # # # # # # # #
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
View File
Whitespace-only changes.
+290
View File
@@ -0,0 +1,290 @@
import asyncio
import logging
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
from lbry.stream.managed_stream import ManagedStream
from lbry.torrent.torrent_manager import TorrentSource
from lbry.utils import cache_concurrent
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager, Output
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
class FileManager:
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.wallet_manager = wallet_manager
self.storage = storage
self.analytics_manager = analytics_manager
self.source_managers: typing.Dict[str, SourceManager] = {}
self.started = asyncio.Event()
@property
def streams(self):
return self.source_managers['stream']._sources
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
if 'stream' in self.source_managers:
return await self.source_managers['stream'].create(file_path, key, **kwargs)
raise NotImplementedError
async def start(self):
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
for manager in self.source_managers.values():
await manager.started.wait()
self.started.set()
def stop(self):
for manager in self.source_managers.values():
# fixme: pop or not?
manager.stop()
self.started.clear()
@cache_concurrent
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: Optional[float] = None, file_name: Optional[str] = None,
download_directory: Optional[str] = None,
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
wallet = wallet or self.wallet_manager.default_wallet
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
claim = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
payment = None
try:
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(
self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
resolve_timeout
)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
if 'error' in resolved_result:
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
if not resolved_result or uri not in resolved_result:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
txo = resolved_result[uri]
if isinstance(txo, dict):
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
claim = txo.claim
outpoint = f"{txo.tx_ref.id}:{txo.position}"
resolved_time = self.loop.time() - start_time
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
####################
# update or replace
####################
if claim.stream.source.bt_infohash:
source_manager = self.source_managers['torrent']
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
else:
source_manager = self.source_managers['stream']
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
# resume or update an existing stream, if the stream changed: download it and delete the old one after
to_replace, updated_stream = None, None
if existing and existing[0].claim_id != txo.claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
if claim.stream.source.bt_infohash:
await self.storage.save_torrent_content_claim(
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
existing[0].set_claim(claim_info, claim)
else:
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await source_manager._update_content_claim(existing[0])
updated_stream = existing[0]
else:
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
if save_file and existing_for_claim_id[0].output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
existing_for_claim_id[0].downloader.node = source_manager.node
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
if not existing_for_claim_id[0].output_file_exists and (
save_file or file_name or download_directory):
await existing_for_claim_id[0].save_file(
file_name=file_name, download_directory=download_directory
)
to_replace = existing_for_claim_id[0]
# resume or update an existing stream, if the stream changed: download it and delete the old one after
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
updated_stream.downloader.node = source_manager.node
await updated_stream.start(timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory
)
return updated_stream
####################
# pay fee
####################
if not to_replace and txo.has_price and not txo.purchase_receipt:
payment = await self.wallet_manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
####################
# make downloader and wait for start
####################
if not claim.stream.source.bt_infohash:
# fixme: this shouldnt be here
stream = ManagedStream(
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager
)
stream.downloader.node = source_manager.node
else:
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
file_name=file_name, download_directory=download_directory or self.config.download_dir,
status=ManagedStream.STATUS_RUNNING,
analytics_manager=self.analytics_manager,
torrent_session=source_manager.torrent_session
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(timeout, save_file)
####################
# success case: delete to_replace if applicable, broadcast fee payment
####################
if to_replace: # delete old stream now that the replacement has started downloading
await source_manager.delete(to_replace)
if payment is not None:
await self.wallet_manager.broadcast_or_release(payment)
payment = None # to avoid releasing in `finally` later
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
source_manager.add(stream)
if not claim.stream.source.bt_infohash:
await self.storage.save_content_claim(stream.stream_hash, outpoint)
else:
await self.storage.save_torrent_content_claim(
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
stream.set_claim(claim_info, claim)
if save_file:
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
elif isinstance(err, asyncio.CancelledError):
pass
else:
log.exception("Unexpected error downloading stream:")
error = err
raise
finally:
if payment is not None:
# payment is set to None after broadcasting, if we're here an exception probably happened
await self.wallet_manager.ledger.release_tx(payment)
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
# TODO: analytics for torrents
pass
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
server = self.wallet_manager.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"
)
)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
async def delete(self, source: ManagedDownloadSource, delete_file=False):
for manager in self.source_managers.values():
await manager.delete(source, delete_file)
+161
View File
@@ -0,0 +1,161 @@
import os
import asyncio
import typing
import logging
import binascii
from typing import Optional
from lbry.utils import generate_id
from lbry.extras.daemon.storage import StoredContentClaim
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class ManagedDownloadSource:
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
file_name: Optional[str] = None, download_directory: Optional[str] = None,
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
download_id: Optional[str] = None, rowid: Optional[int] = None,
content_fee: Optional['Transaction'] = None,
analytics_manager: Optional['AnalyticsManager'] = None,
added_on: Optional[int] = None):
self.loop = loop
self.storage = storage
self.config = config
self.identifier = identifier
self.download_directory = download_directory
self._file_name = file_name
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.content_fee = content_fee
self.purchase_receipt = None
self._added_on = added_on
self.analytics_manager = analytics_manager
self.saving = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop)
self.finished_write_attempt = asyncio.Event(loop=self.loop)
# @classmethod
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
# key: Optional[bytes] = None,
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
# raise NotImplementedError()
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
raise NotImplementedError()
async def stop(self, finished: bool = False):
raise NotImplementedError()
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
raise NotImplementedError()
def stop_tasks(self):
raise NotImplementedError()
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
self.stream_claim_info = StoredContentClaim(
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
# if not claim_info:
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
# self.set_claim(claim_info, claim_info['value'])
@property
def file_name(self) -> Optional[str]:
return self._file_name
@property
def added_on(self) -> Optional[int]:
return self._added_on
@property
def status(self) -> str:
return self._status
@property
def completed(self):
raise NotImplementedError()
# @property
# def stream_url(self):
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
@property
def finished(self) -> bool:
return self.status == self.STATUS_FINISHED
@property
def running(self) -> bool:
return self.status == self.STATUS_RUNNING
@property
def claim_id(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
@property
def txid(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.txid
@property
def nout(self) -> Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.nout
@property
def outpoint(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
@property
def claim_height(self) -> Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.height
@property
def channel_claim_id(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
@property
def channel_name(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
@property
def claim_name(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) -> Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property
def full_path(self) -> Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
+138
View File
@@ -0,0 +1,138 @@
import os
import asyncio
import logging
import typing
from typing import Optional
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
'g': lambda a, b: a > b,
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
}
class SourceManager:
filter_fields = {
'rowid',
'status',
'file_name',
'added_on',
'download_path',
'claim_name',
'claim_height',
'claim_id',
'outpoint',
'txid',
'nout',
'channel_claim_id',
'channel_name',
'completed'
}
set_filter_fields = {
"claim_ids": "claim_id",
"channel_claim_ids": "channel_claim_id",
"outpoints": "outpoint"
}
source_class = ManagedDownloadSource
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.storage = storage
self.analytics_manager = analytics_manager
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
self.started = asyncio.Event(loop=self.loop)
def add(self, source: ManagedDownloadSource):
self._sources[source.identifier] = source
def remove(self, source: ManagedDownloadSource):
if source.identifier not in self._sources:
return
self._sources.pop(source.identifier)
source.stop_tasks()
async def initialize_from_database(self):
raise NotImplementedError()
async def start(self):
await self.initialize_from_database()
self.started.set()
def stop(self):
while self._sources:
_, source = self._sources.popitem()
source.stop_tasks()
self.started.clear()
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
raise NotImplementedError()
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
self.remove(source)
if delete_file and source.output_file_exists:
os.remove(source.full_path)
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
if sort_by and sort_by not in self.filter_fields:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in COMPARISON_OPERATORS:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in self.filter_fields:
raise ValueError(f"'{search}' is not a valid search operation")
compare_sets = {}
if isinstance(search_by.get('claim_id'), list):
compare_sets['claim_ids'] = search_by.pop('claim_id')
if isinstance(search_by.get('outpoint'), list):
compare_sets['outpoints'] = search_by.pop('outpoint')
if isinstance(search_by.get('channel_claim_id'), list):
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
if search_by or compare_sets:
comparison = comparison or 'eq'
streams = []
for stream in self._sources.values():
if compare_sets and not all(
getattr(stream, self.set_filter_fields[set_search]) in val
for set_search, val in compare_sets.items()):
continue
if search_by and not all(
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
for search, val in search_by.items()):
continue
streams.append(stream)
else:
streams = list(self._sources.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
+152 -61
View File
@@ -3,10 +3,14 @@ import json
import logging
import os
import pathlib
import platform
import re
import shlex
import shutil
import subprocess
from math import ceil
import lbry.utils
from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__)
@@ -14,60 +18,104 @@ log = logging.getLogger(__name__)
class VideoFileAnalyzer:
def _replace_or_pop_env(self, variable):
if variable + '_ORIG' in self._env_copy:
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
else:
self._env_copy.pop(variable, None)
def __init__(self, conf: TranscodeConfig):
self._conf = conf
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
self._ffmpeg_installed = None
self._which_ffmpeg = None
self._which_ffprobe = None
self._env_copy = dict(os.environ)
self._checked_ffmpeg = False
if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment:
self._replace_or_pop_env('LD_LIBRARY_PATH')
async def _execute(self, command, arguments):
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(
os.path.join(self._conf.ffmpeg_folder, command), *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
async def _verify_executable(self, name):
@staticmethod
def _execute(command, environment):
# log.debug("Executing: %s", command)
try:
version, code = await self._execute(name, "-version")
with subprocess.Popen(
shlex.split(command) if platform.system() != 'Windows' else command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
) as process:
(stdout, stderr) = process.communicate() # blocks until the process exits
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
except subprocess.SubprocessError as e:
return str(e), -1
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
async def _execute_ffmpeg(self, arguments):
arguments = self._which_ffmpeg + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _execute_ffprobe(self, arguments):
arguments = self._which_ffprobe + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _verify_executables(self):
try:
await self._execute_ffprobe("-version")
version, code = await self._execute_ffmpeg("-version")
except Exception as e:
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
code = -1
version = ""
if code != 0 or not version.startswith(name):
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
version = str(e)
if code != 0 or not version.startswith("ffmpeg"):
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
raise FileNotFoundError(f"Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
f"and ensure that it is callable via PATH or conf.ffmpeg_path")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
return version
@staticmethod
def _which_ffmpeg_and_ffmprobe(path):
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
async def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed:
return
await self._verify_executable("ffprobe")
version = await self._verify_executable("ffmpeg")
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
self._ffmpeg_installed = False
path = self._conf.ffmpeg_path
if hasattr(self._conf, "data_dir"):
path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
path += os.path.pathsep + self._env_copy.get("PATH", "")
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
None, self._which_ffmpeg_and_ffmprobe, path
)
if not self._which_ffmpeg:
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
if not self._which_ffprobe:
log.warning("Unable to locate ffprobe executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
log.warning("ffmpeg and ffprobe are in different folders!")
await self._verify_executables()
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def status(self, reset=False, recheck=False):
if reset:
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
self._ffmpeg_installed = None
if self._checked_ffmpeg and not recheck:
installed = self._ffmpeg_installed
else:
installed = True
pass
elif self._ffmpeg_installed is None:
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
pass
self._checked_ffmpeg = True
return {
"available": installed,
"which": self._which,
"available": self._ffmpeg_installed,
"which": self._which_ffmpeg,
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
}
@@ -75,9 +123,24 @@ class VideoFileAnalyzer:
def _verify_container(scan_data: json):
container = scan_data["format"]["format_name"]
log.debug(" Detected container is %s", container)
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
splits = container.split(",")
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
return "Container format is not in the approved list of WebM, MP4. " \
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
if "matroska" in splits:
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
codec = stream["codec_name"]
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
return "WebM format requires VP8/9 or AV1 video. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
elif stream["codec_type"] == "audio":
codec = stream["codec_name"]
if not {"vorbis", "opus"}.intersection(codec.split(",")):
return "WebM format requires Vorbis or Opus audio. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
return ""
@staticmethod
@@ -106,12 +169,12 @@ class VideoFileAnalyzer:
bit_rate = float(scan_data["format"]["bit_rate"])
else:
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
if bit_rate > bit_rate_max:
return "The bit rate is above the configured maximum. Actual: " \
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
return ""
@@ -120,7 +183,7 @@ class VideoFileAnalyzer:
if {"webm", "ogg"}.intersection(container.split(",")):
return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
if match and int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
@@ -148,8 +211,8 @@ class VideoFileAnalyzer:
if not validate_volume:
return ""
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
try:
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
@@ -184,7 +247,7 @@ class VideoFileAnalyzer:
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.video_encoder.split(" ", 1)[0]
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
@@ -218,7 +281,7 @@ class VideoFileAnalyzer:
wants_opus = extension != "mp4"
if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.audio_encoder.split(" ", 1)[0]
if wants_opus and 'opus' in encoder:
@@ -251,25 +314,27 @@ class VideoFileAnalyzer:
# if we are vp8/vp9/av1 we want webm
# use mp4 for anything else
if not video_encoder: # not re-encoding video
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if video_encoder: # not re-encoding video
if "theora" in video_encoder:
return "ogv"
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
codec = stream["codec_name"].split(",")
if "theora" in codec:
return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm"
if "theora" in video_encoder:
return "ogv"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
async def _get_scan_data(self, validate, file_path):
result, _ = await self._execute("ffprobe",
f'-v quiet -print_format json -show_format -show_streams "{file_path}"')
arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
result, _ = await self._execute_ffprobe(arguments)
try:
scan_data = json.loads(result)
except Exception as e:
@@ -278,7 +343,7 @@ class VideoFileAnalyzer:
if "format" not in scan_data or "duration" not in scan_data["format"]:
log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
raise ValueError(f'Media file does not appear to contain video content: {file_path}')
if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
@@ -286,20 +351,46 @@ class VideoFileAnalyzer:
return scan_data
@staticmethod
def _build_spec(scan_data):
assert scan_data
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
width = -1
height = -1
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
width = max(width, int(stream["width"]))
height = max(height, int(stream["height"]))
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
spec = {"duration": duration}
if height >= 0:
spec["height"] = height
if width >= 0:
spec["width"] = width
return spec
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair:
return file_path
return file_path, {}
if ignore_non_video and not file_path:
return file_path, {}
await self._verify_ffmpeg_installed()
try:
scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path
return file_path, {}
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path)
spec = self._build_spec(scan_data)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data, file_path)
@@ -309,7 +400,7 @@ class VideoFileAnalyzer:
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
if not any(messages):
return file_path
return file_path, spec
if not repair:
errors = ["Streamability verification failed:"]
@@ -334,7 +425,6 @@ class VideoFileAnalyzer:
transcode_command.append("copy")
transcode_command.append("-movflags +faststart -c:a")
path = pathlib.Path(file_path)
extension = self._get_best_container_extension(scan_data, video_encoder)
if audio_msg or volume_msg:
@@ -347,12 +437,13 @@ class VideoFileAnalyzer:
transcode_command.append("copy")
# TODO: put it in a temp folder and delete it after we upload?
path = pathlib.Path(file_path)
output = path.parent / f"{path.stem}_fixed.{extension}"
transcode_command.append(f'"{output}"')
ffmpeg_command = " ".join(transcode_command)
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
result, code = await self._execute("ffmpeg", ffmpeg_command)
result, code = await self._execute_ffmpeg(ffmpeg_command)
if code != 0:
raise Exception(f"Failure to complete the transcode command. Output: {result}")
except Exception as e:
@@ -360,6 +451,6 @@ class VideoFileAnalyzer:
raise
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
# TODO: delete partial output file here if it exists?
return file_path
return file_path, spec
return str(output)
return str(output), spec
+32
View File
@@ -0,0 +1,32 @@
import logging
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
async def start(self, interface: str, port: int):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
await self.runner.cleanup()
+1 -1
View File
@@ -253,7 +253,7 @@ class Stream(BaseClaim):
if stream_type in ('image', 'video', 'audio'):
media = getattr(self, stream_type)
media_args = {'file_metadata': None}
if file_path is not None:
if file_path is not None and not all((duration, width, height)):
try:
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
except:
+1
View File
@@ -141,6 +141,7 @@ types_map = {
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.flac': ('audio/flac', 'audio'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4a': ('audio/mp4', 'audio'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
+17 -10
View File
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
yield os.urandom(AES.block_size // 8)
def file_reader(file_path: str):
def read_bytes(file_path: str, offset: int, to_read: int):
with open(file_path, 'rb') as f:
f.seek(offset)
return f.read(to_read)
async def file_reader(file_path: str):
length = int(os.stat(file_path).st_size)
offset = 0
with open(file_path, 'rb') as stream_file:
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = stream_file.read(bytes_to_read)
yield blob_bytes
offset += bytes_to_read
while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read:
break
blob_bytes = await asyncio.get_event_loop().run_in_executor(
None, read_bytes, file_path, offset, bytes_to_read
)
yield blob_bytes
offset += bytes_to_read
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
@@ -245,7 +252,7 @@ class StreamDescriptor:
iv_generator = iv_generator or random_iv_generator()
key = key or os.urandom(AES.block_size // 8)
blob_num = -1
for blob_bytes in file_reader(file_path):
async for blob_bytes in file_reader(file_path):
blob_num += 1
blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
+6 -6
View File
@@ -51,15 +51,15 @@ class StreamDownloader:
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
self.peer_queue.put_nowait([
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
for address, port in addresses
])
if not self.config.reflector_servers:
if not self.config.fixed_peers:
return
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
(await resolve_host(url, port, proto='tcp'), port)
for url, port in self.config.fixed_peers
]
if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()) > 0:
@@ -92,8 +92,8 @@ class StreamDownloader:
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
# set up peer accumulation
if node:
self.node = node
self.node = node or self.node # fixme: this shouldnt be set here!
if self.node:
if self.accumulate_task and not self.accumulate_task.done():
self.accumulate_task.cancel()
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
+70 -173
View File
@@ -3,9 +3,8 @@ import asyncio
import time
import typing
import logging
import binascii
from typing import Optional
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbry.utils import generate_id
from lbry.error import DownloadSDTimeoutError
from lbry.schema.mime_types import guess_media_type
from lbry.stream.downloader import StreamDownloader
@@ -13,6 +12,7 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
from lbry.stream.reflector.client import StreamReflectorClient
from lbry.extras.daemon.storage import StoredContentClaim
from lbry.blob import MAX_BLOB_SIZE
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.conf import Config
@@ -40,76 +40,35 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class ManagedStream:
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
__slots__ = [
'loop',
'config',
'blob_manager',
'sd_hash',
'download_directory',
'_file_name',
'_added_on',
'_status',
'stream_claim_info',
'download_id',
'rowid',
'content_fee',
'purchase_receipt',
'downloader',
'analytics_manager',
'fully_reflected',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
'streaming',
'_running',
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt'
]
class ManagedStream(ManagedDownloadSource):
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
descriptor: typing.Optional[StreamDescriptor] = None,
content_fee: typing.Optional['Transaction'] = None,
analytics_manager: typing.Optional['AnalyticsManager'] = None,
added_on: typing.Optional[int] = None):
self.loop = loop
self.config = config
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
claim: Optional[StoredContentClaim] = None,
download_id: Optional[str] = None, rowid: Optional[int] = None,
descriptor: Optional[StreamDescriptor] = None,
content_fee: Optional['Transaction'] = None,
analytics_manager: Optional['AnalyticsManager'] = None,
added_on: Optional[int] = None):
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
download_id, rowid, content_fee, analytics_manager, added_on)
self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.download_directory = download_directory
self._file_name = file_name
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.content_fee = content_fee
self.purchase_receipt = None
self._added_on = added_on
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
self.fully_reflected = asyncio.Event(loop=self.loop)
self.streaming = asyncio.Event(loop=self.loop)
self._running = asyncio.Event(loop=self.loop)
self.saving = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop)
self.finished_write_attempt = asyncio.Event(loop=self.loop)
@property
def sd_hash(self) -> str:
return self.identifier
@property
def is_fully_reflected(self) -> bool:
@@ -124,17 +83,9 @@ class ManagedStream:
return self.descriptor.stream_hash
@property
def file_name(self) -> typing.Optional[str]:
def file_name(self) -> Optional[str]:
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
@property
def added_on(self) -> typing.Optional[int]:
return self._added_on
@property
def status(self) -> str:
return self._status
@property
def written_bytes(self) -> int:
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
@@ -152,55 +103,6 @@ class ManagedStream:
self._status = status
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
@property
def finished(self) -> bool:
return self.status == self.STATUS_FINISHED
@property
def running(self) -> bool:
return self.status == self.STATUS_RUNNING
@property
def claim_id(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
@property
def txid(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.txid
@property
def nout(self) -> typing.Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.nout
@property
def outpoint(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
@property
def claim_height(self) -> typing.Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.height
@property
def channel_claim_id(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
@property
def channel_name(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
@property
def claim_name(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property
def blobs_completed(self) -> int:
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
@@ -214,39 +116,34 @@ class ManagedStream:
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
@property
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
@classmethod
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
"""
Generate a stream from a file and save it to the db
"""
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=blob_manager.blob_completed
)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
@property
def download_path(self):
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
# @classmethod
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
# file_path: str, key: Optional[bytes] = None,
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
# """
# Generate a stream from a file and save it to the db
# """
# descriptor = await StreamDescriptor.create_stream(
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
# blob_completed_callback=blob_manager.blob_completed
# )
# await blob_manager.storage.store_stream(
# blob_manager.get_blob(descriptor.sd_hash), descriptor
# )
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
# os.path.dirname(file_path), 0)
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
async def start(self, timeout: Optional[float] = None,
save_now: bool = False):
timeout = timeout or self.config.download_timeout
if self._running.is_set():
@@ -254,7 +151,7 @@ class ManagedStream:
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
self._running.set()
try:
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
except asyncio.TimeoutError:
self._running.clear()
raise DownloadSDTimeoutError(self.sd_hash)
@@ -264,6 +161,11 @@ class ManagedStream:
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
if not await self.blob_manager.storage.file_exists(self.sd_hash):
if save_now:
if not self._file_name:
self._file_name = await get_next_available_file_name(
self.loop, self.download_directory,
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
)
file_name, download_dir = self._file_name, self.download_directory
else:
file_name, download_dir = None, None
@@ -283,7 +185,7 @@ class ManagedStream:
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
if start_blob_num >= len(self.descriptor.blobs[:-1]):
raise IndexError(start_blob_num)
@@ -295,13 +197,13 @@ class ManagedStream:
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
async def stream_file(self, request: Request) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
await self.start(node)
await self.start()
response = StreamResponse(
status=206,
headers=headers
@@ -339,9 +241,10 @@ class ManagedStream:
self.streaming.clear()
@staticmethod
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
def _write_decrypted_blob(output_path: str, data: bytes):
with open(output_path, 'ab') as handle:
handle.write(data)
handle.flush()
async def _save_file(self, output_path: str):
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
@@ -351,12 +254,12 @@ class ManagedStream:
self.finished_writing.clear()
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
open(output_path, 'wb').close()
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
if not self.started_writing.is_set():
self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_finished(
@@ -386,9 +289,8 @@ class ManagedStream:
self.saving.clear()
self.finished_write_attempt.set()
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
node: typing.Optional['Node'] = None):
await self.start(node)
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
await self.start()
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
self.file_output_task.cancel()
self.download_directory = download_directory or self.download_directory or self.config.download_dir
@@ -430,6 +332,7 @@ class ManagedStream:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
@@ -445,9 +348,10 @@ class ManagedStream:
]
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
self.claim_name, self.claim_id)
for blob_hash in we_have:
for i, blob_hash in enumerate(we_have):
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
@@ -455,20 +359,13 @@ class ManagedStream:
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return sent
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
self.stream_claim_info = StoredContentClaim(
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
if not claim_info:
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
self.set_claim(claim_info, claim_info['value'])
+6 -4
View File
@@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None
self.connected.clear()
if self.pending_request:
self.pending_request.cancel()
if self.reflected_blobs:
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
@@ -56,11 +58,11 @@ class StreamReflectorClient(asyncio.Protocol):
self.response_buff = b''
return
async def send_request(self, request_dict: typing.Dict):
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.pending_request = self.loop.create_task(self.response_queue.get())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request
finally:
self.pending_request = None
@@ -87,7 +89,7 @@ class StreamReflectorClient(asyncio.Protocol):
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await self.response_queue.get()
received = await asyncio.wait_for(self.response_queue.get(), 30)
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
@@ -111,7 +113,7 @@ class StreamReflectorClient(asyncio.Protocol):
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await self.response_queue.get()
received = await asyncio.wait_for(self.response_queue.get(), 30)
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
+82 -322
View File
@@ -6,83 +6,67 @@ import random
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
from lbry.utils import cache_concurrent
from lbry.error import InvalidStreamDescriptorError
from lbry.file.source_manager import SourceManager
from lbry.stream.descriptor import StreamDescriptor
from lbry.stream.managed_stream import ManagedStream
from lbry.schema.claim import Claim
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet import Output
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.blob.blob_manager import BlobManager
from lbry.dht.node import Node
from lbry.wallet.wallet import WalletManager
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
FILTER_FIELDS = [
'rowid',
'status',
'file_name',
'added_on',
'sd_hash',
'stream_hash',
'claim_name',
'claim_height',
'claim_id',
'outpoint',
'txid',
'nout',
'channel_claim_id',
'channel_name',
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream'
]
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
'g': lambda a, b: a > b,
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
}
def path_or_none(path) -> Optional[str]:
if not path:
def path_or_none(encoded_path) -> Optional[str]:
if not encoded_path:
return
return binascii.unhexlify(path).decode()
return binascii.unhexlify(encoded_path).decode()
class StreamManager:
class StreamManager(SourceManager):
_sources: typing.Dict[str, ManagedStream]
filter_fields = SourceManager.filter_fields
filter_fields.update({
'sd_hash',
'stream_hash',
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream',
'uploading_to_reflector',
'is_fully_reflected'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
super().__init__(loop, config, storage, analytics_manager)
self.blob_manager = blob_manager
self.wallet_manager = wallet_manager
self.storage = storage
self.node = node
self.analytics_manager = analytics_manager
self.streams: typing.Dict[str, ManagedStream] = {}
self.resume_saving_task: Optional[asyncio.Task] = None
self.re_reflect_task: Optional[asyncio.Task] = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
self.started = asyncio.Event(loop=self.loop)
@property
def streams(self):
return self._sources
def add(self, source: ManagedStream):
super().add(source)
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = []
@@ -113,10 +97,10 @@ class StreamManager:
# if self.blob_manager._save_blobs:
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int], fully_reflected: bool):
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int], fully_reflected: Optional[bool]):
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err:
@@ -129,10 +113,9 @@ class StreamManager:
)
if fully_reflected:
stream.fully_reflected.set()
self.streams[sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
self.add(stream)
async def load_and_resume_streams_from_database(self):
async def initialize_from_database(self):
to_recover = []
to_start = []
@@ -146,7 +129,6 @@ class StreamManager:
to_recover.append(file_info)
to_start.append(file_info)
if to_recover:
log.info("Recover %i files", len(to_recover))
await self.recover_streams(to_recover)
log.info("Initializing %i files", len(to_start))
@@ -157,7 +139,7 @@ class StreamManager:
download_directory = path_or_none(file_info['download_directory'])
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
add_stream_tasks.append(self.loop.create_task(self.add_stream(
add_stream_tasks.append(self.loop.create_task(self._load_stream(
file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'],
file_info['claim'], file_info['content_fee'],
@@ -165,25 +147,22 @@ class StreamManager:
)))
if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop)
log.info("Started stream manager with %i files", len(self.streams))
log.info("Started stream manager with %i files", len(self._sources))
if not self.node:
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
if to_resume_saving:
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
async def resume(self, to_resume_saving):
log.info("Resuming saving %i files", len(to_resume_saving))
await asyncio.gather(
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
for (file_name, download_directory, sd_hash) in to_resume_saving),
loop=self.loop
)
log.info("Resuming saving %i files", len(to_resume_saving))
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
*(self._sources[sd_hash].save_file(file_name, download_directory)
for (file_name, download_directory, sd_hash) in to_resume_saving),
loop=self.loop
))
async def reflect_streams(self):
while True:
if self.config.reflect_streams and self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
batch = []
while sd_hashes:
stream = self.streams[sd_hashes.pop()]
@@ -199,18 +178,15 @@ class StreamManager:
await asyncio.sleep(300, loop=self.loop)
async def start(self):
await self.load_and_resume_streams_from_database()
await super().start()
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
self.started.set()
def stop(self):
super().stop()
if self.resume_saving_task and not self.resume_saving_task.done():
self.resume_saving_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
_, stream = self.streams.popitem()
stream.stop_tasks()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads:
@@ -233,261 +209,45 @@ class StreamManager:
)
return task
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
descriptor = await StreamDescriptor.create_stream(
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=self.blob_manager.blob_completed
)
await self.storage.store_stream(
self.blob_manager.get_blob(descriptor.sd_hash), descriptor
)
row_id = await self.storage.save_published_file(
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
)
stream = ManagedStream(
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
rowid=row_id, descriptor=descriptor
)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers:
self.reflect_stream(stream)
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
if stream.sd_hash in self.running_reflector_uploads:
self.running_reflector_uploads[stream.sd_hash].cancel()
stream.stop_tasks()
if stream.sd_hash in self.streams:
del self.streams[stream.sd_hash]
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
if not isinstance(source, ManagedStream):
return
if source.identifier in self.running_reflector_uploads:
self.running_reflector_uploads[source.identifier].cancel()
source.stop_tasks()
if source.identifier in self.streams:
del self.streams[source.identifier]
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor)
if delete_file and stream.output_file_exists:
os.remove(stream.full_path)
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
if streams:
return streams[0]
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
comparison: Optional[str] = None,
**search_by) -> typing.List[ManagedStream]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
if sort_by and sort_by not in FILTER_FIELDS:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in COMPARISON_OPERATORS:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in FILTER_FIELDS:
raise ValueError(f"'{search}' is not a valid search operation")
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams.values():
for search, val in search_by.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
streams.append(stream)
break
else:
streams = list(self.streams.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
return None, existing_for_claim_id[0]
return None, None
@staticmethod
def _convert_to_old_resolve_output(wallet_manager, resolves):
result = {}
for url, txo in resolves.items():
if isinstance(txo, Output):
tx_height = txo.tx_ref.height
best_height = wallet_manager.ledger.headers.height
result[url] = {
'name': txo.claim_name,
'value': txo.claim,
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
'claim_id': txo.claim_id,
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': dewies_to_lbc(txo.amount),
'effective_amount': txo.meta.get('effective_amount', 0),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'claim_sequence': -1,
'address': txo.get_address(wallet_manager.ledger),
'valid_at_height': txo.meta.get('activation_height', None),
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
'supports': []
}
else:
result[url] = txo
return result
@cache_concurrent
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: Optional[float] = None,
file_name: Optional[str] = None,
download_directory: Optional[str] = None,
save_file: Optional[bool] = None,
resolve_timeout: float = 3.0,
wallet: Optional['Wallet'] = None) -> ManagedStream:
manager = self.wallet_manager
wallet = wallet or manager.default_wallet
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
txo: Optional[Output] = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
payment = None
try:
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
response = await asyncio.wait_for(
manager.ledger.resolve(wallet.accounts, [uri]),
resolve_timeout
)
resolved_result = self._convert_to_old_resolve_output(manager, response)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values() if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
txo = response[uri]
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed: download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory, node=self.node
)
return updated_stream
if not to_replace and txo.has_price and not txo.purchase_receipt:
payment = await manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
stream = ManagedStream(
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
if payment is not None:
await manager.broadcast_or_release(payment)
payment = None # to avoid releasing in `finally` later
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
elif isinstance(err, asyncio.CancelledError):
pass
else:
log.exception("Unexpected error downloading stream:")
error = err
raise
finally:
if payment is not None:
# payment is set to None after broadcasting, if we're here an exception probably happened
await manager.ledger.release_tx(payment)
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
server = self.wallet_manager.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"
)
)
await self.storage.delete_stream(source.descriptor)
if delete_file and source.output_file_exists:
os.remove(source.full_path)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.streams[sd_hash].stream_file(request, self.node)
stream = self._sources[sd_hash]
if not stream.downloader.node:
stream.downloader.node = self.node
return await stream.stream_file(request)
+42 -4
View File
@@ -386,6 +386,7 @@ class CommandTestCase(IntegrationTestCase):
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.fixed_peers = [('127.0.0.1', 5567)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.components_to_skip = [
@@ -490,8 +491,8 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
)
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
return self.confirm_and_render(
async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
)
@@ -502,6 +503,11 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
)
async def purchase_create(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
)
async def publish(self, name, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
@@ -548,8 +554,28 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
)
async def resolve(self, uri):
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
async def support_abandon(self, *args, confirm=True, **kwargs):
if 'blocking' not in kwargs:
kwargs['blocking'] = False
return await self.confirm_and_render(
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
)
async def wallet_send(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
)
async def txo_spend(self, *args, confirm=True, **kwargs):
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
if confirm:
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
await self.generate(1)
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
return self.sout(txs)
async def resolve(self, uri, **kwargs):
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
@@ -557,6 +583,15 @@ class CommandTestCase(IntegrationTestCase):
async def file_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
async def txo_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
async def txo_sum(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
async def txo_plot(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
@@ -566,6 +601,9 @@ class CommandTestCase(IntegrationTestCase):
async def channel_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
async def transaction_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
@staticmethod
def get_claim_id(tx):
return tx['outputs'][0]['claim_id']
View File
Whitespace-only changes.
+290
View File
@@ -0,0 +1,290 @@
import asyncio
import binascii
import os
import logging
import random
from hashlib import sha1
from tempfile import mkdtemp
from typing import Optional
import libtorrent
NOTIFICATION_MASKS = [
"error",
"peer",
"port_mapping",
"storage",
"tracker",
"debug",
"status",
"progress",
"ip_block",
"dht",
"stats",
"session_log",
"torrent_log",
"peer_log",
"incoming_request",
"dht_log",
"dht_operation",
"port_mapping_log",
"picker_log",
"file_progress",
"piece_progress",
"upload",
"block_progress"
]
log = logging.getLogger(__name__)
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
libtorrent.add_torrent_params_flags_t.flag_auto_managed
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
)
def get_notification_type(notification) -> str:
for i, notification_type in enumerate(NOTIFICATION_MASKS):
if (1 << i) & notification:
return notification_type
raise ValueError("unrecognized notification type")
class TorrentHandle:
def __init__(self, loop, executor, handle):
self._loop = loop
self._executor = executor
self._handle: libtorrent.torrent_handle = handle
self.started = asyncio.Event(loop=loop)
self.finished = asyncio.Event(loop=loop)
self.metadata_completed = asyncio.Event(loop=loop)
self.size = 0
self.total_wanted_done = 0
self.name = ''
self.tasks = []
self.torrent_file: Optional[libtorrent.file_storage] = None
self._base_path = None
self._handle.set_sequential_download(1)
@property
def largest_file(self) -> Optional[str]:
if not self.torrent_file:
return None
index = self.largest_file_index
return os.path.join(self._base_path, self.torrent_file.at(index).path)
@property
def largest_file_index(self):
largest_size, index = 0, 0
for file_num in range(self.torrent_file.num_files()):
if self.torrent_file.file_size(file_num) > largest_size:
largest_size = self.torrent_file.file_size(file_num)
index = file_num
return index
def stop_tasks(self):
while self.tasks:
self.tasks.pop().cancel()
def _show_status(self):
# fixme: cleanup
if not self._handle.is_valid():
return
status = self._handle.status()
if status.has_metadata:
self.size = status.total_wanted
self.total_wanted_done = status.total_wanted_done
self.name = status.name
if not self.metadata_completed.is_set():
self.metadata_completed.set()
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
self.torrent_file = self._handle.get_torrent_info().files()
self._base_path = status.save_path
first_piece = self.torrent_file.at(self.largest_file_index).offset
if not self.started.is_set():
if self._handle.have_piece(first_piece):
self.started.set()
else:
# prioritize it
self._handle.set_piece_deadline(first_piece, 100)
if not status.is_seeding:
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.num_seeds, status.state, status.save_path)
elif not self.finished.is_set():
self.finished.set()
log.info("Torrent finished: %s", self.name)
async def status_loop(self):
while True:
self._show_status()
if self.finished.is_set():
break
await asyncio.sleep(0.1, loop=self._loop)
async def pause(self):
await self._loop.run_in_executor(
self._executor, self._handle.pause
)
async def resume(self):
await self._loop.run_in_executor(
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
)
class TorrentSession:
def __init__(self, loop, executor):
self._loop = loop
self._executor = executor
self._session: Optional[libtorrent.session] = None
self._handles = {}
self.tasks = []
self.wait_start = True
async def add_fake_torrent(self):
tmpdir = mkdtemp()
info, btih = _create_fake_torrent(tmpdir)
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
handle = self._session.add_torrent({
'ti': info, 'save_path': tmpdir, 'flags': flags
})
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
return btih
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
settings = {
'listen_interfaces': f"{interface}:{port}",
'enable_outgoing_utp': True,
'enable_incoming_utp': True,
'enable_outgoing_tcp': False,
'enable_incoming_tcp': False
}
self._session = await self._loop.run_in_executor(
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
)
self.tasks.append(self._loop.create_task(self.process_alerts()))
def stop(self):
while self.tasks:
self.tasks.pop().cancel()
self._session.save_state()
self._session.pause()
self._session.stop_dht()
self._session.stop_lsd()
self._session.stop_natpmp()
self._session.stop_upnp()
self._session = None
def _pop_alerts(self):
for alert in self._session.pop_alerts():
log.info("torrent alert: %s", alert)
async def process_alerts(self):
while True:
await self._loop.run_in_executor(
self._executor, self._pop_alerts
)
await asyncio.sleep(1, loop=self._loop)
async def pause(self):
await self._loop.run_in_executor(
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
)
await self._loop.run_in_executor(
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
)
async def resume(self):
await self._loop.run_in_executor(
self._executor, self._session.resume
)
def _add_torrent(self, btih: str, download_directory: Optional[str]):
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
if download_directory:
params['save_path'] = download_directory
handle = self._session.add_torrent(params)
handle.force_dht_announce()
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
def full_path(self, btih):
return self._handles[btih].largest_file
async def add_torrent(self, btih, download_path):
await self._loop.run_in_executor(
self._executor, self._add_torrent, btih, download_path
)
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
await self._handles[btih].metadata_completed.wait()
if self.wait_start:
# fixme: temporary until we add streaming support, otherwise playback fails!
await self._handles[btih].started.wait()
def remove_torrent(self, btih, remove_files=False):
if btih in self._handles:
handle = self._handles[btih]
handle.stop_tasks()
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
self._handles.pop(btih)
async def save_file(self, btih, download_directory):
handle = self._handles[btih]
await handle.resume()
def get_size(self, btih):
return self._handles[btih].size
def get_name(self, btih):
return self._handles[btih].name
def get_downloaded(self, btih):
return self._handles[btih].total_wanted_done
def is_completed(self, btih):
return self._handles[btih].finished.is_set()
def get_magnet_uri(btih):
return f"magnet:?xt=urn:btih:{btih}"
def _create_fake_torrent(tmpdir):
# beware, that's just for testing
path = os.path.join(tmpdir, 'tmp')
with open(path, 'wb') as myfile:
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
file_storage = libtorrent.file_storage()
file_storage.add_file('tmp', size)
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
libtorrent.set_piece_hashes(t, tmpdir)
info = libtorrent.torrent_info(t.generate())
btih = sha1(info.metadata()).hexdigest()
return info, btih
async def main():
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
executor = None
session = TorrentSession(asyncio.get_event_loop(), executor)
session2 = TorrentSession(asyncio.get_event_loop(), executor)
await session.bind('localhost', port=4040)
await session2.bind('localhost', port=4041)
btih = await session.add_fake_torrent()
session2._session.add_dht_node(('localhost', 4040))
await session2.add_torrent(btih, "/tmp/down")
while True:
await asyncio.sleep(100)
await session.pause()
executor.shutdown()
if __name__ == "__main__":
asyncio.run(main())
+72
View File
@@ -0,0 +1,72 @@
import asyncio
import logging
import typing
log = logging.getLogger(__name__)
class TorrentInfo:
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
self.dht_seeds = dht_seeds
self.http_seeds = http_seeds
self.trackers = trackers
self.total_size = total_size
@classmethod
def from_libtorrent_info(cls, torrent_info):
return cls(
torrent_info.nodes(), tuple(
{
'url': web_seed['url'],
'type': web_seed['type'],
'auth': web_seed['auth']
} for web_seed in torrent_info.web_seeds()
), tuple(
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
), torrent_info.total_size()
)
class Torrent:
def __init__(self, loop, handle):
self._loop = loop
self._handle = handle
self.finished = asyncio.Event(loop=loop)
def _threaded_update_status(self):
status = self._handle.status()
if not status.is_seeding:
log.info(
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.state
)
elif not self.finished.is_set():
self.finished.set()
async def wait_for_finished(self):
while True:
await self._loop.run_in_executor(
None, self._threaded_update_status
)
if self.finished.is_set():
log.info("finished downloading torrent!")
await self.pause()
break
await asyncio.sleep(1, loop=self._loop)
async def pause(self):
log.info("pause torrent")
await self._loop.run_in_executor(
None, self._handle.pause
)
async def resume(self):
await self._loop.run_in_executor(
None, self._handle.resume
)
+140
View File
@@ -0,0 +1,140 @@
import asyncio
import binascii
import logging
import os
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.torrent.session import TorrentSession
from lbry.conf import Config
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.storage import StoredContentClaim
log = logging.getLogger(__name__)
def path_or_none(encoded_path) -> Optional[str]:
if not encoded_path:
return
return binascii.unhexlify(encoded_path).decode()
class TorrentSource(ManagedDownloadSource):
STATUS_STOPPED = "stopped"
filter_fields = SourceManager.filter_fields
filter_fields.update({
'bt_infohash'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
file_name: Optional[str] = None, download_directory: Optional[str] = None,
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
download_id: Optional[str] = None, rowid: Optional[int] = None,
content_fee: Optional['Transaction'] = None,
analytics_manager: Optional['AnalyticsManager'] = None,
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
rowid, content_fee, analytics_manager, added_on)
self.torrent_session = torrent_session
@property
def full_path(self) -> Optional[str]:
full_path = self.torrent_session.full_path(self.identifier)
self.download_directory = os.path.dirname(full_path)
return full_path
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
async def stop(self, finished: bool = False):
await self.torrent_session.remove_torrent(self.identifier)
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
await self.torrent_session.save_file(self.identifier, download_directory)
@property
def torrent_length(self):
return self.torrent_session.get_size(self.identifier)
@property
def written_bytes(self):
return self.torrent_session.get_downloaded(self.identifier)
@property
def torrent_name(self):
return self.torrent_session.get_name(self.identifier)
@property
def bt_infohash(self):
return self.identifier
def stop_tasks(self):
pass
@property
def completed(self):
return self.torrent_session.is_completed(self.identifier)
class TorrentManager(SourceManager):
_sources: typing.Dict[str, ManagedDownloadSource]
filter_fields = set(SourceManager.filter_fields)
filter_fields.update({
'bt_infohash',
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
'blobs_in_stream'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
super().__init__(loop, config, storage, analytics_manager)
self.torrent_session: 'TorrentSession' = torrent_session
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
raise NotImplementedError
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int]):
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
torrent_session=self.torrent_session
)
self.add(stream)
async def initialize_from_database(self):
pass
async def start(self):
await super().start()
def stop(self):
super().stop()
log.info("finished stopping the torrent manager")
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
await super().delete(source, delete_file)
self.torrent_session.remove_torrent(source.identifier, delete_file)
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
raise NotImplementedError
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
raise NotImplementedError
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
# await self.storage.delete_stream(source.descriptor)
async def stream_partial_content(self, request: Request, sd_hash: str):
raise NotImplementedError
+29
View File
@@ -3,7 +3,9 @@ import codecs
import datetime
import random
import socket
import time
import string
import sys
import json
import typing
import asyncio
@@ -276,3 +278,30 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
return response['data']['ip']
except Exception:
return
def is_running_from_bundle():
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
class LockWithMetrics(asyncio.Lock):
def __init__(self, acquire_metric, held_time_metric, loop=None):
super().__init__(loop=loop)
self._acquire_metric = acquire_metric
self._lock_held_time_metric = held_time_metric
self._lock_acquired_time = None
async def acquire(self):
start = time.perf_counter()
try:
return await super().acquire()
finally:
self._lock_acquired_time = time.perf_counter()
self._acquire_metric.observe(self._lock_acquired_time - start)
def release(self):
try:
return super().release()
finally:
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+39 -24
View File
@@ -71,6 +71,7 @@ class AddressManager:
def _query_addresses(self, **constraints):
return self.account.ledger.db.get_addresses(
read_only=constraints.pop("read_only", False),
accounts=[self.account],
chain=self.chain_number,
**constraints
@@ -434,9 +435,9 @@ class Account:
addresses.extend(new_addresses)
return addresses
async def get_addresses(self, **constraints) -> List[str]:
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
return [r[0] for r in rows]
async def get_addresses(self, read_only=False, **constraints) -> List[str]:
rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
return [r['address'] for r in rows]
def get_address_records(self, **constraints):
return self.ledger.db.get_addresses(accounts=[self], **constraints)
@@ -451,13 +452,13 @@ class Account:
def get_public_key(self, chain: int, index: int) -> PubKey:
return self.address_managers[chain].get_public_key(index)
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
if not include_claims:
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
if confirmations > 0:
height = self.ledger.headers.height - (confirmations-1)
constraints.update({'height__lte': height, 'height__gt': 0})
return self.ledger.db.get_balance(accounts=[self], **constraints)
return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
async def get_max_gap(self):
change_gap = await self.change.get_max_gap()
@@ -467,6 +468,12 @@ class Account:
'max_receiving_gap': receiving_gap,
}
def get_txos(self, **constraints):
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
def get_txo_count(self, **constraints):
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
def get_utxos(self, **constraints):
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
@@ -518,13 +525,18 @@ class Account:
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
def get_channel_private_key(self, public_key_bytes):
async def get_channel_private_key(self, public_key_bytes):
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
if private_key_pem:
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
return await asyncio.get_event_loop().run_in_executor(
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
)
async def maybe_migrate_certificates(self):
def to_der(private_key_pem):
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256).get_verifying_key().to_der()
if not self.channel_keys:
return
channel_keys = {}
@@ -533,8 +545,7 @@ class Account:
continue
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
continue
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
public_key_der = private_key.get_verifying_key().to_der()
public_key_der = await asyncio.get_event_loop().run_in_executor(None, to_der, private_key_pem)
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
if self.channel_keys != channel_keys:
self.channel_keys = channel_keys
@@ -555,20 +566,20 @@ class Account:
if gap_changed:
self.wallet.save()
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False, read_only=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
get_total_balance = partial(self.get_balance, read_only=read_only, confirmations=confirmations,
include_claims=True)
total = await get_total_balance()
if reserved_subtotals:
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
for amount, spent, from_me, to_me, height in await self.get_support_summary():
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
for txo in await self.get_support_summary():
if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1):
continue
if not spent and to_me:
if from_me:
supports_balance += amount
else:
tips_balance += amount
if txo.is_my_input:
supports_balance += txo.amount
else:
tips_balance += txo.amount
reserved = claims_balance + supports_balance + tips_balance
else:
reserved = await self.get_balance(
@@ -585,11 +596,15 @@ class Account:
} if reserved_subtotals else None
}
def get_transaction_history(self, **constraints):
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
def get_transaction_history(self, read_only=False, **constraints):
return self.ledger.get_transaction_history(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_transaction_history_count(self, **constraints):
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
def get_transaction_history_count(self, read_only=False, **constraints):
return self.ledger.get_transaction_history_count(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_claims(self, **constraints):
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
@@ -622,7 +637,7 @@ class Account:
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
def get_support_summary(self):
return self.ledger.db.get_supports_summary(account_id=self.id)
return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self)
+743
View File
@@ -0,0 +1,743 @@
HASHES = {
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
}
+9 -4
View File
@@ -6,6 +6,7 @@ COIN = 100*CENT
TIMEOUT = 30.0
TXO_TYPES = {
"other": 0,
"stream": 1,
"channel": 2,
"support": 3,
@@ -14,9 +15,13 @@ TXO_TYPES = {
"repost": 6,
}
CLAIM_TYPE_NAMES = [
'stream',
'channel',
'collection',
'repost',
]
CLAIM_TYPES = [
TXO_TYPES['stream'],
TXO_TYPES['channel'],
TXO_TYPES['collection'],
TXO_TYPES['repost'],
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
]
+484 -150
View File
@@ -1,45 +1,132 @@
import os
import logging
import asyncio
import sqlite3
import platform
from binascii import hexlify
from dataclasses import dataclass
from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from prometheus_client import Gauge, Counter, Histogram
from lbry.utils import LockWithMetrics
from .bip32 import PubKey
from .transaction import Transaction, Output, OutputScript, TXRefImmutable
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
log = logging.getLogger(__name__)
sqlite3.enable_callback_tracebacks(True)
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
@dataclass
class ReaderProcessState:
cursor: sqlite3.Cursor
reader_context: Optional[ContextVar[ReaderProcessState]] = ContextVar('reader_context')
def initializer(path):
db = sqlite3.connect(path)
db.row_factory = dict_row_factory
db.executescript("pragma journal_mode=WAL;")
reader = ReaderProcessState(db.cursor())
reader_context.set(reader)
def run_read_only_fetchall(sql, params):
cursor = reader_context.get().cursor
try:
return cursor.execute(sql, params).fetchall()
except (Exception, OSError) as e:
log.exception('Error running transaction:', exc_info=e)
raise
def run_read_only_fetchone(sql, params):
cursor = reader_context.get().cursor
try:
return cursor.execute(sql, params).fetchone()
except (Exception, OSError) as e:
log.exception('Error running transaction:', exc_info=e)
raise
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
ReaderExecutorClass = ThreadPoolExecutor
else:
ReaderExecutorClass = ProcessPoolExecutor
class AIOSQLite:
reader_executor: ReaderExecutorClass
waiting_writes_metric = Gauge(
"waiting_writes_count", "Number of waiting db writes", namespace="daemon_database"
)
waiting_reads_metric = Gauge(
"waiting_reads_count", "Number of waiting db writes", namespace="daemon_database"
)
write_count_metric = Counter(
"write_count", "Number of database writes", namespace="daemon_database"
)
read_count_metric = Counter(
"read_count", "Number of database reads", namespace="daemon_database"
)
acquire_write_lock_metric = Histogram(
f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS
)
held_write_lock_metric = Histogram(
f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database",
buckets=HISTOGRAM_BUCKETS
)
def __init__(self):
# has to be single threaded as there is no mapping of thread:connection
self.executor = ThreadPoolExecutor(max_workers=1)
self.connection: sqlite3.Connection = None
self.writer_executor = ThreadPoolExecutor(max_workers=1)
self.writer_connection: Optional[sqlite3.Connection] = None
self._closing = False
self.query_count = 0
self.write_lock = LockWithMetrics(self.acquire_write_lock_metric, self.held_write_lock_metric)
self.writers = 0
self.read_ready = asyncio.Event()
self.urgent_read_done = asyncio.Event()
@classmethod
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
sqlite3.enable_callback_tracebacks(True)
def _connect():
return sqlite3.connect(path, *args, **kwargs)
db = cls()
db.connection = await asyncio.get_event_loop().run_in_executor(db.executor, _connect)
def _connect_writer():
db.writer_connection = sqlite3.connect(path, *args, **kwargs)
readers = max(os.cpu_count() - 2, 2)
db.reader_executor = ReaderExecutorClass(
max_workers=readers, initializer=initializer, initargs=(path, )
)
await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer)
db.read_ready.set()
db.urgent_read_done.set()
return db
async def close(self):
if self._closing:
return
self._closing = True
await asyncio.get_event_loop().run_in_executor(self.executor, self.connection.close)
self.executor.shutdown(wait=True)
self.connection = None
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
self.writer_executor.shutdown(wait=True)
self.reader_executor.shutdown(wait=True)
self.read_ready.clear()
self.writer_connection = None
def executemany(self, sql: str, params: Iterable):
params = params if params is not None else []
@@ -49,52 +136,121 @@ class AIOSQLite:
def executescript(self, script: str) -> Awaitable:
return self.run(lambda conn: conn.executescript(script))
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
async def _execute_fetch(self, sql: str, parameters: Iterable = None,
read_only=False, fetch_all: bool = False) -> List[dict]:
read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
still_waiting = False
urgent_read = False
if read_only:
self.waiting_reads_metric.inc()
self.read_count_metric.inc()
try:
while self.writers: # more writes can come in while we are waiting for the first
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
# throttle the writes if they pile up
self.urgent_read_done.clear()
urgent_read = True
# wait until the running writes have finished
await self.read_ready.wait()
still_waiting = True
return await asyncio.get_event_loop().run_in_executor(
self.reader_executor, read_only_fn, sql, parameters
)
finally:
if urgent_read:
# unthrottle the writers if they had to be throttled
self.urgent_read_done.set()
self.waiting_reads_metric.dec()
if fetch_all:
return await self.run(lambda conn: conn.execute(sql, parameters).fetchall())
return await self.run(lambda conn: conn.execute(sql, parameters).fetchone())
def execute_fetchone(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchone())
async def execute_fetchall(self, sql: str, parameters: Iterable = None,
read_only=False) -> List[dict]:
return await self._execute_fetch(sql, parameters, read_only, fetch_all=True)
async def execute_fetchone(self, sql: str, parameters: Iterable = None,
read_only=False) -> List[dict]:
return await self._execute_fetch(sql, parameters, read_only, fetch_all=False)
def execute(self, sql: str, parameters: Iterable = None) -> Awaitable[sqlite3.Cursor]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters))
def run(self, fun, *args, **kwargs) -> Awaitable:
return asyncio.get_event_loop().run_in_executor(
self.executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
async def run(self, fun, *args, **kwargs):
self.write_count_metric.inc()
self.waiting_writes_metric.inc()
# it's possible many writes are coming in one after the other, these can
# block reader calls for a long time
# if the reader waits for the writers to finish and then has to wait for
# yet more, it will clear the urgent_read_done event to block more writers
# piling on
try:
await self.urgent_read_done.wait()
except Exception as e:
self.waiting_writes_metric.dec()
raise e
self.writers += 1
# block readers
self.read_ready.clear()
try:
async with self.write_lock:
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
)
finally:
self.writers -= 1
self.waiting_writes_metric.dec()
if not self.writers:
# unblock the readers once the last enqueued writer finishes
self.read_ready.set()
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
self.connection.execute('begin')
self.writer_connection.execute('begin')
try:
self.query_count += 1
result = fun(self.connection, *args, **kwargs) # type: ignore
self.connection.commit()
result = fun(self.writer_connection, *args, **kwargs) # type: ignore
self.writer_connection.commit()
return result
except (Exception, OSError) as e:
log.exception('Error running transaction:', exc_info=e)
self.connection.rollback()
self.writer_connection.rollback()
log.warning("rolled back")
raise
def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable:
return asyncio.get_event_loop().run_in_executor(
self.executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
async def run_with_foreign_keys_disabled(self, fun, *args, **kwargs):
self.write_count_metric.inc()
self.waiting_writes_metric.inc()
try:
await self.urgent_read_done.wait()
except Exception as e:
self.waiting_writes_metric.dec()
raise e
self.writers += 1
self.read_ready.clear()
try:
async with self.write_lock:
return await asyncio.get_event_loop().run_in_executor(
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
)
finally:
self.writers -= 1
self.waiting_writes_metric.dec()
if not self.writers:
self.read_ready.set()
def __run_transaction_with_foreign_keys_disabled(self,
fun: Callable[[sqlite3.Connection, Any, Any], Any],
args, kwargs):
foreign_keys_enabled, = self.connection.execute("pragma foreign_keys").fetchone()
foreign_keys_enabled, = self.writer_connection.execute("pragma foreign_keys").fetchone()
if not foreign_keys_enabled:
raise sqlite3.IntegrityError("foreign keys are disabled, use `AIOSQLite.run` instead")
try:
self.connection.execute('pragma foreign_keys=off').fetchone()
self.writer_connection.execute('pragma foreign_keys=off').fetchone()
return self.__run_transaction(fun, *args, **kwargs)
finally:
self.connection.execute('pragma foreign_keys=on').fetchone()
self.writer_connection.execute('pragma foreign_keys=on').fetchone()
def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
@@ -107,7 +263,9 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
if not key:
sql.append(constraint)
continue
if key.startswith('$'):
if key.startswith('$$'):
col, key = col[2:], key[1:]
elif key.startswith('$'):
values[key] = constraint
continue
if key.endswith('__not'):
@@ -173,6 +331,7 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
limit = constraints.pop('limit', None)
offset = constraints.pop('offset', None)
order_by = constraints.pop('order_by', None)
group_by = constraints.pop('group_by', None)
accounts = constraints.pop('accounts', [])
if accounts:
@@ -183,6 +342,9 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
sql.append('WHERE')
sql.append(where)
if group_by is not None:
sql.append(f'GROUP BY {group_by}')
if order_by:
sql.append('ORDER BY')
if isinstance(order_by, str):
@@ -214,11 +376,17 @@ def interpolate(sql, values):
return sql
def rows_to_dict(rows, fields):
if rows:
return [dict(zip(fields, r)) for r in rows]
else:
return []
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
if value is not None:
if isinstance(value, list):
value = [convert(v) for v in value]
if len(value) == 1:
constraints[column] = value[0]
elif len(value) > 1:
constraints[f"{column}__in"] = value
else:
constraints[column] = convert(value)
return constraints
class SQLiteMixin:
@@ -291,9 +459,16 @@ class SQLiteMixin:
return sql, values
def dict_row_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
class Database(SQLiteMixin):
SCHEMA_VERSION = "1.1"
SCHEMA_VERSION = "1.3"
PRAGMAS = """
pragma journal_mode=WAL;
@@ -328,7 +503,8 @@ class Database(SQLiteMixin):
height integer not null,
position integer not null,
is_verified boolean not null default 0,
purchased_claim_id text
purchased_claim_id text,
day integer
);
create index if not exists tx_purchased_claim_id_idx on tx (purchased_claim_id);
"""
@@ -345,22 +521,29 @@ class Database(SQLiteMixin):
txo_type integer not null default 0,
claim_id text,
claim_name text
claim_name text,
channel_id text,
reposted_claim_id text
);
create index if not exists txo_txid_idx on txo (txid);
create index if not exists txo_address_idx on txo (address);
create index if not exists txo_claim_id_idx on txo (claim_id);
create index if not exists txo_claim_id_idx on txo (claim_id, txo_type);
create index if not exists txo_claim_name_idx on txo (claim_name);
create index if not exists txo_txo_type_idx on txo (txo_type);
create index if not exists txo_channel_id_idx on txo (channel_id);
create index if not exists txo_reposted_claim_idx on txo (reposted_claim_id);
"""
CREATE_TXI_TABLE = """
create table if not exists txi (
txid text references tx,
txoid text references txo,
address text references pubkey_address
txoid text references txo primary key,
address text references pubkey_address,
position integer not null
);
create index if not exists txi_address_idx on txi (address);
create index if not exists txi_txoid_idx on txi (txoid);
create index if not exists first_input_idx on txi (txid, address) where position=0;
"""
CREATE_TABLES_QUERY = (
@@ -372,19 +555,27 @@ class Database(SQLiteMixin):
CREATE_TXI_TABLE
)
@staticmethod
def txo_to_row(tx, address, txo):
async def open(self):
await super().open()
self.db.writer_connection.row_factory = dict_row_factory
def txo_to_row(self, tx, txo):
row = {
'txid': tx.id,
'txoid': txo.id,
'address': address,
'address': txo.get_address(self.ledger),
'position': txo.position,
'amount': txo.amount,
'script': sqlite3.Binary(txo.script.source)
}
if txo.is_claim:
if txo.can_decode_claim:
row['txo_type'] = TXO_TYPES.get(txo.claim.claim_type, TXO_TYPES['stream'])
claim = txo.claim
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
if claim.is_repost:
row['reposted_claim_id'] = claim.repost.reference.claim_id
if claim.is_signed:
row['channel_id'] = claim.signing_channel_id
else:
row['txo_type'] = TXO_TYPES['stream']
elif txo.is_support:
@@ -397,14 +588,14 @@ class Database(SQLiteMixin):
row['claim_name'] = txo.claim_name
return row
@staticmethod
def tx_to_row(tx):
def tx_to_row(self, tx):
row = {
'txid': tx.id,
'raw': sqlite3.Binary(tx.raw),
'height': tx.height,
'position': tx.position,
'is_verified': tx.is_verified
'is_verified': tx.is_verified,
'day': tx.get_julian_day(self.ledger),
}
txos = tx.outputs
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
@@ -423,25 +614,29 @@ class Database(SQLiteMixin):
def _transaction_io(self, conn: sqlite3.Connection, tx: Transaction, address, txhash):
conn.execute(*self._insert_sql('tx', self.tx_to_row(tx), replace=True)).fetchall()
for txo in tx.outputs:
if txo.script.is_pay_pubkey_hash and txo.pubkey_hash == txhash:
conn.execute(*self._insert_sql(
"txo", self.txo_to_row(tx, address, txo), ignore_duplicate=True
)).fetchall()
elif txo.script.is_pay_script_hash:
# TODO: implement script hash payments
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
is_my_input = False
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
txo = txi.txo_ref.txo
if txo.has_address and txo.get_address(self.ledger) == address:
is_my_input = True
conn.execute(*self._insert_sql("txi", {
'txid': tx.id,
'txoid': txo.id,
'address': address,
'position': txi.position
}, ignore_duplicate=True)).fetchall()
for txo in tx.outputs:
if txo.script.is_pay_pubkey_hash and (txo.pubkey_hash == txhash or is_my_input):
conn.execute(*self._insert_sql(
"txo", self.txo_to_row(tx, txo), ignore_duplicate=True
)).fetchall()
elif txo.script.is_pay_script_hash:
# TODO: implement script hash payments
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
def save_transaction_io(self, tx: Transaction, address, txhash, history):
return self.save_transaction_io_batch([tx], address, txhash, history)
@@ -459,7 +654,7 @@ class Database(SQLiteMixin):
return self.db.run(__many)
async def reserve_outputs(self, txos, is_reserved=True):
txoids = ((is_reserved, txo.id) for txo in txos)
txoids = [(is_reserved, txo.id) for txo in txos]
await self.db.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", txoids)
async def release_outputs(self, txos):
@@ -471,24 +666,29 @@ class Database(SQLiteMixin):
# 2. update address histories removing deleted TXs
return True
async def select_transactions(self, cols, accounts=None, **constraints):
async def select_transactions(self, cols, accounts=None, read_only=False, **constraints):
if not {'txid', 'txid__in'}.intersection(constraints):
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
constraints.update({
f'$account{i}': a.public_key.address for i, a in enumerate(accounts)
where, values = constraints_to_sql({
'$$account_address.account__in': [a.public_key.address for a in accounts]
})
account_values = ', '.join([f':$account{i}' for i in range(len(accounts))])
where = f" WHERE account_address.account IN ({account_values})"
constraints['txid__in'] = f"""
SELECT txo.txid FROM txo JOIN account_address USING (address) {where}
SELECT txo.txid FROM txo JOIN account_address USING (address) WHERE {where}
UNION
SELECT txi.txid FROM txi JOIN account_address USING (address) {where}
SELECT txi.txid FROM txi JOIN account_address USING (address) WHERE {where}
"""
constraints.update(values)
return await self.db.execute_fetchall(
*query(f"SELECT {cols} FROM tx", **constraints)
*query(f"SELECT {cols} FROM tx", **constraints), read_only=read_only
)
TXO_NOT_MINE = Output(None, None, is_my_output=False)
async def get_transactions(self, wallet=None, **constraints):
include_is_spent = constraints.pop('include_is_spent', False)
include_is_my_input = constraints.pop('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
tx_rows = await self.select_transactions(
'txid, raw, height, position, is_verified',
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
@@ -500,9 +700,10 @@ class Database(SQLiteMixin):
txids, txs, txi_txoids = [], [], []
for row in tx_rows:
txids.append(row[0])
txids.append(row['txid'])
txs.append(Transaction(
raw=row[1], height=row[2], position=row[3], is_verified=bool(row[4])
raw=row['raw'], height=row['height'], position=row['position'],
is_verified=bool(row['is_verified'])
))
for txi in txs[-1].inputs:
txi_txoids.append(txi.txo_ref.id)
@@ -514,7 +715,10 @@ class Database(SQLiteMixin):
txo.id: txo for txo in
(await self.get_txos(
wallet=wallet,
txid__in=txids[offset:offset+step],
txid__in=txids[offset:offset+step], order_by='txo.txid',
include_is_spent=include_is_spent,
include_is_my_input=include_is_my_input,
include_is_my_output=include_is_my_output,
))
})
@@ -524,7 +728,8 @@ class Database(SQLiteMixin):
txo.id: txo for txo in
(await self.get_txos(
wallet=wallet,
txoid__in=txi_txoids[offset:offset+step],
txoid__in=txi_txoids[offset:offset+step], order_by='txo.txoid',
include_is_my_output=include_is_my_output,
))
})
@@ -538,7 +743,7 @@ class Database(SQLiteMixin):
if _txo:
txo.update_annotations(_txo)
else:
txo.update_annotations(None)
txo.update_annotations(self.TXO_NOT_MINE)
for tx in txs:
txos = tx.outputs
@@ -552,58 +757,158 @@ class Database(SQLiteMixin):
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = await self.select_transactions('count(*)', **constraints)
return count[0][0]
count = await self.select_transactions('COUNT(*) as total', **constraints)
return count[0]['total'] or 0
async def get_transaction(self, **constraints):
txs = await self.get_transactions(limit=1, **constraints)
if txs:
return txs[0]
async def select_txos(self, cols, **constraints):
sql = f"SELECT {cols} FROM txo JOIN tx USING (txid)"
if 'accounts' in constraints:
sql += " JOIN account_address USING (address)"
return await self.db.execute_fetchall(*query(sql, **constraints))
async def select_txos(
self, cols, accounts=None, is_my_input=None, is_my_output=True,
is_my_input_or_output=None, exclude_internal_transfers=False,
include_is_spent=False, include_is_my_input=False,
is_spent=None, read_only=False, **constraints):
for rename_col in ('txid', 'txoid'):
for rename_constraint in (rename_col, rename_col+'__in', rename_col+'__not_in'):
if rename_constraint in constraints:
constraints['txo.'+rename_constraint] = constraints.pop(rename_constraint)
if accounts:
account_in_sql, values = constraints_to_sql({
'$$account__in': [a.public_key.address for a in accounts]
})
my_addresses = f"SELECT address FROM account_address WHERE {account_in_sql}"
constraints.update(values)
if is_my_input_or_output:
include_is_my_input = True
constraints['received_or_sent__or'] = {
'txo.address__in': my_addresses,
'sent__and': {
'txi.address__is_not_null': True,
'txi.address__in': my_addresses
}
}
else:
if is_my_output:
constraints['txo.address__in'] = my_addresses
elif is_my_output is False:
constraints['txo.address__not_in'] = my_addresses
if is_my_input:
include_is_my_input = True
constraints['txi.address__is_not_null'] = True
constraints['txi.address__in'] = my_addresses
elif is_my_input is False:
include_is_my_input = True
constraints['is_my_input_false__or'] = {
'txi.address__is_null': True,
'txi.address__not_in': my_addresses
}
if exclude_internal_transfers:
include_is_my_input = True
constraints['exclude_internal_payments__or'] = {
'txo.txo_type__not': TXO_TYPES['other'],
'txo.address__not_in': my_addresses,
'txi.address__is_null': True,
'txi.address__not_in': my_addresses,
}
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
if is_spent:
constraints['spent.txoid__is_not_null'] = True
elif is_spent is False:
constraints['is_reserved'] = False
constraints['spent.txoid__is_null'] = True
if include_is_spent or is_spent is not None:
sql.append("LEFT JOIN txi AS spent ON (spent.txoid=txo.txoid)")
if include_is_my_input:
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)
async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraints):
include_is_spent = constraints.get('include_is_spent', False)
include_is_my_input = constraints.get('include_is_my_input', False)
include_is_my_output = constraints.pop('include_is_my_output', False)
include_received_tips = constraints.pop('include_received_tips', False)
select_columns = [
"tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, "
"txo_type, txo.position as txo_position, amount, script"
]
async def get_txos(self, wallet=None, no_tx=False, **constraints):
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
if 'order_by' not in constraints:
my_accounts_sql = ""
if include_is_my_output or include_is_my_input:
my_accounts_sql, values = constraints_to_sql({'$$account__in#_wallet': my_accounts})
constraints.update(values)
if include_is_my_output and my_accounts:
if constraints.get('is_my_output', None) in (True, False):
select_columns.append(f"{1 if constraints['is_my_output'] else 0} AS is_my_output")
else:
select_columns.append(f"""(
txo.address IN (SELECT address FROM account_address WHERE {my_accounts_sql})
) AS is_my_output""")
if include_is_my_input and my_accounts:
if constraints.get('is_my_input', None) in (True, False):
select_columns.append(f"{1 if constraints['is_my_input'] else 0} AS is_my_input")
else:
select_columns.append(f"""(
txi.address IS NOT NULL AND
txi.address IN (SELECT address FROM account_address WHERE {my_accounts_sql})
) AS is_my_input""")
if include_is_spent:
select_columns.append("spent.txoid IS NOT NULL AS is_spent")
if include_received_tips:
select_columns.append(f"""(
SELECT COALESCE(SUM(support.amount), 0) FROM txo AS support WHERE
support.claim_id = txo.claim_id AND
support.txo_type = {TXO_TYPES['support']} AND
support.address IN (SELECT address FROM account_address WHERE {my_accounts_sql}) AND
support.txoid NOT IN (SELECT txoid FROM txi)
) AS received_tips""")
if 'order_by' not in constraints or constraints['order_by'] == 'height':
constraints['order_by'] = [
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
]
rows = await self.select_txos(
"""
tx.txid, raw, tx.height, tx.position, tx.is_verified, txo.position, amount, script, (
select group_concat(account||"|"||chain) from account_address
where account_address.address=txo.address
)
""",
**constraints
)
elif constraints.get('order_by', None) == 'none':
del constraints['order_by']
rows = await self.select_txos(', '.join(select_columns), read_only=read_only, **constraints)
txos = []
txs = {}
for row in rows:
if no_tx:
txo = Output(
amount=row[6],
script=OutputScript(row[7]),
tx_ref=TXRefImmutable.from_id(row[0], row[2]),
position=row[5]
amount=row['amount'],
script=OutputScript(row['script']),
tx_ref=TXRefImmutable.from_id(row['txid'], row['height']),
position=row['txo_position']
)
else:
if row[0] not in txs:
txs[row[0]] = Transaction(
row[1], height=row[2], position=row[3], is_verified=row[4]
if row['txid'] not in txs:
txs[row['txid']] = Transaction(
row['raw'], height=row['height'], position=row['tx_position'],
is_verified=bool(row['is_verified'])
)
txo = txs[row[0]].outputs[row[5]]
row_accounts = dict(a.split('|') for a in row[8].split(','))
account_match = set(row_accounts) & my_accounts
if account_match:
txo.is_my_account = True
txo.is_change = row_accounts[account_match.pop()] == '1'
else:
txo.is_change = txo.is_my_account = False
txo = txs[row['txid']].outputs[row['txo_position']]
if include_is_spent:
txo.is_spent = bool(row['is_spent'])
if include_is_my_input:
txo.is_my_input = bool(row['is_my_input'])
if include_is_my_output:
txo.is_my_output = bool(row['is_my_output'])
if include_is_my_input and include_is_my_output:
if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']:
txo.is_internal_transfer = True
else:
txo.is_internal_transfer = False
if include_received_tips:
txo.received_tips = row['received_tips']
txos.append(txo)
channel_ids = set()
@@ -613,7 +918,7 @@ class Database(SQLiteMixin):
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and wallet:
for account in wallet.accounts:
private_key = account.get_channel_private_key(
private_key = await account.get_channel_private_key(
txo.claim.channel.public_key_bytes
)
if private_key:
@@ -625,7 +930,8 @@ class Database(SQLiteMixin):
txo.claim_id: txo for txo in
(await self.get_channels(
wallet=wallet,
claim_id__in=channel_ids
claim_id__in=channel_ids,
read_only=read_only
))
}
for txo in txos:
@@ -634,48 +940,76 @@ class Database(SQLiteMixin):
return txos
async def get_txo_count(self, **constraints):
constraints.pop('resolve', None)
@staticmethod
def _clean_txo_constraints_for_aggregation(constraints):
constraints.pop('include_is_spent', None)
constraints.pop('include_is_my_input', None)
constraints.pop('include_is_my_output', None)
constraints.pop('include_received_tips', None)
constraints.pop('wallet', None)
constraints.pop('resolve', None)
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = await self.select_txos('count(*)', **constraints)
return count[0][0]
@staticmethod
def constrain_utxo(constraints):
constraints['is_reserved'] = False
constraints['txoid__not_in'] = "SELECT txoid FROM txi"
async def get_txo_count(self, **constraints):
self._clean_txo_constraints_for_aggregation(constraints)
count = await self.select_txos('COUNT(*) AS total', **constraints)
return count[0]['total'] or 0
def get_utxos(self, **constraints):
self.constrain_utxo(constraints)
return self.get_txos(**constraints)
async def get_txo_sum(self, **constraints):
self._clean_txo_constraints_for_aggregation(constraints)
result = await self.select_txos('SUM(amount) AS total', **constraints)
return result[0]['total'] or 0
async def get_txo_plot(self, start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
self._clean_txo_constraints_for_aggregation(constraints)
if start_day is None:
constraints['day__gte'] = self.ledger.headers.estimated_julian_day(
self.ledger.headers.height
) - days_back
else:
constraints['day__gte'] = date_to_julian_day(
date.fromisoformat(start_day)
)
if end_day is not None:
constraints['day__lte'] = date_to_julian_day(
date.fromisoformat(end_day)
)
elif days_after is not None:
constraints['day__lte'] = constraints['day__gte'] + days_after
return await self.select_txos(
"DATE(day) AS day, SUM(amount) AS total",
group_by='day', order_by='day', **constraints
)
def get_utxos(self, read_only=False, **constraints):
return self.get_txos(is_spent=False, read_only=read_only, **constraints)
def get_utxo_count(self, **constraints):
self.constrain_utxo(constraints)
return self.get_txo_count(**constraints)
return self.get_txo_count(is_spent=False, **constraints)
async def get_balance(self, wallet=None, accounts=None, **constraints):
async def get_balance(self, wallet=None, accounts=None, read_only=False, **constraints):
assert wallet or accounts, \
"'wallet' or 'accounts' constraints required to calculate balance"
constraints['accounts'] = accounts or wallet.accounts
self.constrain_utxo(constraints)
balance = await self.select_txos('SUM(amount)', **constraints)
return balance[0][0] or 0
balance = await self.select_txos(
'SUM(amount) as total', is_spent=False, read_only=read_only, **constraints
)
return balance[0]['total'] or 0
async def select_addresses(self, cols, **constraints):
async def select_addresses(self, cols, read_only=False, **constraints):
return await self.db.execute_fetchall(*query(
f"SELECT {cols} FROM pubkey_address JOIN account_address USING (address)",
**constraints
))
), read_only=read_only)
async def get_addresses(self, cols=None, **constraints):
async def get_addresses(self, cols=None, read_only=False, **constraints):
cols = cols or (
'address', 'account', 'chain', 'history', 'used_times',
'pubkey', 'chain_code', 'n', 'depth'
)
addresses = rows_to_dict(await self.select_addresses(', '.join(cols), **constraints), cols)
addresses = await self.select_addresses(', '.join(cols), read_only=read_only, **constraints)
if 'pubkey' in cols:
for address in addresses:
address['pubkey'] = PubKey(
@@ -684,12 +1018,12 @@ class Database(SQLiteMixin):
)
return addresses
async def get_address_count(self, cols=None, **constraints):
count = await self.select_addresses('count(*)', **constraints)
return count[0][0]
async def get_address_count(self, cols=None, read_only=False, **constraints):
count = await self.select_addresses('COUNT(*) as total', read_only=read_only, **constraints)
return count[0]['total'] or 0
async def get_address(self, **constraints):
addresses = await self.get_addresses(limit=1, **constraints)
async def get_address(self, read_only=False, **constraints):
addresses = await self.get_addresses(read_only=read_only, limit=1, **constraints)
if addresses:
return addresses[0]
@@ -743,17 +1077,19 @@ class Database(SQLiteMixin):
@staticmethod
def constrain_claims(constraints):
if {'txo_type', 'txo_type__in'}.intersection(constraints):
return
claim_types = constraints.pop('claim_type', None)
if isinstance(claim_types, str) and claim_types:
claim_types = [claim_types]
if isinstance(claim_types, list) and claim_types:
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
if claim_types:
constrain_single_or_list(
constraints, 'txo_type', claim_types, lambda x: TXO_TYPES[x]
)
else:
constraints['txo_type__in'] = CLAIM_TYPES
async def get_claims(self, **constraints) -> List[Output]:
async def get_claims(self, read_only=False, **constraints) -> List[Output]:
self.constrain_claims(constraints)
return await self.get_utxos(**constraints)
return await self.get_utxos(read_only=read_only, **constraints)
def get_claim_count(self, **constraints):
self.constrain_claims(constraints)
@@ -763,9 +1099,9 @@ class Database(SQLiteMixin):
def constrain_streams(constraints):
constraints['txo_type'] = TXO_TYPES['stream']
def get_streams(self, **constraints):
def get_streams(self, read_only=False, **constraints):
self.constrain_streams(constraints)
return self.get_claims(**constraints)
return self.get_claims(read_only=read_only, **constraints)
def get_stream_count(self, **constraints):
self.constrain_streams(constraints)
@@ -815,13 +1151,11 @@ class Database(SQLiteMixin):
" )", (account.public_key.address, )
)
def get_supports_summary(self, account_id):
return self.db.execute_fetchall(f"""
select txo.amount, exists(select * from txi where txi.txoid=txo.txoid) as spent,
(txo.txid in
(select txi.txid from txi join account_address a on txi.address = a.address
where a.account = ?)) as from_me,
(txo.address in (select address from account_address where account=?)) as to_me,
tx.height
from txo join tx using (txid) where txo_type={TXO_TYPES['support']}
""", (account_id, account_id))
def get_supports_summary(self, read_only=False, **constraints):
return self.get_txos(
txo_type=TXO_TYPES['support'],
is_spent=False, is_my_output=True,
include_is_my_input=True,
no_tx=True, read_only=read_only,
**constraints
)
+138 -68
View File
@@ -1,16 +1,18 @@
import base64
import os
import struct
import asyncio
import hashlib
import logging
import zlib
from datetime import date
from io import BytesIO
from contextlib import asynccontextmanager
from typing import Optional, Iterator, Tuple
from typing import Optional, Iterator, Tuple, Callable
from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256
from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES
log = logging.getLogger(__name__)
@@ -32,25 +34,50 @@ class Headers:
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
target_timespan = 150
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
checkpoints = HASHES
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
timestamp_average_offset = 160.6855883050695 # calculated at 733447
validate_difficulty: bool = True
def __init__(self, path) -> None:
if path == ':memory:':
self.io = BytesIO()
self.io = None
self.path = path
self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self):
self.io = BytesIO()
if self.path != ':memory:':
if not os.path.exists(self.path):
self.io = open(self.path, 'w+b')
else:
self.io = open(self.path, 'r+b')
def _readit():
if os.path.exists(self.path):
with open(self.path, 'r+b') as header_file:
self.io.seek(0)
self.io.write(header_file.read())
await asyncio.get_event_loop().run_in_executor(None, _readit)
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
if bytes_size % self.header_size:
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
await self.repair()
else:
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
await self.repair(start_height=max_checkpointed_height)
await self.ensure_checkpointed_size()
await self.get_all_missing_headers()
async def close(self):
self.io.close()
if self.io is not None:
def _close():
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
with open(self.path, flags) as header_file:
header_file.write(self.io.getbuffer())
await asyncio.get_event_loop().run_in_executor(None, _close)
self.io.close()
self.io = None
@staticmethod
def serialize(header):
@@ -97,23 +124,92 @@ class Headers:
return new_target
def __len__(self) -> int:
if self._size is None:
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return self._size
def __bool__(self):
return True
def __getitem__(self, height) -> dict:
async def get(self, height) -> dict:
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
try:
return self.deserialize(height, await self.get_raw_header(height))
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height, try_real_headers=True):
if height <= 0:
return
if try_real_headers and self.has_header(height):
offset = height * self.header_size
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self.deserialize(height, self.get_raw_header(height))
return self._read(height)
def get_raw_header(self, height) -> bytes:
self.io.seek(height * self.header_size, os.SEEK_SET)
return self.io.read(self.header_size)
def _read(self, height, count=1):
offset = height * self.header_size
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
def chunk_hash(self, start, count):
return self.hash_header(self._read(start, count)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
if self.height < max_checkpointed_height:
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
async def fetch_chunk(self, height):
log.info("on-demand fetching height %s", height)
start = (height // 1000) * 1000
headers = await self.chunk_getter(start) # pylint: disable=not-callable
chunk = (
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
self._write(start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
elif start not in self.checkpoints:
return # todo: fixme
raise Exception(
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
@property
def height(self) -> int:
@@ -123,9 +219,9 @@ class Headers:
def bytes_size(self):
return len(self) * self.header_size
def hash(self, height=None) -> bytes:
async def hash(self, height=None) -> bytes:
return self.hash_header(
self.get_raw_header(height if height is not None else self.height)
await self.get_raw_header(height if height is not None else self.height)
)
@staticmethod
@@ -134,44 +230,18 @@ class Headers:
return b'0' * 64
return hexlify(double_sha256(header)[::-1])
@asynccontextmanager
async def checkpointed_connector(self):
buf = BytesIO()
try:
yield buf
finally:
await asyncio.sleep(0)
final_height = len(self) + buf.tell() // self.header_size
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
buf.seek(0)
self.io.seek(0)
h = hashlib.sha256()
h.update(self.io.read())
h.update(buf.read(verifiable_bytes))
if h.hexdigest().encode() == self.checkpoint[1]:
buf.seek(0)
self._write(len(self), buf.read(verifiable_bytes))
remaining = buf.read()
buf.seek(0)
buf.write(remaining)
buf.truncate()
else:
log.warning("Checkpoint mismatch, connecting headers through slow method.")
if buf.tell() > 0:
await self.connect(len(self), buf.getvalue())
async def connect(self, start: int, headers: bytes) -> int:
added = 0
bail = False
for height, chunk in self._iterate_chunks(start, headers):
try:
# validate_chunk() is CPU bound and reads previous chunks from file system
self.validate_chunk(height, chunk)
await self.validate_chunk(height, chunk)
except InvalidHeader as e:
bail = True
chunk = chunk[:(height-e.height)*self.header_size]
added += self._write(height, chunk) if chunk else 0
if chunk:
added += self._write(height, chunk)
if bail:
break
return added
@@ -179,20 +249,21 @@ class Headers:
def _write(self, height, verified_chunk):
self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(verified_chunk) // self.header_size
self.io.truncate()
# self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = self.io.tell() // self.header_size
self._size = max(self._size or 0, self.io.tell() // self.header_size)
return written
def validate_chunk(self, height, chunk):
async def validate_chunk(self, height, chunk):
previous_hash, previous_header, previous_previous_header = None, None, None
if height > 0:
previous_header = self[height-1]
previous_hash = self.hash(height-1)
raw = await self.get_raw_header(height-1)
previous_header = self.deserialize(height-1, raw)
previous_hash = self.hash_header(raw)
if height > 1:
previous_previous_header = self[height-2]
previous_previous_header = await self.get(height-2)
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
for current_hash, current_header in self._iterate_headers(height, chunk):
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
@@ -231,28 +302,30 @@ class Headers:
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
)
async def repair(self):
async def repair(self, start_height=0):
previous_header_hash = fail = None
batch_size = 36
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
for height in range(start_height, self.height, batch_size):
headers = self._read(height, batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers):
for header_hash, header in self._iterate_headers(height, headers):
height = header['block_height']
if height:
if previous_header_hash:
if header['prev_block_hash'] != previous_header_hash:
fail = True
else:
elif height == 0:
if header_hash != self.genesis_hash:
fail = True
else:
# for sanity and clarity, since it is the only way we can end up here
assert start_height > 0 and height == start_height
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return
previous_header_hash = header_hash
@@ -276,10 +349,6 @@ class Headers:
header = headers[start:end]
yield self.hash_header(header), self.deserialize(height+idx, header)
@property
def claim_trie_root(self):
return self[self.height]['claim_trie_root']
@staticmethod
def header_hash_to_pow_hash(header_hash: bytes):
header_hash_bytes = unhexlify(header_hash)[::-1]
@@ -295,3 +364,4 @@ class UnvalidatedHeaders(Headers):
validate_difficulty = False
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
checkpoints = {}
+206 -75
View File
@@ -1,15 +1,15 @@
import os
import zlib
import base64
import copy
import time
import asyncio
import logging
from io import StringIO
from datetime import datetime
from functools import partial
from operator import itemgetter
from collections import namedtuple, defaultdict
from collections import defaultdict
from binascii import hexlify, unhexlify
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
import pylru
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
@@ -25,7 +25,8 @@ from .account import Account, AddressManager, SingleKey
from .network import Network
from .transaction import Transaction, Output
from .header import Headers, UnvalidatedHeaders
from .constants import TXO_TYPES, COIN, NULL_HASH32
from .checkpoints import HASHES
from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
from .bip32 import PubKey, PrivateKey
from .coinselection import CoinSelector
@@ -53,25 +54,29 @@ class LedgerRegistry(type):
return mcs.ledgers[ledger_id]
class TransactionEvent(namedtuple('TransactionEvent', ('address', 'tx'))):
pass
class TransactionEvent(NamedTuple):
address: str
tx: Transaction
class AddressesGeneratedEvent(namedtuple('AddressesGeneratedEvent', ('address_manager', 'addresses'))):
pass
class AddressesGeneratedEvent(NamedTuple):
address_manager: AddressManager
addresses: List[str]
class BlockHeightEvent(namedtuple('BlockHeightEvent', ('height', 'change'))):
pass
class BlockHeightEvent(NamedTuple):
height: int
change: int
class TransactionCacheItem:
__slots__ = '_tx', 'lock', 'has_tx'
__slots__ = '_tx', 'lock', 'has_tx', 'pending_verifications'
def __init__(self, tx: Optional[Transaction] = None, lock: Optional[asyncio.Lock] = None):
self.has_tx = asyncio.Event()
self.lock = lock or asyncio.Lock()
self._tx = self.tx = tx
self.pending_verifications = 0
@property
def tx(self) -> Optional[Transaction]:
@@ -105,6 +110,8 @@ class Ledger(metaclass=LedgerRegistry):
default_fee_per_byte = 50
default_fee_per_name_char = 200000
checkpoints = HASHES
def __init__(self, config=None):
self.config = config or {}
self.db: Database = self.config.get('db') or Database(
@@ -114,6 +121,7 @@ class Ledger(metaclass=LedgerRegistry):
self.headers: Headers = self.config.get('headers') or self.headers_class(
os.path.join(self.path, "headers")
)
self.headers.checkpoints = self.checkpoints
self.network: Network = self.config.get('network') or Network(self)
self.network.on_header.listen(self.receive_header)
self.network.on_status.listen(self.process_status_update)
@@ -150,8 +158,9 @@ class Ledger(metaclass=LedgerRegistry):
self._on_ready_controller = StreamController()
self.on_ready = self._on_ready_controller.stream
self._tx_cache = pylru.lrucache(100000)
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 100_000))
self._update_tasks = TaskGroup()
self._other_tasks = TaskGroup() # that we dont need to start
self._utxo_reservation_lock = asyncio.Lock()
self._header_processing_lock = asyncio.Lock()
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
@@ -262,6 +271,21 @@ class Ledger(metaclass=LedgerRegistry):
self.constraint_spending_utxos(constraints)
return self.db.get_utxo_count(**constraints)
async def get_txos(self, resolve=False, **constraints) -> List[Output]:
txos = await self.db.get_txos(**constraints)
if resolve:
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
return txos
def get_txo_count(self, **constraints):
return self.db.get_txo_count(**constraints)
def get_txo_sum(self, **constraints):
return self.db.get_txo_sum(**constraints)
def get_txo_plot(self, **constraints):
return self.db.get_txo_plot(**constraints)
def get_transactions(self, **constraints):
return self.db.get_transactions(**constraints)
@@ -297,12 +321,12 @@ class Ledger(metaclass=LedgerRegistry):
self.db.open(),
self.headers.open()
])
first_connection = self.network.on_connected.first
asyncio.ensure_future(self.network.start())
await first_connection
fully_synced = self.on_ready.first
asyncio.create_task(self.network.start())
await self.network.on_connected.first
async with self._header_processing_lock:
await self._update_tasks.add(self.initial_headers_sync())
await self._on_ready_controller.stream.first
await fully_synced
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
if len(self.accounts) > 10:
@@ -313,15 +337,15 @@ class Ledger(metaclass=LedgerRegistry):
async def join_network(self, *_):
log.info("Subscribing and updating accounts.")
async with self._header_processing_lock:
await self.update_headers()
await self.subscribe_accounts()
await self._update_tasks.add(self.subscribe_accounts())
await self._update_tasks.done.wait()
self._on_ready_controller.add(True)
async def stop(self):
self._update_tasks.cancel()
self._other_tasks.cancel()
await self._update_tasks.done.wait()
await self._other_tasks.done.wait()
await self.network.stop()
await self.db.close()
await self.headers.close()
@@ -331,19 +355,15 @@ class Ledger(metaclass=LedgerRegistry):
return max(self.headers.height, self._download_height)
async def initial_headers_sync(self):
target = self.network.remote_height + 1
current = len(self.headers)
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=4096, b64=True)
chunks = [asyncio.create_task(get_chunk(height)) for height in range(current, target, 4096)]
total = 0
async with self.headers.checkpointed_connector() as buffer:
for chunk in chunks:
headers = await chunk
total += buffer.write(
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
self._download_height = current + total // self.headers.header_size
log.info("Headers sync: %s / %s", self._download_height, target)
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True)
self.headers.chunk_getter = get_chunk
async def doit():
for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)):
async with self._header_processing_lock:
await self.headers.ensure_chunk_at(height)
self._other_tasks.add(doit())
await self.update_headers()
async def update_headers(self, height=None, headers=None, subscription_update=False):
rewound = 0
@@ -480,14 +500,15 @@ class Ledger(metaclass=LedgerRegistry):
if not we_need:
return True
cache_tasks: List[asyncio.Future[Transaction]] = []
cache_tasks: List[asyncio.Task[Transaction]] = []
synced_history = StringIO()
loop = asyncio.get_running_loop()
for i, (txid, remote_height) in enumerate(remote_history):
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
synced_history.write(f'{txid}:{remote_height}:')
else:
check_local = (txid, remote_height) not in we_need
cache_tasks.append(asyncio.ensure_future(
cache_tasks.append(loop.create_task(
self.cache_transaction(txid, remote_height, check_local=check_local)
))
@@ -509,7 +530,9 @@ class Ledger(metaclass=LedgerRegistry):
check_db_for_txos.append(txi.txo_ref.id)
referenced_txos = {} if not check_db_for_txos else {
txo.id: txo for txo in await self.db.get_txos(txoid__in=check_db_for_txos, no_tx=True)
txo.id: txo for txo in await self.db.get_txos(
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
)
}
for txi in tx.inputs:
@@ -561,6 +584,14 @@ class Ledger(metaclass=LedgerRegistry):
(cache_item.tx.is_verified or remote_height < 1):
return cache_item.tx # cached tx is already up-to-date
try:
cache_item.pending_verifications += 1
return await self._update_cache_item(cache_item, txid, remote_height, check_local)
finally:
cache_item.pending_verifications -= 1
async def _update_cache_item(self, cache_item, txid, remote_height, check_local=True):
async with cache_item.lock:
tx = cache_item.tx
@@ -569,21 +600,31 @@ class Ledger(metaclass=LedgerRegistry):
# check local db
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
merkle = None
if tx is None:
# fetch from network
_raw = await self.network.retriable_call(self.network.get_transaction, txid, remote_height)
tx = Transaction(unhexlify(_raw))
_raw, merkle = await self.network.retriable_call(
self.network.get_transaction_and_merkle, txid, remote_height
)
tx = Transaction(unhexlify(_raw), height=merkle.get('block_height'))
cache_item.tx = tx # make sure it's saved before caching it
await self.maybe_verify_transaction(tx, remote_height)
await self.maybe_verify_transaction(tx, remote_height, merkle)
return tx
async def maybe_verify_transaction(self, tx, remote_height):
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
tx.height = remote_height
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
cached = self._tx_cache.get(tx.id)
if not cached:
# cache txs looked up by transaction_show too
cached = TransactionCacheItem()
cached.tx = tx
self._tx_cache[tx.id] = cached
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
if not merkle:
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = self.headers[remote_height]
header = await self.headers.get(remote_height)
tx.position = merkle['pos']
tx.is_verified = merkle_root == header['merkle_root']
@@ -599,6 +640,7 @@ class Ledger(metaclass=LedgerRegistry):
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: Transaction, height=-1, timeout=1):
timeout = timeout or 600 # after 10 minutes there is almost 0 hope
addresses = set()
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
@@ -608,13 +650,20 @@ class Ledger(metaclass=LedgerRegistry):
for txo in tx.outputs:
if txo.has_address:
addresses.add(self.hash160_to_address(txo.pubkey_hash))
start = int(time.perf_counter())
while timeout and (int(time.perf_counter()) - start) <= timeout:
if await self._wait_round(tx, height, addresses):
return
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]):
records = await self.db.get_addresses(address__in=addresses)
_, pending = await asyncio.wait([
self.on_transaction.where(partial(
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
address_record['address']
)) for address_record in records
], timeout=timeout)
], timeout=1)
if pending:
records = await self.db.get_addresses(address__in=addresses)
for record in records:
@@ -626,10 +675,17 @@ class Ledger(metaclass=LedgerRegistry):
if txid == tx.id and local_height >= height:
found = True
if not found:
print(record['history'], addresses, tx.id)
raise asyncio.TimeoutError('Timed out waiting for transaction.')
log.debug("timeout: %s, %s, %s", record['history'], addresses, tx.id)
return False
return True
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
async def _inflate_outputs(
self, query, accounts,
include_purchase_receipt=False,
include_is_my_output=False,
include_sent_supports=False,
include_sent_tips=False,
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
@@ -637,12 +693,33 @@ class Ledger(metaclass=LedgerRegistry):
txs: List[Transaction] = await asyncio.gather(*(
self.cache_transaction(*tx) for tx in outputs.txs
))
if accounts:
_txos, blocked = outputs.inflate(txs)
txos = []
for txo in _txos:
if isinstance(txo, Output):
# transactions and outputs are cached and shared between wallets
# we don't want to leak informaion between wallet so we add the
# wallet specific metadata on throw away copies of the txos
txo = copy.copy(txo)
channel = txo.channel
txo.purchase_receipt = None
txo.update_annotations(None)
txo.channel = channel
txos.append(txo)
includes = (
include_purchase_receipt, include_is_my_output,
include_sent_supports, include_sent_tips
)
if accounts and any(includes):
receipts = {}
if include_purchase_receipt:
priced_claims = []
for tx in txs:
for txo in tx.outputs:
if txo.has_price:
priced_claims.append(txo)
for txo in txos:
if isinstance(txo, Output) and txo.has_price:
priced_claims.append(txo)
if priced_claims:
receipts = {
txo.purchased_claim_id: txo for txo in
@@ -651,14 +728,49 @@ class Ledger(metaclass=LedgerRegistry):
purchased_claim_id__in=[c.claim_id for c in priced_claims]
)
}
for txo in priced_claims:
for txo in txos:
if isinstance(txo, Output) and txo.can_decode_claim:
if include_purchase_receipt:
txo.purchase_receipt = receipts.get(txo.claim_id)
txos, blocked = outputs.inflate(txs)
if include_is_my_output:
mine = await self.db.get_txo_count(
claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True,
is_spent=False, accounts=accounts
)
if mine:
txo.is_my_output = True
else:
txo.is_my_output = False
if include_sent_supports:
supports = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=True, is_my_output=True,
is_spent=False, accounts=accounts
)
txo.sent_supports = supports
if include_sent_tips:
tips = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=True, is_my_output=False,
accounts=accounts
)
txo.sent_tips = tips
if include_received_tips:
tips = await self.db.get_txo_sum(
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
is_my_input=False, is_my_output=True,
accounts=accounts
)
txo.received_tips = tips
return txos, blocked, outputs.offset, outputs.total
async def resolve(self, accounts, urls):
async def resolve(self, accounts, urls, **kwargs):
resolve = partial(self.network.retriable_call, self.network.resolve)
txos = (await self._inflate_outputs(resolve(urls), accounts))[0]
urls_copy = list(urls)
txos = []
while urls_copy:
batch, urls_copy = urls_copy[:500], urls_copy[500:]
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
@@ -671,11 +783,17 @@ class Ledger(metaclass=LedgerRegistry):
result[url] = txo
return result
async def claim_search(self, accounts, **kwargs) -> Tuple[List[Output], dict, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs), accounts)
async def claim_search(
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
**kwargs) -> Tuple[List[Output], dict, int, int]:
return await self._inflate_outputs(
self.network.claim_search(**kwargs), accounts,
include_purchase_receipt=include_purchase_receipt,
include_is_my_output=include_is_my_output
)
async def get_claim_by_claim_id(self, accounts, claim_id) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id))[0]:
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
return claim
async def _report_state(self):
@@ -735,9 +853,11 @@ class Ledger(metaclass=LedgerRegistry):
async def _resolve_for_local_results(self, accounts, txos):
results = []
response = await self.resolve(accounts, [txo.permanent_url for txo in txos])
response = await self.resolve(
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
)
for txo in txos:
resolved = response[txo.permanent_url]
resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None
if isinstance(resolved, Output):
resolved.update_annotations(txo)
results.append(resolved)
@@ -811,12 +931,15 @@ class Ledger(metaclass=LedgerRegistry):
def get_support_count(self, **constraints):
return self.db.get_support_count(**constraints)
async def get_transaction_history(self, **constraints):
txs: List[Transaction] = await self.db.get_transactions(**constraints)
async def get_transaction_history(self, read_only=False, **constraints):
txs: List[Transaction] = await self.db.get_transactions(
include_is_my_output=True, include_is_spent=True,
read_only=read_only, **constraints
)
headers = self.headers
history = []
for tx in txs: # pylint: disable=too-many-nested-blocks
ts = headers[tx.height]['timestamp'] if tx.height > 0 else None
ts = headers.estimated_timestamp(tx.height)
item = {
'txid': tx.id,
'timestamp': ts,
@@ -828,7 +951,7 @@ class Ledger(metaclass=LedgerRegistry):
'abandon_info': [],
'purchase_info': []
}
is_my_inputs = all([txi.is_my_account for txi in tx.inputs])
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
if is_my_inputs:
# fees only matter if we are the ones paying them
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
@@ -844,7 +967,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_update_outputs:
if is_my_inputs: # updating my own claim
@@ -863,7 +987,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
else: # someone sent us their claim
item['update_info'].append({
@@ -872,7 +997,8 @@ class Ledger(metaclass=LedgerRegistry):
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_support_outputs:
item['support_info'].append({
@@ -882,7 +1008,8 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': not is_my_inputs,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
if is_my_inputs:
for txo in tx.other_support_outputs:
@@ -893,7 +1020,8 @@ class Ledger(metaclass=LedgerRegistry):
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
for txo in tx.my_abandon_outputs:
item['abandon_info'].append({
@@ -910,13 +1038,14 @@ class Ledger(metaclass=LedgerRegistry):
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.purchased_claim_id,
'nout': txo.position
'nout': txo.position,
'is_spent': txo.is_spent,
})
history.append(item)
return history
def get_transaction_history_count(self, **constraints):
return self.db.get_transaction_count(**constraints)
def get_transaction_history_count(self, read_only=False, **constraints):
return self.db.get_transaction_count(read_only=read_only, **constraints)
async def get_detailed_balance(self, accounts, confirmations=0):
result = {
@@ -949,6 +1078,7 @@ class TestNetLedger(Ledger):
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
checkpoints = {}
class RegTestLedger(Ledger):
@@ -963,3 +1093,4 @@ class RegTestLedger(Ledger):
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
genesis_bits = 0x207fffff
target_timespan = 1
checkpoints = {}
+16 -13
View File
@@ -184,6 +184,7 @@ class WalletManager:
'auto_connect': True,
'default_servers': config.lbryum_servers,
'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
}
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
@@ -248,26 +249,28 @@ class WalletManager:
log.warning("Failed to migrate %s receiving addresses!",
len(set(receiving_addresses).difference(set(migrated_receiving))))
def get_best_blockhash(self):
async def get_best_blockhash(self):
if len(self.ledger.headers) <= 0:
return self.ledger.genesis_hash
return self.ledger.headers.hash(self.ledger.headers.height).decode()
return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
def get_unused_address(self):
return self.default_account.receiving.get_or_create_usable_address()
async def get_transaction(self, txid):
async def get_transaction(self, txid: str):
tx = await self.db.get_transaction(txid=txid)
if not tx:
try:
raw = await self.ledger.network.get_transaction(txid)
height = await self.ledger.network.get_transaction_height(txid)
except CodeMessageError as e:
if 'No such mempool or blockchain transaction.' in e.message:
return {'success': False, 'code': 404, 'message': 'transaction not found'}
return {'success': False, 'code': e.code, 'message': e.message}
tx = Transaction(unhexlify(raw))
await self.ledger.maybe_verify_transaction(tx, height)
if tx:
return tx
try:
raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
except CodeMessageError as e:
if 'No such mempool or blockchain transaction.' in e.message:
return {'success': False, 'code': 404, 'message': 'transaction not found'}
return {'success': False, 'code': e.code, 'message': e.message}
height = merkle.get('block_height')
tx = Transaction(unhexlify(raw), height=height)
if height and height > 0:
await self.ledger.maybe_verify_transaction(tx, height, merkle)
return tx
async def create_purchase_transaction(
+8 -3
View File
@@ -55,7 +55,7 @@ class ClientSession(BaseClientSession):
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s to %s:%i", method, *self.server)
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
try:
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
@@ -87,7 +87,7 @@ class ClientSession(BaseClientSession):
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
# self.synchronous_close()
raise
finally:
self.pending_amount -= 1
@@ -156,7 +156,7 @@ class ClientSession(BaseClientSession):
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 59, 0)
MINIMUM_REQUIRED = (0, 65, 0)
def __init__(self, ledger):
self.ledger = ledger
@@ -256,6 +256,11 @@ class Network:
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
def get_transaction_and_merkle(self, tx_hash, known_height=None):
# use any server if its old, otherwise restrict to who gave us the history
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.info', [tx_hash], restricted)
def get_transaction_height(self, tx_hash, known_height=None):
restricted = not known_height or 0 > known_height > self.remote_height - 10
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
+41 -6
View File
@@ -55,7 +55,8 @@ class Conductor:
async def start_blockchain(self):
if not self.blockchain_started:
await self.blockchain_node.start()
asyncio.create_task(self.blockchain_node.start())
await self.blockchain_node.running.wait()
await self.blockchain_node.generate(200)
self.blockchain_started = True
@@ -255,6 +256,10 @@ class BlockchainNode:
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
self.rpcuser = 'rpcuser'
self.rpcpassword = 'rpcpassword'
self.stopped = False
self.restart_ready = asyncio.Event()
self.restart_ready.set()
self.running = asyncio.Event()
@property
def rpc_url(self):
@@ -315,13 +320,27 @@ class BlockchainNode:
f'-port={self.peerport}'
]
self.log.info(' '.join(command))
self.transport, self.protocol = await loop.subprocess_exec(
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
while not self.stopped:
if self.running.is_set():
await asyncio.sleep(1)
continue
await self.restart_ready.wait()
try:
self.transport, self.protocol = await loop.subprocess_exec(
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
self.running.set()
except asyncio.CancelledError:
self.running.clear()
raise
except Exception as e:
self.running.clear()
log.exception('failed to start lbrycrdd', exc_info=e)
async def stop(self, cleanup=True):
self.stopped = True
try:
self.transport.terminate()
await self.protocol.stopped.wait()
@@ -330,6 +349,16 @@ class BlockchainNode:
if cleanup:
self.cleanup()
async def clear_mempool(self):
self.restart_ready.clear()
self.transport.terminate()
await self.protocol.stopped.wait()
self.transport.close()
self.running.clear()
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
self.restart_ready.set()
await self.running.wait()
def cleanup(self):
shutil.rmtree(self.data_path, ignore_errors=True)
@@ -361,6 +390,12 @@ class BlockchainNode:
def get_block_hash(self, block):
return self._cli_cmnd('getblockhash', str(block))
def sendrawtransaction(self, tx):
return self._cli_cmnd('sendrawtransaction', tx)
async def get_block(self, block_hash):
return json.loads(await self._cli_cmnd('getblock', block_hash, '1'))
def get_raw_change_address(self):
return self._cli_cmnd('getrawchangeaddress')
+25 -6
View File
@@ -33,14 +33,16 @@ from asyncio import Event, CancelledError
import logging
import time
from contextlib import suppress
from prometheus_client import Counter, Histogram
from lbry.wallet.tasks import TaskGroup
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
from .jsonrpc import RPCError, ProtocolError
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
from .util import Concurrency
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class Connector:
@@ -373,10 +375,26 @@ class BatchRequest:
raise BatchError(self)
NAMESPACE = "wallet_server"
class RPCSession(SessionBase):
"""Base class for protocols where a message can lead to a response,
for example JSON RPC."""
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS)
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
namespace=NAMESPACE, labelnames=("method", "version"))
REQUEST_ERRORS_COUNT = Counter(
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
labelnames=("method", "version")
)
RESET_CONNECTIONS = Counter(
"reset_clients", "Number of reset connections by client version",
namespace=NAMESPACE, labelnames=("version",)
)
def __init__(self, *, framer=None, loop=None, connection=None):
super().__init__(framer=framer, loop=loop)
self.connection = connection or self.default_connection()
@@ -389,6 +407,7 @@ class RPCSession(SessionBase):
except MemoryError:
self.logger.warning('received oversized message from %s:%s, dropping connection',
self._address[0], self._address[1])
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
self._close()
return
@@ -422,7 +441,7 @@ class RPCSession(SessionBase):
'internal server error')
if isinstance(request, Request):
message = request.send_result(result)
RESPONSE_TIMES.labels(
self.RESPONSE_TIMES.labels(
method=request.method,
version=self.client_version
).observe(time.perf_counter() - start)
@@ -430,7 +449,7 @@ class RPCSession(SessionBase):
await self._send_message(message)
if isinstance(result, Exception):
self._bump_errors()
REQUEST_ERRORS_COUNT.labels(
self.REQUEST_ERRORS_COUNT.labels(
method=request.method,
version=self.client_version
).inc()
@@ -467,7 +486,7 @@ class RPCSession(SessionBase):
async def send_notification(self, method, args=()):
"""Send an RPC notification over the network."""
message = self.connection.send_notification(Notification(method, args))
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
await self._send_message(message)
def send_batch(self, raise_errors=False):
+24 -6
View File
@@ -2,7 +2,8 @@ import time
import asyncio
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
from typing import Optional
from prometheus_client import Gauge, Histogram
import lbry
from lbry.schema.claim import Claim
from lbry.wallet.server.db.writer import SQLDB
@@ -10,7 +11,6 @@ from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
class Prefetcher:
@@ -129,6 +129,12 @@ class ChainError(Exception):
"""Raised on error processing blocks."""
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class BlockProcessor:
"""Process blocks and update the DB state to match.
@@ -136,6 +142,16 @@ class BlockProcessor:
Coordinate backing up in case of chain reorganisations.
"""
block_count_metric = Gauge(
"block_count", "Number of processed blocks", namespace=NAMESPACE
)
block_update_time_metric = Histogram(
"block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
reorg_count_metric = Gauge(
"reorg_count", "Number of reorgs", namespace=NAMESPACE
)
def __init__(self, env, db, daemon, notifications):
self.env = env
self.db = db
@@ -199,8 +215,8 @@ class BlockProcessor:
cache.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
BLOCK_COUNT.set(self.height)
BLOCK_UPDATE_TIMES.observe(processed_time)
self.block_count_metric.set(self.height)
self.block_update_time_metric.observe(processed_time)
if not self.db.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
@@ -219,7 +235,7 @@ class BlockProcessor:
'resetting the prefetcher')
await self.prefetcher.reset_height(self.height)
async def reorg_chain(self, count=None):
async def reorg_chain(self, count: Optional[int] = None):
"""Handle a chain reorganisation.
Count is the number of blocks to simulate a reorg, or None for
@@ -253,7 +269,9 @@ class BlockProcessor:
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
await self.run_in_thread_with_lock(flush_backup)
last -= len(raw_blocks)
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
await self.prefetcher.reset_height(self.height)
self.reorg_count_metric.inc()
async def reorg_hashes(self, count):
"""Return a pair (start, last, hashes) of blocks to back up during a
@@ -270,7 +288,7 @@ class BlockProcessor:
return start, last, await self.db.fs_block_hashes(start, count)
async def calc_reorg_range(self, count):
async def calc_reorg_range(self, count: Optional[int]):
"""Calculate the reorg range"""
def diff_pos(hashes1, hashes2):
+19 -7
View File
@@ -6,11 +6,12 @@ from functools import wraps
from pylru import lrucache
import aiohttp
from prometheus_client import Gauge, Histogram
from lbry.wallet.rpc.jsonrpc import RPCError
from lbry.wallet.server.util import hex_to_bytes, class_logger
from lbry.wallet.rpc import JSONRPC
from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT
class DaemonError(Exception):
"""Raised when the daemon returns an error in its results."""
@@ -24,12 +25,23 @@ class WorkQueueFullError(Exception):
"""Internal - when the daemon's work queue is full."""
NAMESPACE = "wallet_server"
class Daemon:
"""Handles connections to a daemon at the given URL."""
WARMING_UP = -28
id_counter = itertools.count()
lbrycrd_request_time_metric = Histogram(
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
)
lbrycrd_pending_count_metric = Gauge(
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
labelnames=("method",)
)
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
max_retry=4.0):
self.coin = coin
@@ -43,8 +55,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = lrucache(1000000)
self._block_cache = lrucache(100000)
self._block_hash_cache = lrucache(100000)
self._block_cache = lrucache(10000)
async def close(self):
if self.connector:
@@ -129,7 +141,7 @@ class Daemon:
while True:
try:
for method in methods:
LBRYCRD_PENDING_COUNT.labels(method=method).inc()
self.lbrycrd_pending_count_metric.labels(method=method).inc()
result = await self._send_data(data)
result = processor(result)
if on_good_message:
@@ -154,7 +166,7 @@ class Daemon:
on_good_message = 'running normally'
finally:
for method in methods:
LBRYCRD_PENDING_COUNT.labels(method=method).dec()
self.lbrycrd_pending_count_metric.labels(method=method).dec()
await asyncio.sleep(retry)
retry = max(min(self.max_retry, retry * 2), self.init_retry)
@@ -175,7 +187,7 @@ class Daemon:
if params:
payload['params'] = params
result = await self._send(payload, processor)
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start)
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
return result
async def _send_vector(self, method, params_iterable, replace_errs=False):
@@ -200,7 +212,7 @@ class Daemon:
result = []
if payload:
result = await self._send(payload, processor)
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start)
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
return result
async def _is_rpc_available(self, method):
+26 -10
View File
@@ -545,11 +545,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
@@ -596,11 +604,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
((claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
"""
else:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
+3 -2
View File
@@ -1,8 +1,9 @@
from . import zscore
from . import ar
from . import variable_decay
TRENDING_ALGORITHMS = {
'zscore': zscore,
'ar': ar
'ar': ar,
'variable_decay': variable_decay
}
@@ -0,0 +1,431 @@
"""
Delayed AR with variable decay rate.
The spike height function is also simpler.
"""
import copy
import time
import apsw
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
HALF_LIFE = 200
# Whale threshold (higher -> less DB writing)
WHALE_THRESHOLD = 3.0
# Decay coefficient per block
DECAY = 0.5**(1.0/HALF_LIFE)
# How frequently to write trending values to the db
SAVE_INTERVAL = 10
# Renormalisation interval
RENORM_INTERVAL = 1000
# Assertion
assert RENORM_INTERVAL % SAVE_INTERVAL == 0
# Decay coefficient per renormalisation interval
DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
# Log trending calculations?
TRENDING_LOG = True
def install(connection):
"""
Install the trending algorithm.
"""
check_trending_values(connection)
if TRENDING_LOG:
f = open("trending_variable_decay.log", "a")
f.close()
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
reset them. This will allow resyncing from a standard snapshot.
"""
c = connection.cursor()
needs_reset = False
for row in c.execute("SELECT COUNT(*) num FROM claim WHERE trending_global <> 0;"):
if row[0] != 0:
needs_reset = True
break
if needs_reset:
print("Resetting some columns. This might take a while...", flush=True, end="")
c.execute(""" BEGIN;
UPDATE claim SET trending_group = 0;
UPDATE claim SET trending_mixed = 0;
UPDATE claim SET trending_global = 0;
UPDATE claim SET trending_local = 0;
COMMIT;""")
print("done.")
def spike_height(x, x_old):
"""
Compute the size of a trending spike (normed - constant units).
"""
# Sign of trending spike
sign = 1.0
if x < x_old:
sign = -1.0
# Magnitude
mag = abs(x**0.25 - x_old**0.25)
# Minnow boost
mag *= 1.0 + 2E4/(x + 100.0)**2
return sign*mag
def get_time_boost(height):
"""
Return the time boost at a given height.
"""
return 1.0/DECAY**(height % RENORM_INTERVAL)
def trending_log(s):
"""
Log a string.
"""
if TRENDING_LOG:
fout = open("trending_variable_decay.log", "a")
fout.write(s)
fout.flush()
fout.close()
class TrendingData:
"""
An object of this class holds trending data
"""
def __init__(self):
# Dict from claim id to some trending info.
# Units are TIME VARIABLE in here
self.claims = {}
# Claims with >= WHALE_THRESHOLD LBC total amount
self.whales = set([])
# Have all claims been read from db yet?
self.initialised = False
# List of pending spikes.
# Units are CONSTANT in here
self.pending_spikes = []
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
self.add_whale(claim_hash)
def add_whale(self, claim_hash):
self.whales.add(claim_hash)
def apply_spikes(self, height):
"""
Apply all pending spikes that are due at this height.
Apply with time boost ON.
"""
time_boost = get_time_boost(height)
for spike in self.pending_spikes:
if spike["height"] > height:
# Ignore
pass
if spike["height"] == height:
# Apply
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
self.claims[spike["claim_hash"]]["changed"] = True
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
self.add_whale(spike["claim_hash"])
if spike["claim_hash"] in self.whales and \
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
self.whales.remove(spike["claim_hash"])
# Keep only future spikes
self.pending_spikes = [s for s in self.pending_spikes \
if s["height"] > height]
def update_claim(self, height, claim_hash, total_amount):
"""
Update trending data for a claim, given its new total amount.
"""
assert self.initialised
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
"changed": False}
# Calculate LBC change
change = total_amount - old_state["total_amount"]
# Modify data if there was an LBC change
if change != 0.0:
spike = spike_height(total_amount,
old_state["total_amount"])
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
if change < 0.0:
# How big would the spike be for the inverse movement?
reverse_spike = spike_height(old_state["total_amount"], total_amount)
# Remove that much spike from future pending ones
for future_spike in self.pending_spikes:
if future_spike["claim_hash"] == claim_hash:
if reverse_spike >= future_spike["size"]:
reverse_spike -= future_spike["size"]
future_spike["size"] = 0.0
elif reverse_spike > 0.0:
future_spike["size"] -= reverse_spike
reverse_spike = 0.0
delay = 0
spike = -reverse_spike
self.pending_spikes.append({"height": height + delay,
"claim_hash": claim_hash,
"size": spike})
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": old_state["trending_score"],
"changed": False}
def process_whales(self, height):
"""
Whale claims decay faster.
"""
if height % SAVE_INTERVAL != 0:
return
for claim_hash in self.whales:
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
# Overall multiplication factor for decay rate
decay_rate_factor = trending_normed/WHALE_THRESHOLD
# The -1 is because this is just the *extra* part being applied
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
# print(claim_hash, trending_normed, decay_rate_factor)
self.claims[claim_hash]["trending_score"] *= factor
self.claims[claim_hash]["changed"] = True
def test_trending():
"""
Quick trending test for claims with different support patterns.
Actually use the run() function.
"""
# Create a fake "claims.db" for testing
# pylint: disable=I1101
dbc = apsw.Connection(":memory:")
db = dbc.cursor()
# Create table
db.execute("""
BEGIN;
CREATE TABLE claim (claim_hash TEXT PRIMARY KEY,
amount REAL NOT NULL DEFAULT 0.0,
support_amount REAL NOT NULL DEFAULT 0.0,
trending_mixed REAL NOT NULL DEFAULT 0.0);
COMMIT;
""")
# Insert initial states of claims
everything = {"huge_whale": 0.01,
"huge_whale_botted": 0.01,
"medium_whale": 0.01,
"small_whale": 0.01,
"minnow": 0.01}
def to_list_of_tuples(stuff):
l = []
for key in stuff:
l.append((key, stuff[key]))
return l
db.executemany("""
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
""", to_list_of_tuples(everything))
height = 0
run(db, height, height, everything.keys())
# Save trajectories for plotting
trajectories = {}
for key in trending_data.claims:
trajectories[key] = [trending_data.claims[key]["trending_score"]]
# Main loop
for height in range(1, 1000):
# One-off supports
if height == 1:
everything["huge_whale"] += 5E5
everything["medium_whale"] += 5E4
everything["small_whale"] += 5E3
# Every block
if height < 500:
everything["huge_whale_botted"] += 5E5/500
everything["minnow"] += 1
# Remove supports
if height == 500:
for key in everything:
everything[key] = 0.01
# Whack into the db
db.executemany("""
UPDATE claim SET amount = 1E8*? WHERE claim_hash = ?;
""", [(y, x) for (x, y) in to_list_of_tuples(everything)])
# Call run()
run(db, height, height, everything.keys())
for key in trending_data.claims:
trajectories[key].append(trending_data.claims[key]["trending_score"]\
/get_time_boost(height))
dbc.close()
# pylint: disable=C0415
import matplotlib.pyplot as plt
for key in trending_data.claims:
plt.plot(trajectories[key], label=key)
plt.legend()
plt.show()
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
return
start = time.time()
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
trending_log(" Length of trending data = {l}.\n"\
.format(l=len(trending_data.claims)))
# Renormalise trending scores and mark all as having changed
if height % RENORM_INTERVAL == 0:
trending_log(" Renormalising trending scores...")
keys = trending_data.claims.keys()
trending_data.whales = set([])
for key in keys:
if trending_data.claims[key]["trending_score"] != 0.0:
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
trending_data.claims[key]["changed"] = True
# Tiny becomes zero
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
trending_data.claims[key]["trending_score"] = 0.0
# Re-mark whales
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
trending_data.add_whale(key)
trending_log("done.\n")
# Regular message.
trending_log(" Reading total_amounts from db and updating"\
+ " trending scores in RAM...")
# Update claims from db
if not trending_data.initialised:
trending_log("initial load...")
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
"""):
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
(amount + support_amount)
AS total_amount
FROM claim
WHERE claim_hash IN
({','.join('?' for _ in recalculate_claim_hashes)});
""", recalculate_claim_hashes):
trending_data.update_claim(height, row[0], 1E-8*row[1])
# Apply pending spikes
trending_data.apply_spikes(height)
trending_log("done.\n")
# Write trending scores to DB
if height % SAVE_INTERVAL == 0:
trending_log(" Finding and processing whales...")
trending_log(str(len(trending_data.whales)) + " whales found...")
trending_data.process_whales(height)
trending_log("done.\n")
trending_log(" Writing trending scores to db...")
the_list = []
keys = trending_data.claims.keys()
for key in keys:
if trending_data.claims[key]["changed"]:
the_list.append((trending_data.claims[key]["trending_score"], key))
trending_data.claims[key]["changed"] = False
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
the_list)
trending_log("done.\n")
trending_log("Trending operations took {time} seconds.\n\n"\
.format(time=time.time() - start))
if __name__ == "__main__":
test_trending()
+9
View File
@@ -433,6 +433,15 @@ class SQLDB:
return {r.channel_hash for r in affected_channels}
return set()
def delete_claims_above_height(self, height: int):
claim_hashes = [x[0] for x in self.execute(
"SELECT claim_hash FROM claim WHERE height>?", (height, )
).fetchall()]
while claim_hashes:
batch = set(claim_hashes[:500])
claim_hashes = claim_hashes[500:]
self.delete_claims(batch)
def _clear_claim_metadata(self, claim_hashes: Set[bytes]):
if claim_hashes:
for table in ('tag',): # 'language', 'location', etc
-82
View File
@@ -1,82 +0,0 @@
import os
from aiohttp import web
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
from lbry import __version__ as version
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server import util
import lbry.wallet.server.version as wallet_server_version
NAMESPACE = "wallet_server"
CPU_COUNT = f"{os.cpu_count()}"
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
VERSION_INFO.info({
'build': BUILD,
"commit": COMMIT_HASH,
"docker_tag": DOCKER_TAG,
'version': version,
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
"cpu_count": CPU_COUNT
})
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version", ))
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
namespace=NAMESPACE, labelnames=("method", "version"))
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
labelnames=("method", "version"))
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
)
SQLITE_INTERNAL_ERROR_COUNT = Counter(
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
)
SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE)
SQLITE_PENDING_COUNT = Gauge(
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
)
LBRYCRD_REQUEST_TIMES = Histogram(
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
)
LBRYCRD_PENDING_COUNT = Gauge(
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, labelnames=("method",)
)
CLIENT_VERSIONS = Counter(
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
BLOCK_COUNT = Gauge(
"block_count", "Number of processed blocks", namespace=NAMESPACE
)
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
class PrometheusServer:
def __init__(self):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.runner = None
async def start(self, port: int):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
await self.runner.cleanup()
+2 -2
View File
@@ -6,7 +6,7 @@ import typing
import lbry
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
from lbry.wallet.server.prometheus import PrometheusServer
from lbry.prometheus import PrometheusServer
class Notifications:
@@ -143,4 +143,4 @@ class Server:
async def start_prometheus(self):
if not self.prometheus_server and self.env.prometheus_port:
self.prometheus_server = PrometheusServer()
await self.prometheus_server.start(self.env.prometheus_port)
await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port)
+108 -16
View File
@@ -20,16 +20,15 @@ from functools import partial
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
from lbry.wallet.rpc.framing import NewlineFramer
import lbry.wallet.server.version as VERSION
@@ -119,9 +118,49 @@ class SessionGroup:
self.semaphore = asyncio.Semaphore(20)
NAMESPACE = "wallet_server"
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
)
class SessionManager:
"""Holds global state about all sessions."""
version_info_metric = Info(
'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE
)
version_info_metric.info({
'build': BUILD,
"commit": COMMIT_HASH,
"docker_tag": DOCKER_TAG,
'version': lbry.__version__,
"min_version": util.version_string(VERSION.PROTOCOL_MIN),
"cpu_count": os.cpu_count()
})
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
labelnames=("version",))
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
labelnames=("method", "version"))
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
db_operational_error_metric = Counter(
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
)
db_error_metric = Counter(
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
)
executor_time_metric = Histogram(
"executor_time", "SQLite executor times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
)
pending_query_metric = Gauge(
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
)
client_version_metric = Counter(
"clients", "Number of connections received per client version",
namespace=NAMESPACE, labelnames=("version",)
)
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
shutdown_event: asyncio.Event):
env.max_send = max(350000, env.max_send)
@@ -677,7 +716,7 @@ class SessionBase(RPCSession):
context = {'conn_id': f'{self.session_id}'}
self.logger = util.ConnectionLogger(self.logger, context)
self.group = self.session_mgr.add_session(self)
SESSIONS_COUNT.labels(version=self.client_version).inc()
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
peer_addr_str = self.peer_address_str()
self.logger.info(f'{self.kind} {peer_addr_str}, '
f'{self.session_mgr.session_count():,d} total')
@@ -686,7 +725,7 @@ class SessionBase(RPCSession):
"""Handle client disconnection."""
super().connection_lost(exc)
self.session_mgr.remove_session(self)
SESSIONS_COUNT.labels(version=self.client_version).dec()
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
msg = ''
if not self._can_send.is_set():
msg += ' whilst paused'
@@ -710,7 +749,7 @@ class SessionBase(RPCSession):
"""Handle an incoming request. ElectrumX doesn't receive
notifications from client sessions.
"""
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
self.session_mgr.request_count_metric.labels(method=request.method, version=self.client_version).inc()
if isinstance(request, Request):
handler = self.request_handlers.get(request.method)
handler = partial(handler, self)
@@ -801,6 +840,8 @@ class LBRYElectrumX(SessionBase):
'blockchain.scripthash.subscribe': cls.scripthash_subscribe,
'blockchain.transaction.broadcast': cls.transaction_broadcast,
'blockchain.transaction.get': cls.transaction_get,
'blockchain.transaction.get_batch': cls.transaction_get_batch,
'blockchain.transaction.info': cls.transaction_info,
'blockchain.transaction.get_merkle': cls.transaction_merkle,
'server.add_peer': cls.add_peer,
'server.banner': cls.banner,
@@ -944,7 +985,7 @@ class LBRYElectrumX(SessionBase):
async def run_in_executor(self, query_name, func, kwargs):
start = time.perf_counter()
try:
SQLITE_PENDING_COUNT.inc()
self.session_mgr.pending_query_metric.inc()
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
@@ -953,18 +994,18 @@ class LBRYElectrumX(SessionBase):
except reader.SQLiteInterruptedError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_interrupt(start, error.metrics)
SQLITE_INTERRUPT_COUNT.inc()
self.session_mgr.interrupt_count_metric.inc()
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, error.metrics)
SQLITE_OPERATIONAL_ERROR_COUNT.inc()
self.session_mgr.db_operational_error_metric.inc()
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except Exception:
log.exception("dear devs, please handle this exception better")
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, {})
SQLITE_INTERNAL_ERROR_COUNT.inc()
self.session_mgr.db_error_metric.inc()
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
else:
if self.env.track_metrics:
@@ -973,8 +1014,8 @@ class LBRYElectrumX(SessionBase):
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
finally:
SQLITE_PENDING_COUNT.dec()
SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start)
self.session_mgr.pending_query_metric.dec()
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
async def run_and_cache_query(self, query_name, function, kwargs):
metrics = self.get_metrics_or_placeholder_for_api(query_name)
@@ -1179,7 +1220,10 @@ class LBRYElectrumX(SessionBase):
return await self.address_status(hashX)
async def hashX_unsubscribe(self, hashX, alias):
del self.hashX_subs[hashX]
try:
del self.hashX_subs[hashX]
except ValueError:
pass
def address_to_hashX(self, address):
try:
@@ -1441,10 +1485,10 @@ class LBRYElectrumX(SessionBase):
raise RPCError(BAD_REQUEST,
f'unsupported client: {client_name}')
if self.client_version != client_name[:17]:
SESSIONS_COUNT.labels(version=self.client_version).dec()
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
self.client_version = client_name[:17]
SESSIONS_COUNT.labels(version=self.client_version).inc()
CLIENT_VERSIONS.labels(version=self.client_version).inc()
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
self.session_mgr.client_version_metric.labels(version=self.client_version).inc()
# Find the highest common protocol version. Disconnect if
# that protocol version in unsupported.
@@ -1479,6 +1523,54 @@ class LBRYElectrumX(SessionBase):
raise RPCError(BAD_REQUEST, 'the transaction was rejected by '
f'network rules.\n\n{message}\n[{raw_tx}]')
async def transaction_info(self, tx_hash: str):
assert_tx_hash(tx_hash)
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
block_hash = tx_info.get('blockhash')
if not block_hash:
return raw_tx, {'block_height': -1}
merkle_height = (await self.daemon_request('deserialised_block', block_hash))['height']
merkle = await self.transaction_merkle(tx_hash, merkle_height)
return raw_tx, merkle
async def transaction_get_batch(self, *tx_hashes):
if len(tx_hashes) > 100:
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
for tx_hash in tx_hashes:
assert_tx_hash(tx_hash)
batch_result = {}
height = None
block_hash = None
block = None
for tx_hash in tx_hashes:
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
raw_tx = tx_info['hex']
if height is None:
if 'blockhash' in tx_info:
block_hash = tx_info['blockhash']
block = await self.daemon_request('deserialised_block', block_hash)
height = block['height']
else:
height = -1
if block_hash != tx_info.get('blockhash'):
raise RPCError(BAD_REQUEST, f'request contains a mix of transaction heights')
else:
if not block_hash:
merkle = {'block_height': -1}
else:
try:
pos = block['tx'].index(tx_hash)
except ValueError:
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
merkle = {
"merkle": self._get_merkle_branch(block['tx'], pos),
"pos": pos
}
batch_result[tx_hash] = [raw_tx, merkle]
return batch_result
async def transaction_get(self, tx_hash, verbose=False):
"""Return the serialized raw transaction given its hash
+7
View File
@@ -7,10 +7,15 @@ class TaskGroup:
self._loop = loop or get_event_loop()
self._tasks = set()
self.done = Event()
self.started = Event()
def __len__(self):
return len(self._tasks)
def add(self, coro):
task = self._loop.create_task(coro)
self._tasks.add(task)
self.started.set()
self.done.clear()
task.add_done_callback(self._remove)
return task
@@ -19,8 +24,10 @@ class TaskGroup:
self._tasks.remove(task)
if len(self._tasks) < 1:
self.done.set()
self.started.clear()
def cancel(self):
for task in self._tasks:
task.cancel()
self.done.set()
self.started.clear()
+58 -31
View File
@@ -2,6 +2,7 @@ import struct
import hashlib
import logging
import typing
import asyncio
from binascii import hexlify, unhexlify
from typing import List, Iterable, Optional, Tuple
@@ -159,11 +160,11 @@ class Input(InputOutput):
return self.txo_ref.txo.amount
@property
def is_my_account(self) -> Optional[bool]:
def is_my_input(self) -> Optional[bool]:
""" True if the output this input spends is yours. """
if self.txo_ref.txo is None:
return False
return self.txo_ref.txo.is_my_account
return self.txo_ref.txo.is_my_output
@classmethod
def deserialize_from(cls, stream):
@@ -207,22 +208,30 @@ class OutputEffectiveAmountEstimator:
class Output(InputOutput):
__slots__ = (
'amount', 'script', 'is_change', 'is_my_account',
'channel', 'private_key', 'meta',
'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input',
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
'purchase', 'purchased_claim', 'purchase_receipt',
'reposted_claim', 'claims',
)
def __init__(self, amount: int, script: OutputScript,
tx_ref: TXRef = None, position: int = None,
is_change: Optional[bool] = None, is_my_account: Optional[bool] = None,
is_internal_transfer: Optional[bool] = None, is_spent: Optional[bool] = None,
is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None,
sent_supports: Optional[int] = None, sent_tips: Optional[int] = None,
received_tips: Optional[int] = None,
channel: Optional['Output'] = None, private_key: Optional[str] = None
) -> None:
super().__init__(tx_ref, position)
self.amount = amount
self.script = script
self.is_change = is_change
self.is_my_account = is_my_account
self.is_internal_transfer = is_internal_transfer
self.is_spent = is_spent
self.is_my_output = is_my_output
self.is_my_input = is_my_input
self.sent_supports = sent_supports
self.sent_tips = sent_tips
self.received_tips = received_tips
self.channel = channel
self.private_key = private_key
self.purchase: 'Output' = None # txo containing purchase metadata
@@ -232,13 +241,23 @@ class Output(InputOutput):
self.claims: List['Output'] = None # resolved claims for collection
self.meta = {}
def update_annotations(self, annotated):
def update_annotations(self, annotated: 'Output'):
if annotated is None:
self.is_change = False
self.is_my_account = False
self.is_internal_transfer = None
self.is_spent = None
self.is_my_output = None
self.is_my_input = None
self.sent_supports = None
self.sent_tips = None
self.received_tips = None
else:
self.is_change = annotated.is_change
self.is_my_account = annotated.is_my_account
self.is_internal_transfer = annotated.is_internal_transfer
self.is_spent = annotated.is_spent
self.is_my_output = annotated.is_my_output
self.is_my_input = annotated.is_my_input
self.sent_supports = annotated.sent_supports
self.sent_tips = annotated.sent_tips
self.received_tips = annotated.received_tips
self.channel = annotated.channel if annotated else None
self.private_key = annotated.private_key if annotated else None
@@ -394,8 +413,10 @@ class Output(InputOutput):
self.channel = None
self.claim.clear_signature()
def generate_channel_private_key(self):
self.private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
async def generate_channel_private_key(self):
self.private_key = await asyncio.get_event_loop().run_in_executor(
None, ecdsa.SigningKey.generate, ecdsa.SECP256k1, None, hashlib.sha256
)
self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der()
self.script.generate()
return self.private_key
@@ -476,7 +497,7 @@ class Output(InputOutput):
class Transaction:
def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
height: int = -2, position: int = -1) -> None:
height: int = -2, position: int = -1, julian_day: int = None) -> None:
self._raw = raw
self._raw_sans_segwit = None
self.is_segwit_flag = 0
@@ -494,6 +515,7 @@ class Transaction:
# +num: confirmed in a specific block (height)
self.height = height
self.position = position
self._day = julian_day
if raw is not None:
self._deserialize()
@@ -517,6 +539,11 @@ class Transaction:
def hash(self):
return self.ref.hash
def get_julian_day(self, ledger):
if self._day is None and self.height > 0:
self._day = ledger.headers.estimated_julian_day(self.height)
return self._day
@property
def raw(self):
if self._raw is None:
@@ -587,21 +614,21 @@ class Transaction:
for txi in self.inputs:
if txi.txo_ref.txo is None:
continue
if txi.is_my_account is None:
raise ValueError(
"Cannot access net_account_balance if inputs/outputs do not "
"have is_my_account set properly."
)
if txi.is_my_account:
if txi.is_my_input is True:
balance -= txi.amount
for txo in self.outputs:
if txo.is_my_account is None:
elif txi.is_my_input is None:
raise ValueError(
"Cannot access net_account_balance if inputs/outputs do not "
"have is_my_account set properly."
"Cannot access net_account_balance if inputs do not "
"have is_my_input set properly."
)
if txo.is_my_account:
for txo in self.outputs:
if txo.is_my_output is True:
balance += txo.amount
elif txo.is_my_output is None:
raise ValueError(
"Cannot access net_account_balance if outputs do not "
"have is_my_output set properly."
)
return balance
@property
@@ -746,7 +773,7 @@ class Transaction:
change_hash160 = change_account.ledger.address_to_hash160(change_address)
change_amount = change - cost_of_change
change_output = Output.pay_pubkey_hash(change_amount, change_hash160)
change_output.is_change = True
change_output.is_internal_transfer = True
tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)])
if tx._outputs:
@@ -851,17 +878,17 @@ class Transaction:
@property
def my_inputs(self):
for txi in self.inputs:
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_account:
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output:
yield txi
def _filter_my_outputs(self, f):
for txo in self.outputs:
if txo.is_my_account and f(txo.script):
if txo.is_my_output and f(txo.script):
yield txo
def _filter_other_outputs(self, f):
for txo in self.outputs:
if not txo.is_my_account and f(txo.script):
if not txo.is_my_output and f(txo.script):
yield txo
def _filter_any_outputs(self, f):
@@ -893,7 +920,7 @@ class Transaction:
def my_abandon_outputs(self):
for txi in self.inputs:
abandon = txi.txo_ref.txo
if abandon is not None and abandon.is_my_account and abandon.script.is_claim_involved:
if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved:
is_update = False
if abandon.script.is_claim_name or abandon.script.is_update_claim:
for update in self.my_update_outputs:
+4
View File
@@ -3,6 +3,10 @@ from typing import TypeVar, Sequence, Optional
from .constants import COIN
def date_to_julian_day(d):
return d.toordinal() + 1721424.5
def coins_to_satoshis(coins):
if not isinstance(coins, str):
raise ValueError("{coins} must be a string")
+2 -5
View File
@@ -17,7 +17,7 @@ def enable_logging():
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)
@@ -33,7 +33,7 @@ async def process_video(analyzer, video_file):
transcode = input("Would you like to make a repaired clone now? [y/N] ")
if transcode == "y":
try:
new_video_file = await analyzer.verify_or_repair(True, True, video_file)
new_video_file, _ = await analyzer.verify_or_repair(True, True, video_file)
print("Successfully created ", new_video_file)
except Exception as e:
print("Unable to complete the transcode. Message: ", str(e))
@@ -49,9 +49,6 @@ def main():
video_file = sys.argv[1]
conf = TranscodeConfig()
analyzer = VideoFileAnalyzer(conf)
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
try:
asyncio.run(process_video(analyzer, video_file))
except KeyboardInterrupt:
+35
View File
@@ -0,0 +1,35 @@
import asyncio
import os
from lbry.extras.cli import ensure_directory_exists
from lbry.conf import Config
from lbry.wallet.header import Headers
import lbry.wallet.checkpoints
async def main():
outpath = lbry.wallet.checkpoints.__file__
ledger_path = os.path.join(Config().wallet_dir, 'lbc_mainnet')
ensure_directory_exists(ledger_path)
headers_path = os.path.join(ledger_path, 'headers')
headers = Headers(headers_path)
await headers.open()
print(f"Working on headers at {outpath}")
print("Verifying integrity, might take a while.")
await headers.repair()
target = ((headers.height - 100) // 1000) * 1000
current_checkpoint_tip = max(lbry.wallet.checkpoints.HASHES.keys())
if target <= current_checkpoint_tip:
print(f"We have nothing to add: Local: {target}, checkpoint: {current_checkpoint_tip}")
return
print(f"Headers file at {headers.height}, checkpointing up to {target}."
f"Current checkpoint at {current_checkpoint_tip}.")
with open(outpath, 'w') as outfile:
print('HASHES = {', file=outfile)
for height in range(0, target, 1000):
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
print('}', file=outfile)
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(main())
+1
View File
@@ -3,6 +3,7 @@ import os
import asyncio
import socket
import ipaddress
import lbry.wallet
from lbry.conf import Config
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.blob.blob_manager import BlobManager
+11 -5
View File
@@ -59,7 +59,8 @@ def get_label(pr, prefix):
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
RELEASE_TEXT = "release-text:"
RELEASE_TEXT = 'release-text:'
RELEASE_TEXT_LINES = 'release-text-lines:'
def get_backwards_incompatible(desc: str):
@@ -69,9 +70,15 @@ def get_backwards_incompatible(desc: str):
def get_release_text(desc: str):
in_release_lines = False
for line in desc.splitlines():
if line.startswith(RELEASE_TEXT):
yield line[len(RELEASE_TEXT):]
if in_release_lines:
yield line.rstrip()
elif line.startswith(RELEASE_TEXT_LINES):
in_release_lines = True
elif line.startswith(RELEASE_TEXT):
yield line[len(RELEASE_TEXT):].strip()
yield ''
class Version:
@@ -149,7 +156,7 @@ def release(args):
for incompat in get_backwards_incompatible(pr.body or ""):
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
for release_text in get_release_text(pr.body or ""):
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
release_texts.append(release_text)
if type_label == 'fixup':
fixups.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
else:
@@ -169,7 +176,6 @@ def release(args):
w('')
for release_text in release_texts:
w(release_text)
w('')
if incompats:
w('')
w(f'### Backwards Incompatible Changes')
+2 -1
View File
@@ -6,7 +6,7 @@ source =
lbry
.tox/*/lib/python*/site-packages/lbry
[cryptography.*,coincurve.*,pbkdf2]
[cryptography.*,coincurve.*,pbkdf2, libtorrent]
ignore_missing_imports = True
[pylint]
@@ -18,6 +18,7 @@ max-line-length=120
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l
valid-metaclass-classmethod-first-arg=mcs
disable=
c-extension-no-member,
fixme,
broad-except,
no-else-return,
@@ -1,33 +1,120 @@
import logging
from lbry.testcase import IntegrationTestCase
import asyncio
from binascii import hexlify
from lbry.testcase import CommandTestCase
class BlockchainReorganizationTests(IntegrationTestCase):
class BlockchainReorganizationTests(CommandTestCase):
VERBOSITY = logging.WARN
async def assertBlockHash(self, height):
self.assertEqual(
self.ledger.headers.hash(height).decode(),
(await self.ledger.headers.hash(height)).decode(),
await self.blockchain.get_block_hash(height)
)
async def test_reorg(self):
bp = self.conductor.spv_node.server.bp
bp.reorg_count_metric.set(0)
# invalidate current block, move forward 2
self.assertEqual(self.ledger.headers.height, 200)
await self.assertBlockHash(200)
await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode())
self.assertEqual(self.ledger.headers.height, 206)
await self.assertBlockHash(206)
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.generate(2)
await self.ledger.on_header.where(lambda e: e.height == 201)
self.assertEqual(self.ledger.headers.height, 201)
await self.assertBlockHash(200)
await self.assertBlockHash(201)
await self.ledger.on_header.where(lambda e: e.height == 207)
self.assertEqual(self.ledger.headers.height, 207)
await self.assertBlockHash(206)
await self.assertBlockHash(207)
self.assertEqual(1, bp.reorg_count_metric._samples()[0][2])
# invalidate current block, move forward 3
await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode())
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
await self.blockchain.generate(3)
await self.ledger.on_header.where(lambda e: e.height == 202)
self.assertEqual(self.ledger.headers.height, 202)
await self.assertBlockHash(200)
await self.assertBlockHash(201)
await self.assertBlockHash(202)
await self.ledger.on_header.where(lambda e: e.height == 208)
self.assertEqual(self.ledger.headers.height, 208)
await self.assertBlockHash(206)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
async def test_reorg_change_claim_height(self):
# sanity check
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertListEqual(txos, [])
still_valid = await self.daemon.jsonrpc_stream_create(
'still-valid', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
await self.ledger.wait(still_valid)
await self.generate(1)
# create a claim and verify it's returned by claim_search
self.assertEqual(self.ledger.headers.height, 207)
broadcast_tx = await self.daemon.jsonrpc_stream_create(
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
)
await self.ledger.wait(broadcast_tx)
await self.generate(1)
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
self.assertEqual(self.ledger.headers.height, 208)
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
txo = txos[0]
self.assertEqual(txo.tx_ref.id, broadcast_tx.id)
self.assertEqual(txo.tx_ref.height, 208)
# check that our tx is in block 208 as returned by lbrycrdd
invalidated_block_hash = (await self.ledger.headers.hash(208)).decode()
block_207 = await self.blockchain.get_block(invalidated_block_hash)
self.assertIn(txo.tx_ref.id, block_207['tx'])
self.assertEqual(208, txos[0].tx_ref.height)
# reorg the last block dropping our claim tx
await self.blockchain.invalidate_block(invalidated_block_hash)
await self.blockchain.clear_mempool()
await self.blockchain.generate(2)
# verify the claim was dropped from block 208 as returned by lbrycrdd
reorg_block_hash = await self.blockchain.get_block_hash(208)
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
block_207 = await self.blockchain.get_block(reorg_block_hash)
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
# wait for the client to catch up and verify the reorg
await asyncio.wait_for(self.on_header(209), 3.0)
await self.assertBlockHash(207)
await self.assertBlockHash(208)
await self.assertBlockHash(209)
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
# verify the dropped claim is no longer returned by claim search
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertListEqual(txos, [])
# verify the claim published a block earlier wasn't also reverted
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
self.assertEqual(1, len(txos))
self.assertEqual(207, txos[0].tx_ref.height)
# broadcast the claim in a different block
new_txid = await self.blockchain.sendrawtransaction(hexlify(broadcast_tx.raw).decode())
self.assertEqual(broadcast_tx.id, new_txid)
await self.blockchain.generate(1)
# wait for the client to catch up
await asyncio.wait_for(self.on_header(210), 1.0)
# verify the claim is in the new block and that it is returned by claim_search
block_210 = await self.blockchain.get_block((await self.ledger.headers.hash(210)).decode())
self.assertIn(txo.tx_ref.id, block_210['tx'])
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
self.assertEqual(1, len(txos))
self.assertEqual(txos[0].tx_ref.id, new_txid)
self.assertEqual(210, txos[0].tx_ref.height)
# this should still be unchanged
txos, _, _, _ = await self.ledger.claim_search([], name='still-valid')
self.assertEqual(1, len(txos))
self.assertEqual(207, txos[0].tx_ref.height)
@@ -1,7 +1,7 @@
import asyncio
import os.path
import tempfile
import logging
import asyncio
from binascii import unhexlify
from urllib.request import urlopen
@@ -10,6 +10,7 @@ from lbry.error import InsufficientFundsError
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
from lbry.testcase import CommandTestCase
from lbry.wallet.transaction import Transaction
from lbry.wallet.util import satoshis_to_coins as lbc
log = logging.getLogger(__name__)
@@ -79,6 +80,12 @@ class ClaimSearchCommand(ClaimTestCase):
] * 23828
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
# this should do nothing... if the resolve (which is retried) results in the server disconnecting,
# it kerplodes
await asyncio.wait_for(self.daemon.jsonrpc_resolve([
f'0000000000000000000000000000000000000000{i}' for i in range(30000)
]), 30)
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
@@ -399,17 +406,270 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([], text='cloud')
class TransactionCommands(ClaimTestCase):
async def test_transaction_list(self):
channel_id = self.get_claim_id(await self.channel_create())
await self.channel_update(channel_id, bid='0.5')
await self.channel_abandon(claim_id=channel_id)
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, bid='0.5')
await self.stream_abandon(claim_id=stream_id)
r = await self.transaction_list()
self.assertEqual(7, len(r))
self.assertEqual(stream_id, r[0]['abandon_info'][0]['claim_id'])
self.assertEqual(stream_id, r[1]['update_info'][0]['claim_id'])
self.assertEqual(stream_id, r[2]['claim_info'][0]['claim_id'])
self.assertEqual(channel_id, r[3]['abandon_info'][0]['claim_id'])
self.assertEqual(channel_id, r[4]['update_info'][0]['claim_id'])
self.assertEqual(channel_id, r[5]['claim_info'][0]['claim_id'])
class TransactionOutputCommands(ClaimTestCase):
async def test_txo_list_and_sum_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
await self.channel_update(channel_id, bid='0.5')
self.assertEqual('0.5', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
self.assertEqual('1.5', lbc(await self.txo_sum(type='channel')))
stream_id = self.get_claim_id(await self.stream_create(bid='1.3'))
self.assertEqual('1.3', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
await self.stream_update(stream_id, bid='0.7')
self.assertEqual('0.7', lbc(await self.txo_sum(type='stream', is_not_spent=True)))
self.assertEqual('2.0', lbc(await self.txo_sum(type='stream')))
self.assertEqual('1.2', lbc(await self.txo_sum(type=['stream', 'channel'], is_not_spent=True)))
self.assertEqual('3.5', lbc(await self.txo_sum(type=['stream', 'channel'])))
# type filtering
r = await self.txo_list(type='channel')
self.assertEqual(2, len(r))
self.assertEqual('channel', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('channel', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type='stream')
self.assertEqual(2, len(r))
self.assertEqual('stream', r[0]['value_type'])
self.assertFalse(r[0]['is_spent'])
self.assertEqual('stream', r[1]['value_type'])
self.assertTrue(r[1]['is_spent'])
r = await self.txo_list(type=['stream', 'channel'])
self.assertEqual(4, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(claim_id=[stream_id, channel_id])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, _, channel_name, _ = (c['name'] for c in r)
r = await self.txo_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.txo_list(name=stream_name)
self.assertEqual(2, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=[stream_name, channel_name])
self.assertEqual(4, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.txo_list(name=['beef'])
self.assertEqual(0, len(r))
r = await self.txo_list()
self.assertEqual(9, len(r))
await self.stream_abandon(claim_id=stream_id)
r = await self.txo_list()
self.assertEqual(10, len(r))
r = await self.txo_list(claim_id=stream_id)
self.assertEqual(2, len(r))
self.assertTrue(r[0]['is_spent'])
self.assertTrue(r[1]['is_spent'])
async def test_txo_list_my_input_output_filtering(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create('@kept-channel')
await self.channel_create('@sent-channel', claim_address=address2)
await self.wallet_send('2.9', address2)
# all txos on second wallet
received_payment, received_channel = await self.txo_list(
wallet_id=wallet2.id, is_my_input_or_output=True)
self.assertEqual('1.0', received_channel['amount'])
self.assertFalse(received_channel['is_my_input'])
self.assertTrue(received_channel['is_my_output'])
self.assertFalse(received_channel['is_internal_transfer'])
self.assertEqual('2.9', received_payment['amount'])
self.assertFalse(received_payment['is_my_input'])
self.assertTrue(received_payment['is_my_output'])
self.assertFalse(received_payment['is_internal_transfer'])
# all txos on default wallet
r = await self.txo_list(is_my_input_or_output=True)
self.assertEqual(
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
[t['amount'] for t in r]
)
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_payment['is_my_input'])
self.assertFalse(sent_payment['is_my_output'])
self.assertFalse(sent_payment['is_internal_transfer'])
self.assertTrue(change3['is_my_input'])
self.assertTrue(change3['is_my_output'])
self.assertTrue(change3['is_internal_transfer'])
self.assertTrue(sent_channel['is_my_input'])
self.assertFalse(sent_channel['is_my_output'])
self.assertFalse(sent_channel['is_internal_transfer'])
self.assertTrue(change2['is_my_input'])
self.assertTrue(change2['is_my_output'])
self.assertTrue(change2['is_internal_transfer'])
self.assertTrue(kept_channel['is_my_input'])
self.assertTrue(kept_channel['is_my_output'])
self.assertFalse(kept_channel['is_internal_transfer'])
self.assertTrue(change1['is_my_input'])
self.assertTrue(change1['is_my_output'])
self.assertTrue(change1['is_internal_transfer'])
self.assertFalse(initial_funds['is_my_input'])
self.assertTrue(initial_funds['is_my_output'])
self.assertFalse(initial_funds['is_internal_transfer'])
# my stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
# my unspent stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
# only "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
self.assertEqual([change3, change2, change1], r)
# only unspent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
self.assertEqual([change3], r)
# only spent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
self.assertEqual([change2, change1], r)
# all my unspent stuff
r = await self.txo_list(is_my_output=True, is_not_spent=True)
self.assertEqual([change3, kept_channel], r)
# stuff i sent
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_payment, sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
stream_id = self.get_claim_id(await self.stream_create())
await self.support_create(stream_id, '0.3')
await self.support_create(stream_id, '0.2')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.4')
await self.support_create(stream_id, '0.5')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.6')
plot = await self.txo_plot(type='support')
self.assertEqual([
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=1)
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
async def test_txo_spend(self):
stream_id = self.get_claim_id(await self.stream_create())
for _ in range(10):
await self.support_create(stream_id, '0.1')
await self.assertBalance(self.account, '7.978478')
self.assertEqual('1.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
txs = await self.txo_spend(type='support', batch_size=3, include_full_tx=True)
self.assertEqual(4, len(txs))
self.assertEqual(3, len(txs[0]['inputs']))
self.assertEqual(3, len(txs[1]['inputs']))
self.assertEqual(3, len(txs[2]['inputs']))
self.assertEqual(1, len(txs[3]['inputs']))
self.assertEqual('0.0', lbc(await self.txo_sum(type='support', is_not_spent=True)))
await self.assertBalance(self.account, '8.977606')
await self.support_create(stream_id, '0.1')
txs = await self.daemon.jsonrpc_txo_spend(type='support', batch_size=3)
self.assertEqual(1, len(txs))
self.assertEqual({'txid'}, set(txs[0]))
class ClaimCommands(ClaimTestCase):
async def test_claim_list_type_filtering(self):
await self.channel_create()
await self.stream_create()
async def test_claim_list_filtering(self):
channel_id = self.get_claim_id(await self.channel_create())
stream_id = self.get_claim_id(await self.stream_create())
await self.stream_update(stream_id, title='foo')
# type filtering
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
r = await self.claim_list(claim_type='stream')
# catch a bug where cli sends is_spent=False by default
r = await self.claim_list(claim_type='stream', is_spent=False)
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
@@ -417,6 +677,31 @@ class ClaimCommands(ClaimTestCase):
self.assertEqual(2, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
# claim_id filtering
r = await self.claim_list(claim_id=stream_id)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(claim_id=[stream_id, channel_id])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
stream_name, channel_name = (c['name'] for c in r)
r = await self.claim_list(claim_id=['beef'])
self.assertEqual(0, len(r))
# claim_name filtering
r = await self.claim_list(name=stream_name)
self.assertEqual(1, len(r))
self.assertEqual({stream_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=[stream_name, channel_name])
self.assertEqual(2, len(r))
self.assertEqual({stream_id, channel_id}, {c['claim_id'] for c in r})
r = await self.claim_list(name=['beef'])
self.assertEqual(0, len(r))
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
@@ -489,7 +774,61 @@ class ClaimCommands(ClaimTestCase):
self.assertTrue(r[1]['meta']['is_controlling'])
# check that metadata is transfered
self.assertTrue(r[0]['is_mine'])
self.assertTrue(r[0]['is_my_output'])
async def assertClaimList(self, claim_ids, **kwargs):
self.assertEqual(claim_ids, [c['claim_id'] for c in await self.claim_list(**kwargs)])
async def test_list_streams_in_channel_and_order_by(self):
channel1_id = self.get_claim_id(await self.channel_create('@chan-one'))
channel2_id = self.get_claim_id(await self.channel_create('@chan-two'))
stream1_id = self.get_claim_id(await self.stream_create('stream-a', bid='0.3', channel_id=channel1_id))
stream2_id = self.get_claim_id(await self.stream_create('stream-b', bid='0.9', channel_id=channel1_id))
stream3_id = self.get_claim_id(await self.stream_create('stream-c', bid='0.6', channel_id=channel2_id))
await self.assertClaimList([stream2_id, stream1_id], channel_id=channel1_id)
await self.assertClaimList([stream3_id], channel_id=channel2_id)
await self.assertClaimList([stream3_id, stream2_id, stream1_id], channel_id=[channel1_id, channel2_id])
await self.assertClaimList([stream1_id, stream2_id, stream3_id], claim_type='stream', order_by='name')
await self.assertClaimList([stream1_id, stream3_id, stream2_id], claim_type='stream', order_by='amount')
await self.assertClaimList([stream3_id, stream2_id, stream1_id], claim_type='stream', order_by='height')
async def test_claim_list_with_tips(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.wallet_send('5.0', address2)
stream1_id = self.get_claim_id(await self.stream_create('one'))
stream2_id = self.get_claim_id(await self.stream_create('two'))
claims = await self.claim_list()
self.assertNotIn('received_tips', claims[0])
self.assertNotIn('received_tips', claims[1])
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('0.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
await self.support_create(stream1_id, '0.7', tip=True)
await self.support_create(stream1_id, '0.3', tip=True, wallet_id=wallet2.id)
await self.support_create(stream1_id, '0.2', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.4', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.5', tip=True, wallet_id=wallet2.id)
await self.support_create(stream2_id, '0.1', tip=True, wallet_id=wallet2.id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('1.0', claims[0]['received_tips'])
self.assertEqual('1.2', claims[1]['received_tips'])
await self.support_abandon(stream1_id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('1.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
await self.support_abandon(stream2_id)
claims = await self.claim_list(include_received_tips=True)
self.assertEqual('0.0', claims[0]['received_tips'])
self.assertEqual('0.0', claims[1]['received_tips'])
class ChannelCommands(CommandTestCase):
@@ -840,12 +1179,16 @@ class StreamCommands(ClaimTestCase):
claim_id = self.get_claim_id(tx)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
self.assertItemCount(await self.daemon.jsonrpc_txo_list(reposted_claim_id=claim_id), 0)
self.assertItemCount(await self.daemon.jsonrpc_txo_list(type='repost'), 0)
tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1')
repost_id = self.get_claim_id(tx)
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1)
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
self.assertEqual((await self.txo_list(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
self.assertEqual((await self.txo_list(type='repost'))[0]['claim_id'], repost_id)
# tags are inherited (non-common / indexed tags)
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2)
@@ -1364,11 +1707,12 @@ class StreamCommands(ClaimTestCase):
tx = await self.stream_create(bid='2.5') # creates new claim
claim_id = self.get_claim_id(tx)
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['claim_info']), 1)
self.assertEqual(txs[0]['confirmations'], 1)
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.020107')
await self.assertBalance(self.account, '7.479893')
@@ -1378,19 +1722,23 @@ class StreamCommands(ClaimTestCase):
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
await self.stream_update(claim_id, bid='1.0') # updates previous claim
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['update_info']), 1)
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
self.assertFalse(txs[0]['update_info'][0]['is_spent'])
self.assertTrue(txs[1]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.0002165')
await self.assertBalance(self.account, '8.9796765')
await self.stream_abandon(claim_id)
txs = (await self.out(self.daemon.jsonrpc_transaction_list()))['items']
txs = await self.transaction_list()
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[1]['update_info'][0]['is_spent'])
self.assertTrue(txs[2]['claim_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '0.0')
self.assertEqual(txs[0]['fee'], '-0.000107')
await self.assertBalance(self.account, '9.9795695')
@@ -1489,22 +1837,22 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '3.9998585')
# verify that the incoming tip is marked correctly as is_tip=True in account1
txs = (await self.out(self.daemon.jsonrpc_transaction_list(self.account.id)))['items']
txs = await self.transaction_list(account_id=self.account.id)
self.assertEqual(len(txs[0]['support_info']), 1)
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs[0]['support_info'][0]['is_tip'])
self.assertFalse(txs[0]['support_info'][0]['is_spent'])
self.assertEqual(txs[0]['value'], '1.0')
self.assertEqual(txs[0]['fee'], '0.0')
# verify that the outgoing tip is marked correctly as is_tip=True in account2
txs2 = (await self.out(
self.daemon.jsonrpc_transaction_list(wallet_id='wallet2', account_id=account2.id)
))['items']
txs2 = await self.transaction_list(wallet_id='wallet2', account_id=account2.id)
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertTrue(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '-1.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
@@ -1520,14 +1868,24 @@ class SupportCommands(CommandTestCase):
await self.assertBalance(account2, '1.999717')
# verify that the outgoing support is marked correctly as is_tip=False in account2
txs2 = (await self.out(self.daemon.jsonrpc_transaction_list(wallet_id='wallet2')))['items']
txs2 = await self.transaction_list(wallet_id='wallet2')
self.assertEqual(len(txs2[0]['support_info']), 1)
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim_id)
self.assertFalse(txs2[0]['support_info'][0]['is_tip'])
self.assertFalse(txs2[0]['support_info'][0]['is_spent'])
self.assertEqual(txs2[0]['value'], '0.0')
self.assertEqual(txs2[0]['fee'], '-0.0001415')
# abandoning the tip increases balance and shows tip as spent
await self.support_abandon(claim_id)
await self.assertBalance(self.account, '4.979662')
txs = await self.transaction_list(account_id=self.account.id)
self.assertEqual(len(txs[0]['abandon_info']), 1)
self.assertEqual(len(txs[1]['support_info']), 1)
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
class CollectionCommands(CommandTestCase):
@@ -31,7 +31,7 @@ class BasicTransactionTest(IntegrationTestCase):
channel_txo = Output.pay_claim_name_pubkey_hash(
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
)
channel_txo.generate_channel_private_key()
await channel_txo.generate_channel_private_key()
channel_txo.script.generate()
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
+5 -8
View File
@@ -1,4 +1,3 @@
import logging
import asyncio
import lbry
@@ -90,13 +89,11 @@ class ReconnectTests(IntegrationTestCase):
while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1
await asyncio.sleep(0.1)
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
# locks header processing so we make sure we are the only ones modifying it
async with self.ledger._header_processing_lock:
await self.ledger.headers.open()
await self.ledger.network.start()
await self.ledger.network.on_connected.first
await self.ledger.initial_headers_sync()
self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height)
await self.ledger.headers.open()
await self.ledger.network.start()
await self.ledger.network.on_connected.first
await self.ledger.initial_headers_sync()
self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height)
async def test_connection_drop_still_receives_events_after_reconnected(self):
address1 = await self.account.receiving.get_or_create_usable_address()
@@ -142,12 +142,12 @@ class PurchaseCommandTests(CommandTestCase):
self.assertEqual(result['items'][0]['purchase_info'][0]['claim_id'], claim_id2)
self.assertEqual(result['items'][2]['purchase_info'][0]['claim_id'], claim_id1)
result = await self.claim_search()
result = await self.claim_search(include_purchase_receipt=True)
self.assertEqual(result[0]['claim_id'], result[0]['purchase_receipt']['claim_id'])
self.assertEqual(result[1]['claim_id'], result[1]['purchase_receipt']['claim_id'])
url = result[0]['canonical_url']
resolve = await self.resolve(url)
resolve = await self.resolve(url, include_purchase_receipt=True)
self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id'])
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
@@ -49,11 +49,11 @@ class ResolveCommand(BaseResolveTestCase):
self.assertTrue(claim['is_channel_signature_valid'])
self.assertEqual(
claim['timestamp'],
self.ledger.headers[claim['height']]['timestamp']
self.ledger.headers.estimated_timestamp(claim['height'])
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
self.ledger.headers.estimated_timestamp(claim['signing_channel']['height'])
)
# resolving claim foo by itself
@@ -87,7 +87,6 @@ class ResolveCommand(BaseResolveTestCase):
await self.conductor.spv_node.stop()
resolve_task = asyncio.create_task(self.resolve('foo'))
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.on_ready.first
self.assertIsNotNone((await resolve_task)['claim_id'])
async def test_winning_by_effective_amount(self):
@@ -263,6 +262,73 @@ class ResolveCommand(BaseResolveTestCase):
await self.resolve('@olds/bad_example')
)
async def test_resolve_with_includes(self):
wallet2 = await self.daemon.jsonrpc_wallet_create('wallet2', create_account=True)
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.wallet_send('1.0', address2)
stream = await self.stream_create(
'priced', '0.1', wallet_id=wallet2.id,
fee_amount='0.5', fee_currency='LBC', fee_address=address2
)
stream_id = self.get_claim_id(stream)
resolve = await self.resolve('priced')
self.assertNotIn('is_my_output', resolve)
self.assertNotIn('purchase_receipt', resolve)
self.assertNotIn('sent_supports', resolve)
self.assertNotIn('sent_tips', resolve)
self.assertNotIn('received_tips', resolve)
# is_my_output
resolve = await self.resolve('priced', include_is_my_output=True)
self.assertFalse(resolve['is_my_output'])
resolve = await self.resolve('priced', wallet_id=wallet2.id, include_is_my_output=True)
self.assertTrue(resolve['is_my_output'])
# purchase receipt
resolve = await self.resolve('priced', include_purchase_receipt=True)
self.assertNotIn('purchase_receipt', resolve)
await self.purchase_create(stream_id)
resolve = await self.resolve('priced', include_purchase_receipt=True)
self.assertEqual('0.5', resolve['purchase_receipt']['amount'])
# my supports and my tips
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True
)
self.assertEqual('0.0', resolve['sent_supports'])
self.assertEqual('0.0', resolve['sent_tips'])
self.assertEqual('0.0', resolve['received_tips'])
await self.support_create(stream_id, '0.3')
await self.support_create(stream_id, '0.2')
await self.support_create(stream_id, '0.4', tip=True)
await self.support_create(stream_id, '0.5', tip=True)
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True
)
self.assertEqual('0.5', resolve['sent_supports'])
self.assertEqual('0.9', resolve['sent_tips'])
self.assertEqual('0.0', resolve['received_tips'])
resolve = await self.resolve(
'priced', include_sent_supports=True, include_sent_tips=True, include_received_tips=True,
wallet_id=wallet2.id
)
self.assertEqual('0.0', resolve['sent_supports'])
self.assertEqual('0.0', resolve['sent_tips'])
self.assertEqual('0.9', resolve['received_tips'])
self.assertEqual('1.4', resolve['meta']['support_amount'])
# make sure nothing is leaked between wallets through cached tx/txos
resolve = await self.resolve('priced')
self.assertNotIn('is_my_output', resolve)
self.assertNotIn('purchase_receipt', resolve)
self.assertNotIn('sent_supports', resolve)
self.assertNotIn('sent_tips', resolve)
self.assertNotIn('received_tips', resolve)
class ResolveAfterReorg(BaseResolveTestCase):
@@ -270,7 +336,7 @@ class ResolveAfterReorg(BaseResolveTestCase):
blocks = self.ledger.headers.height - start
self.blockchain.block_expected = start - 1
# go back to start
await self.blockchain.invalidate_block(self.ledger.headers.hash(start).decode())
await self.blockchain.invalidate_block((await self.ledger.headers.hash(start)).decode())
# go to previous + 1
await self.generate(blocks + 2)
@@ -122,13 +122,13 @@ class BasicTransactionTests(IntegrationTestCase):
await self.blockchain.generate(1)
await self.ledger.wait(tx) # confirmed
tx = (await account1.get_transactions())[1]
tx = (await account1.get_transactions(include_is_my_input=True, include_is_my_output=True))[1]
self.assertEqual(satoshis_to_coins(tx.inputs[0].amount), '1.1')
self.assertEqual(satoshis_to_coins(tx.inputs[1].amount), '1.1')
self.assertEqual(satoshis_to_coins(tx.outputs[0].amount), '2.0')
self.assertEqual(tx.outputs[0].get_address(self.ledger), address2)
self.assertFalse(tx.outputs[0].is_change)
self.assertTrue(tx.outputs[1].is_change)
self.assertTrue(tx.outputs[0].is_internal_transfer)
self.assertTrue(tx.outputs[1].is_internal_transfer)
async def test_history_edge_cases(self):
await self.blockchain.generate(300)
@@ -19,6 +19,29 @@ class WalletCommands(CommandTestCase):
await self.daemon.jsonrpc_wallet_add(wallet.id)
self.assertEqual(len(session.hashX_subs), 28)
async def test_wallet_syncing_status(self):
address = await self.daemon.jsonrpc_address_unused()
self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing'])
await self.blockchain.send_to_address(address, 1)
await self.ledger._update_tasks.started.wait()
self.assertTrue(self.daemon.jsonrpc_wallet_status()['is_syncing'])
await self.ledger._update_tasks.done.wait()
self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing'])
wallet = self.daemon.component_manager.get_actual_component('wallet')
wallet_manager = wallet.wallet_manager
# when component manager hasn't started yet
wallet.wallet_manager = None
self.assertEqual(
{'is_encrypted': None, 'is_syncing': None, 'is_locked': None},
self.daemon.jsonrpc_wallet_status()
)
wallet.wallet_manager = wallet_manager
self.assertEqual(
{'is_encrypted': False, 'is_syncing': False, 'is_locked': False},
self.daemon.jsonrpc_wallet_status()
)
async def test_wallet_reconnect(self):
await self.conductor.spv_node.stop(True)
self.conductor.spv_node.port = 54320
@@ -82,6 +105,11 @@ class WalletCommands(CommandTestCase):
async def test_granular_balances(self):
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True)
account3 = wallet2.default_account
address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id)
await self.confirm_tx(await self.blockchain.send_to_address(address3, 1))
await self.generate(1)
account_balance = self.daemon.jsonrpc_account_balance
wallet_balance = self.daemon.jsonrpc_wallet_balance
@@ -128,7 +156,7 @@ class WalletCommands(CommandTestCase):
# tip received
support1 = await self.support_create(
self.get_claim_id(stream1), '0.3', tip=True, funding_account_ids=[account2.id]
self.get_claim_id(stream1), '0.3', tip=True, wallet_id=wallet2.id
)
self.assertEqual(await account_balance(), {
'total': '9.27741',
@@ -137,8 +165,8 @@ class WalletCommands(CommandTestCase):
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
self.assertEqual(await wallet_balance(), {
'total': '9.977268',
'available': '6.677268',
'total': '10.27741',
'available': '6.97741',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
@@ -153,8 +181,8 @@ class WalletCommands(CommandTestCase):
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(), {
'total': '9.977161',
'available': '6.977161',
'total': '10.277303',
'available': '7.277303',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
@@ -165,17 +193,17 @@ class WalletCommands(CommandTestCase):
# tip another claim
await self.support_create(
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id]
self.get_claim_id(stream2), '0.2', tip=True, wallet_id=wallet2.id
)
self.assertEqual(await account_balance(), {
'total': '9.077157',
'available': '6.077157',
'total': '9.277303',
'available': '6.277303',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(), {
'total': '9.938908',
'available': '6.638908',
'total': '10.439196',
'available': '7.139196',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'}
})
@@ -263,7 +291,9 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
wallet = daemon.wallet_manager.default_wallet
wallet.save()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {
'is_locked': False, 'is_encrypted': False, 'is_syncing': False
})
self.assertIsNone(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK))
self.assertWalletEncrypted(wallet.storage.path, False)
@@ -275,22 +305,26 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
daemon.jsonrpc_wallet_decrypt() # already not encrypted
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
daemon.jsonrpc_wallet_lock()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True,
'is_syncing': False})
# can't sign transactions with locked wallet
with self.assertRaises(AssertionError):
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_unlock('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': False})
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_decrypt()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False,
'is_syncing': False})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': False})
self.assertWalletEncrypted(wallet.storage.path, False)
@@ -302,7 +336,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
self.assertTrue(daemon2.jsonrpc_wallet_encrypt('password'))
self.assertTrue(daemon2.jsonrpc_wallet_lock())
self.assertTrue(daemon2.jsonrpc_wallet_unlock("password"))
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon2.jsonrpc_wallet_status(),
{'is_locked': False, 'is_encrypted': True, 'is_syncing': False})
async def test_sync_with_encryption_and_password_change(self):
daemon, daemon2 = self.daemon, self.daemon2
@@ -323,7 +358,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True)
# sync_apply with new password2 also sets it as new local password
self.assertEqual(wallet.encryption_password, 'password2')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
@@ -340,7 +376,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
await daemon2.jsonrpc_sync_apply('password3', data=data['data'], blocking=True)
# the other device got new password and on disk encryption
self.assertEqual(wallet2.encryption_password, 'password3')
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True,
'is_syncing': True})
self.assertEqual(daemon2.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet2.storage.path, True)
@@ -45,14 +45,6 @@ class TestSessions(IntegrationTestCase):
await self.ledger.network.broadcast('13370042004200')
class TestSegwitServer(IntegrationTestCase):
LEDGER = lbry.wallet
ENABLE_SEGWIT = True
async def test_at_least_it_starts(self):
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
@@ -81,7 +73,7 @@ class TestUsagePayment(CommandTestCase):
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
@@ -90,7 +82,7 @@ class TestUsagePayment(CommandTestCase):
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
@@ -2,10 +2,60 @@ import asyncio
import os
from binascii import hexlify
from lbry.schema import Claim
from lbry.testcase import CommandTestCase
from lbry.torrent.session import TorrentSession
from lbry.wallet import Transaction
class FileCommands(CommandTestCase):
async def initialize_torrent(self, tx_to_update=None):
if not hasattr(self, 'seeder_session'):
self.seeder_session = TorrentSession(self.loop, None)
self.addCleanup(self.seeder_session.stop)
await self.seeder_session.bind(port=4040)
btih = await self.seeder_session.add_fake_torrent()
address = await self.account.receiving.get_or_create_usable_address()
if not tx_to_update:
claim = Claim()
claim.stream.update(bt_infohash=btih)
tx = await Transaction.claim_create(
'torrent', claim, 1, address, [self.account], self.account
)
else:
claim = tx_to_update.outputs[0].claim
claim.stream.update(bt_infohash=btih)
tx = await Transaction.claim_update(
tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
self.client_session._session.add_dht_node(('localhost', 4040))
self.client_session.wait_start = False # fixme: this is super slow on tests
return tx, btih
async def test_download_torrent(self):
tx, btih = await self.initialize_torrent()
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
# second call, see its there and move on
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih)
self.assertIn(btih, self.client_session._handles)
tx, new_btih = await self.initialize_torrent(tx)
self.assertNotEqual(btih, new_btih)
# claim now points to another torrent, update to it
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih)
self.assertIn(new_btih, self.client_session._handles)
self.assertNotIn(btih, self.client_session._handles)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
await self.daemon.jsonrpc_file_delete(delete_all=True)
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
self.assertNotIn(new_btih, self.client_session._handles)
async def create_streams_in_range(self, *args, **kwargs):
self.stream_claim_ids = []
@@ -21,6 +71,11 @@ class FileCommands(CommandTestCase):
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
@@ -125,14 +180,14 @@ class FileCommands(CommandTestCase):
file_list = await self.file_list()
self.assertEqual(
file_list[0]['timestamp'],
None
self.ledger.headers.estimated_timestamp(file_list[0]['height'])
)
self.assertEqual(file_list[0]['confirmations'], -1)
await self.daemon.jsonrpc_resolve('foo')
file_list = await self.file_list()
self.assertEqual(
file_list[0]['timestamp'],
self.ledger.headers[file_list[0]['height']]['timestamp']
self.ledger.headers.estimated_timestamp(file_list[0]['height'])
)
self.assertEqual(file_list[0]['confirmations'], 1)
@@ -223,11 +278,11 @@ class FileCommands(CommandTestCase):
await self.daemon.jsonrpc_get('lbry://foo')
with open(original_path, 'wb') as handle:
handle.write(b'some other stuff was there instead')
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
self.daemon.file_manager.stop()
await self.daemon.file_manager.start()
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed
# check that internal state got through up to the file list API
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
stream = self.daemon.file_manager.get_filtered(stream_hash=file_info['stream_hash'])[0]
file_info = (await self.file_list())[0]
self.assertEqual(stream.file_name, file_info['file_name'])
# checks if what the API shows is what he have at the very internal level.
@@ -250,7 +305,7 @@ class FileCommands(CommandTestCase):
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
self.assertNotIn('error', resp)
self.assertTrue(os.path.isfile(path))
self.daemon.stream_manager.stop()
self.daemon.file_manager.stop()
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
self.assertFalse(os.path.isfile(path))
@@ -343,8 +398,8 @@ class FileCommands(CommandTestCase):
# restart the daemon and make sure the fee is still there
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
self.daemon.file_manager.stop()
await self.daemon.file_manager.start()
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].content_fee.raw, raw_content_fee)
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
@@ -21,8 +21,8 @@ def get_random_bytes(n: int) -> bytes:
class RangeRequests(CommandTestCase):
async def _restart_stream_manager(self):
self.daemon.stream_manager.stop()
await self.daemon.stream_manager.start()
self.daemon.file_manager.stop()
await self.daemon.file_manager.start()
return
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
+3 -3
View File
@@ -6,7 +6,7 @@ from lbry.conf import Config
from lbry.extras import cli
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
@@ -21,7 +21,7 @@ class CLIIntegrationTest(AsyncioTestCase):
conf.api = 'localhost:5299'
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
)
Daemon.component_attributes = {}
@@ -34,4 +34,4 @@ class CLIIntegrationTest(AsyncioTestCase):
with contextlib.redirect_stdout(actual_output):
cli.main(["--api", "localhost:5299", "status"])
actual_output = actual_output.getvalue()
self.assertIn("connection_status", actual_output)
self.assertIn("connection_status", actual_output)
@@ -106,11 +106,16 @@ class MockedCommentServer:
return False
def hide_comments(self, pieces: list):
comments_hidden = []
hidden = []
for p in pieces:
if self.hide_comment(**p):
comments_hidden.append(p['comment_id'])
return {'hidden': comments_hidden}
hidden.append(p['comment_id'])
comment_ids = {c['comment_id'] for c in pieces}
return {
'hidden': hidden,
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
@@ -138,12 +143,19 @@ class MockedCommentServer:
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[self.get_comment_id(comment_id)]
return {
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
'channel_id': comment['channel_id'],
'channel_name': comment['channel_name'],
}
def get_comments_by_id(self, comment_ids: list):
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
return {
'page': 1,
'page_size': len(comment_ids),
'total_pages': 1,
'items': comments,
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
methods = {
'get_claim_comments': get_claim_comments,
@@ -38,3 +38,20 @@ class SettingsManagement(CommandTestCase):
self.assertTrue(self.daemon.analytics_manager.enabled)
self.assertTrue(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', False)
class TroubleshootingCommands(CommandTestCase):
async def test_tracemalloc_commands(self):
self.addCleanup(self.daemon.jsonrpc_tracemalloc_disable)
self.assertFalse(self.daemon.jsonrpc_tracemalloc_disable())
self.assertTrue(self.daemon.jsonrpc_tracemalloc_enable())
class WeirdObject():
pass
hold_em = [WeirdObject() for _ in range(500)]
top = self.daemon.jsonrpc_tracemalloc_top(1)
self.assertEqual(1, len(top))
self.assertEqual('hold_em = [WeirdObject() for _ in range(500)]', top[0]['code'])
self.assertTrue(top[0]['line'].startswith('other/test_other_commands.py:'))
self.assertGreaterEqual(top[0]['count'], 500)
self.assertGreater(top[0]['size'], 0) # just matters that its a positive integer
+48 -14
View File
@@ -32,13 +32,14 @@ class TranscodeValidation(ClaimTestCase):
self.conf = TranscodeConfig()
self.conf.volume_analysis_time = 0 # disable it as the test file isn't very good here
self.analyzer = VideoFileAnalyzer(self.conf)
self.assertTrue((await self.analyzer.status())["available"]) # ensure ffmpeg path detected
file_ogg = self.make_name("ogg", ".ogg")
self.video_file_ogg = str(file_ogg)
if not file_ogg.exists():
command = f'-i "{self.video_file_name}" -c:v libtheora -q:v 4 -c:a libvorbis -q:a 4 ' \
f'-c:s copy -c:d copy "{file_ogg}"'
with MeasureTime(f"Creating {file_ogg.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
file_webm = self.make_name("webm", ".webm")
@@ -47,16 +48,19 @@ class TranscodeValidation(ClaimTestCase):
command = f'-i "{self.video_file_name}" -c:v libvpx-vp9 -crf 36 -b:v 0 -cpu-used 2 ' \
f'-c:a libopus -b:a 128k -c:s copy -c:d copy "{file_webm}"'
with MeasureTime(f"Creating {file_webm.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
async def test_should_work(self):
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_name)
new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_name)
self.assertEqual(self.video_file_name, new_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg)
new_file_name, _ = await self.analyzer.verify_or_repair(True, False, self.video_file_ogg)
self.assertEqual(self.video_file_ogg, new_file_name)
new_file_name = await self.analyzer.verify_or_repair(True, False, self.video_file_webm)
new_file_name, spec = await self.analyzer.verify_or_repair(True, False, self.video_file_webm)
self.assertEqual(self.video_file_webm, new_file_name)
self.assertEqual(spec["width"], 1280)
self.assertEqual(spec["height"], 720)
self.assertEqual(spec["duration"], 16)
async def test_volume(self):
self.conf.volume_analysis_time = 200
@@ -68,13 +72,13 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Container format is not in the approved list"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_video_codec(self):
@@ -82,7 +86,7 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx265 -preset superfast "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Video codec is not in the approved list"):
@@ -90,22 +94,27 @@ class TranscodeValidation(ClaimTestCase):
with self.assertRaisesRegex(Exception, "faststart flag was not used"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_max_bit_rate(self):
self.conf.video_bitrate_maximum = 100
with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
async def test_video_format(self):
file_name = self.make_name("bad_video_format_1")
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:v libx264 ' \
f'-vf format=yuv444p "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "pixel format does not match the approved"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_audio_codec(self):
@@ -113,13 +122,13 @@ class TranscodeValidation(ClaimTestCase):
if not file_name.exists():
command = f'-i "{self.video_file_name}" -c copy -map 0 -c:a pcm_s16le "{file_name}"'
with MeasureTime(f"Creating {file_name.name}"):
output, code = await self.analyzer._execute("ffmpeg", command)
output, code = await self.analyzer._execute_ffmpeg(command)
self.assertEqual(code, 0, output)
with self.assertRaisesRegex(Exception, "Audio codec is not in the approved list"):
await self.analyzer.verify_or_repair(True, False, file_name)
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
fixed_file, _ = await self.analyzer.verify_or_repair(True, True, file_name)
pathlib.Path(fixed_file).unlink()
async def test_extension_choice(self):
@@ -146,6 +155,31 @@ class TranscodeValidation(ClaimTestCase):
self.assertEqual("ogv", extension)
async def test_no_ffmpeg(self):
self.conf.ffmpeg_folder = "I don't really exist/"
self.conf.ffmpeg_path = "I don't really exist/"
self.analyzer._env_copy.pop("PATH", None)
await self.analyzer.status(reset=True)
with self.assertRaisesRegex(Exception, "Unable to locate"):
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
async def test_dont_recheck_ffmpeg_installation(self):
call_count = 0
original = self.daemon._video_file_analyzer._verify_ffmpeg_installed
def _verify_ffmpeg_installed():
nonlocal call_count
call_count += 1
return original()
self.daemon._video_file_analyzer._verify_ffmpeg_installed = _verify_ffmpeg_installed
self.assertEqual(0, call_count)
await self.daemon.jsonrpc_status()
self.assertEqual(1, call_count)
# counter should not go up again
await self.daemon.jsonrpc_status()
self.assertEqual(1, call_count)
# this should force rechecking the installation
await self.daemon.jsonrpc_ffmpeg_find()
self.assertEqual(2, call_count)
+2 -1
View File
@@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase):
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
await self.storage.open()
async def test_memory_blobs_arent_verifie_but_real_ones_are(self):
async def test_memory_blobs_arent_verified_but_real_ones_are(self):
for save_blobs in (False, True):
await self.setup_blob_manager(save_blobs=save_blobs)
# add a blob file
@@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase):
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes))
blob.save_verified_blob(blob_bytes)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.blob_manager.blob_completed(blob)
self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs)
@@ -34,13 +34,13 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.addCleanup(shutil.rmtree, self.client_dir)
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
reflector_servers=[])
fixed_peers=[])
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
reflector_servers=[])
fixed_peers=[])
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
self.client_peer_manager = PeerManager(self.loop)
@@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase):
write_blob = blob._write_blob
write_called_count = 0
def wrap_write_blob(blob_bytes):
async def _wrap_write_blob(blob_bytes):
nonlocal write_called_count
write_called_count += 1
write_blob(blob_bytes)
await write_blob(blob_bytes)
def wrap_write_blob(blob_bytes):
return asyncio.create_task(_wrap_write_blob(blob_bytes))
blob._write_blob = wrap_write_blob
writer1 = blob.get_blob_writer(peer_port=1)
@@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase):
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
self.assertEqual(1, write_called_count)
await blob.verified.wait()
self.assertTrue(blob.get_is_verified())
self.assertDictEqual({}, blob.writers)
+10 -10
View File
@@ -18,31 +18,31 @@ class TestSigningComments(AsyncioTestCase):
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
}
def test01_successful_create_sign_and_validate_comment(self):
channel = get_channel('@BusterBluth')
async def test01_successful_create_sign_and_validate_comment(self):
channel = await get_channel('@BusterBluth')
stream = get_stream('pop secret')
comment = self.create_claim_comment_body('Cool stream', stream, channel)
sign_comment(comment, channel)
self.assertTrue(is_comment_signed_by_channel(comment, channel))
def test02_fail_to_validate_spoofed_channel(self):
pdiddy = get_channel('@PDitty')
channel2 = get_channel('@TomHaverford')
async def test02_fail_to_validate_spoofed_channel(self):
pdiddy = await get_channel('@PDitty')
channel2 = await get_channel('@TomHaverford')
stream = get_stream()
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
def test03_successful_sign_abandon_comment(self):
rswanson = get_channel('@RonSwanson')
async def test03_successful_sign_abandon_comment(self):
rswanson = await get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
def test04_invalid_signature(self):
rswanson = get_channel('@RonSwanson')
jeanralphio = get_channel('@JeanRalphio')
async def test04_invalid_signature(self):
rswanson = await get_channel('@RonSwanson')
jeanralphio = await get_channel('@JeanRalphio')
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
chair_comment = self.create_claim_comment_body(
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
+14 -11
View File
@@ -16,6 +16,7 @@ class TestComponentManager(AsyncioTestCase):
[
components.DatabaseComponent,
components.ExchangeRateManagerComponent,
components.TorrentComponent,
components.UPnPComponent
],
[
@@ -24,9 +25,9 @@ class TestComponentManager(AsyncioTestCase):
components.WalletComponent
],
[
components.FileManagerComponent,
components.HashAnnouncerComponent,
components.PeerProtocolServerComponent,
components.StreamManagerComponent,
components.WalletServerPaymentsComponent
]
]
@@ -119,6 +120,8 @@ class FakeComponent:
class FakeDelayedWallet(FakeComponent):
component_name = "wallet"
depends_on = []
ledger = None
default_wallet = None
async def stop(self):
await asyncio.sleep(1)
@@ -135,8 +138,8 @@ class FakeDelayedBlobManager(FakeComponent):
await asyncio.sleep(1)
class FakeDelayedStreamManager(FakeComponent):
component_name = "stream_manager"
class FakeDelayedFileManager(FakeComponent):
component_name = "file_manager"
depends_on = [FakeDelayedBlobManager.component_name]
async def start(self):
@@ -153,7 +156,7 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT,
EXCHANGE_RATE_MANAGER_COMPONENT],
wallet=FakeDelayedWallet,
stream_manager=FakeDelayedStreamManager,
file_manager=FakeDelayedFileManager,
blob_manager=FakeDelayedBlobManager
)
@@ -163,17 +166,17 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
await self.advance(0)
self.assertTrue(self.component_manager.get_component('wallet').running)
self.assertFalse(self.component_manager.get_component('blob_manager').running)
self.assertFalse(self.component_manager.get_component('stream_manager').running)
self.assertFalse(self.component_manager.get_component('file_manager').running)
await self.advance(1)
self.assertTrue(self.component_manager.get_component('wallet').running)
self.assertTrue(self.component_manager.get_component('blob_manager').running)
self.assertFalse(self.component_manager.get_component('stream_manager').running)
self.assertFalse(self.component_manager.get_component('file_manager').running)
await self.advance(1)
self.assertTrue(self.component_manager.get_component('wallet').running)
self.assertTrue(self.component_manager.get_component('blob_manager').running)
self.assertTrue(self.component_manager.get_component('stream_manager').running)
self.assertTrue(self.component_manager.get_component('file_manager').running)
async def test_proper_stopping_of_components(self):
asyncio.create_task(self.component_manager.start())
@@ -182,18 +185,18 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
await self.advance(1)
self.assertTrue(self.component_manager.get_component('wallet').running)
self.assertTrue(self.component_manager.get_component('blob_manager').running)
self.assertTrue(self.component_manager.get_component('stream_manager').running)
self.assertTrue(self.component_manager.get_component('file_manager').running)
asyncio.create_task(self.component_manager.stop())
await self.advance(0)
self.assertFalse(self.component_manager.get_component('stream_manager').running)
self.assertFalse(self.component_manager.get_component('file_manager').running)
self.assertTrue(self.component_manager.get_component('blob_manager').running)
self.assertTrue(self.component_manager.get_component('wallet').running)
await self.advance(1)
self.assertFalse(self.component_manager.get_component('stream_manager').running)
self.assertFalse(self.component_manager.get_component('file_manager').running)
self.assertFalse(self.component_manager.get_component('blob_manager').running)
self.assertTrue(self.component_manager.get_component('wallet').running)
await self.advance(1)
self.assertFalse(self.component_manager.get_component('stream_manager').running)
self.assertFalse(self.component_manager.get_component('file_manager').running)
self.assertFalse(self.component_manager.get_component('blob_manager').running)
self.assertFalse(self.component_manager.get_component('wallet').running)
+26 -1
View File
@@ -1,7 +1,7 @@
import asyncio
import unittest
from lbry.utils import generate_id
from lbry.dht.peer import PeerManager, make_kademlia_peer
from lbry.dht.peer import PeerManager, make_kademlia_peer, is_valid_public_ipv4
from lbry.testcase import AsyncioTestCase
@@ -50,6 +50,31 @@ class PeerTest(AsyncioTestCase):
for i in range(32):
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], f"{224 + i}.0.0.0", 1000)
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '255.255.255.255', 1000)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], 'beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', 1000
)
self.assertRaises(
ValueError, make_kademlia_peer, self.node_ids[1], '2001:db8::ff00:42:8329', 1000
)
def test_is_valid_ipv4(self):
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef'))
self.assertFalse(is_valid_public_ipv4('beee:eeee:eeee:eeee:eeee:eeee:eeee:eeef', True))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329'))
self.assertFalse(is_valid_public_ipv4('2001:db8::ff00:42:8329', True))
self.assertFalse(is_valid_public_ipv4('127.0.0.1'))
self.assertTrue(is_valid_public_ipv4('127.0.0.1', True))
self.assertFalse(is_valid_public_ipv4('172.16.0.1'))
self.assertFalse(is_valid_public_ipv4('172.16.0.1', True))
self.assertTrue(is_valid_public_ipv4('1.2.3.4'))
self.assertTrue(is_valid_public_ipv4('1.2.3.4', True))
self.assertFalse(is_valid_public_ipv4('derp'))
self.assertFalse(is_valid_public_ipv4('derp', True))
def test_boolean(self):
self.assertNotEqual(self.first_contact, self.second_contact)
+1 -1
View File
@@ -92,7 +92,7 @@ class TestCostEst(unittest.TestCase):
@unittest.SkipTest
class TestJsonRpc(unittest.TestCase):
def setUp(self):
def noop():
async def noop():
return None
test_utils.reset_time(self)
@@ -0,0 +1,53 @@
import unittest
from aiohttp.test_utils import make_mocked_request as request
from aiohttp.web import HTTPForbidden
from lbry.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.extras.daemon.security import is_request_allowed as allowed, ensure_request_allowed as ensure
class TestAllowedOrigin(unittest.TestCase):
def test_allowed_origin_default(self):
conf = Config()
# lack of Origin is always allowed
self.assertTrue(allowed(request('GET', '/'), conf))
# deny all other Origins
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_star(self):
conf = Config(allowed_origin='*')
# everything is allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_allowed_origin_specified(self):
conf = Config(allowed_origin='localhost')
# no origin and only localhost are allowed
self.assertTrue(allowed(request('GET', '/'), conf))
self.assertTrue(allowed(request('GET', '/', headers={'Origin': 'localhost'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'null'}), conf))
self.assertFalse(allowed(request('GET', '/', headers={'Origin': 'hackers.com'}), conf))
def test_ensure_default(self):
conf = Config()
ensure(request('GET', '/'), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
self.assertIn("'localhost' are not allowed", log.output[0])
def test_ensure_specific(self):
conf = Config(allowed_origin='localhost')
ensure(request('GET', '/', headers={'Origin': 'localhost'}), conf)
with self.assertLogs() as log:
with self.assertRaises(HTTPForbidden):
ensure(request('GET', '/', headers={'Origin': 'hackers.com'}), conf)
self.assertIn("'hackers.com' are not allowed", log.output[0])
self.assertIn("'allowed_origin' limits requests to: 'localhost'", log.output[0])
+4 -3
View File
@@ -76,7 +76,8 @@ class TestManagedStream(BlobExchangeTestBase):
return q2, self.loop.create_task(_task())
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
self.stream.downloader.node = mock_node
await self.stream.save_file()
await self.stream.finished_write_attempt.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
if stop_when_done:
@@ -109,7 +110,6 @@ class TestManagedStream(BlobExchangeTestBase):
await self.setup_stream(2)
mock_node = mock.Mock(spec=Node)
q = asyncio.Queue()
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334, allow_localhost=True)
@@ -123,7 +123,8 @@ class TestManagedStream(BlobExchangeTestBase):
mock_node.accumulate_peers = _mock_accumulate_peers
await self.stream.save_file(node=mock_node)
self.stream.downloader.node = mock_node
await self.stream.save_file()
await self.stream.finished_writing.wait()
self.assertTrue(os.path.isfile(self.stream.full_path))
with open(self.stream.full_path, 'rb') as f:
+3 -1
View File
@@ -39,14 +39,16 @@ class TestStreamAssembler(AsyncioTestCase):
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream = await self.stream_manager.create_stream(file_path)
self.stream = await self.stream_manager.create(file_path)
async def _test_reflect_stream(self, response_chunk_size):
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(100, self.stream.reflector_progress)
self.assertSetEqual(
set(sent),
set(map(lambda b: b.blob_hash,
+32 -20
View File
@@ -5,6 +5,8 @@ from unittest import mock
import asyncio
import json
from decimal import Decimal
from lbry.file.file_manager import FileManager
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
from lbry.testcase import get_fake_exchange_rate_manager
from lbry.utils import generate_id
@@ -84,6 +86,9 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
})
class FakeHeaders:
def estimated_timestamp(self, height):
return 1984
def __init__(self, height):
self.height = height
@@ -105,12 +110,9 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
network=manager.ledger.network, server=('fakespv.lbry.com', 50001)
)
async def mock_resolve(*args):
async def mock_resolve(*args, **kwargs):
result = {txo.meta['permanent_url']: txo}
claims = [
StreamManager._convert_to_old_resolve_output(manager, result)[txo.meta['permanent_url']]
]
await storage.save_claims(claims)
await storage.save_claim_from_output(ledger, txo)
return result
manager.ledger.resolve = mock_resolve
@@ -135,11 +137,20 @@ class TestStreamManager(BlobExchangeTestBase):
)
self.sd_hash = descriptor.sd_hash
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
self.client_storage, get_mock_node(self.server_from_client),
AnalyticsManager(self.client_config,
binascii.hexlify(generate_id()).decode(),
binascii.hexlify(generate_id()).decode()))
analytics_manager = AnalyticsManager(
self.client_config,
binascii.hexlify(generate_id()).decode(),
binascii.hexlify(generate_id()).decode()
)
self.stream_manager = StreamManager(
self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
self.client_storage, get_mock_node(self.server_from_client),
analytics_manager
)
self.file_manager = FileManager(
self.loop, self.client_config, self.mock_wallet, self.client_storage, analytics_manager
)
self.file_manager.source_managers['stream'] = self.stream_manager
self.exchange_rate_manager = get_fake_exchange_rate_manager()
async def _test_time_to_first_bytes(self, check_post, error=None, after_setup=None):
@@ -156,9 +167,9 @@ class TestStreamManager(BlobExchangeTestBase):
self.stream_manager.analytics_manager._post = _check_post
if error:
with self.assertRaises(error):
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
else:
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
await asyncio.sleep(0, loop=self.loop)
self.assertTrue(checked_analytics_event)
@@ -176,7 +187,7 @@ class TestStreamManager(BlobExchangeTestBase):
await self._test_time_to_first_bytes(check_post)
async def test_fixed_peer_delay_dht_peers_found(self):
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)]
server_from_client = None
self.server_from_client, server_from_client = server_from_client, self.server_from_client
@@ -220,7 +231,7 @@ class TestStreamManager(BlobExchangeTestBase):
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup)
async def test_override_fixed_peer_delay_dht_disabled(self):
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)]
self.client_config.components_to_skip = ['dht', 'hash_announcer']
self.client_config.fixed_peer_delay = 9001.0
self.server_from_client = None
@@ -278,7 +289,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.stream_manager.analytics_manager._post = check_post
self.assertDictEqual(self.stream_manager.streams, {})
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
stream_hash = stream.stream_hash
self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream})
self.assertTrue(stream.running)
@@ -299,7 +310,8 @@ class TestStreamManager(BlobExchangeTestBase):
)
self.assertEqual(stored_status, "stopped")
await stream.save_file(node=self.stream_manager.node)
stream.downloader.node = self.stream_manager.node
await stream.save_file()
await stream.finished_writing.wait()
await asyncio.sleep(0, loop=self.loop)
self.assertTrue(stream.finished)
@@ -310,7 +322,7 @@ class TestStreamManager(BlobExchangeTestBase):
)
self.assertEqual(stored_status, "finished")
await self.stream_manager.delete_stream(stream, True)
await self.stream_manager.delete(stream, True)
self.assertDictEqual(self.stream_manager.streams, {})
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
stored_status = await self.client_storage.run_and_return_one_or_none(
@@ -322,7 +334,7 @@ class TestStreamManager(BlobExchangeTestBase):
async def _test_download_error_on_start(self, expected_error, timeout=None):
error = None
try:
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout)
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
@@ -398,7 +410,7 @@ class TestStreamManager(BlobExchangeTestBase):
last_blob_hash = json.loads(sdf.read())['blobs'][-2]['blob_hash']
self.server_blob_manager.delete_blob(last_blob_hash)
self.client_config.blob_download_timeout = 0.1
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
await stream.started_writing.wait()
self.assertEqual('running', stream.status)
self.assertIsNotNone(stream.full_path)
@@ -430,7 +442,7 @@ class TestStreamManager(BlobExchangeTestBase):
self.stream_manager.analytics_manager._post = check_post
self.assertDictEqual(self.stream_manager.streams, {})
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager)
await stream.finished_writing.wait()
await asyncio.sleep(0, loop=self.loop)
self.stream_manager.stop()
+1 -1
View File
@@ -218,7 +218,7 @@ class DaemonDocsTests(TestCase):
try:
docopt.docopt(fn.__doc__, ())
except docopt.DocoptLanguageError as err:
failures.append(f"invalid docstring for {name}, {err.message}")
failures.append(f"invalid docstring for {name}, {err.args[0]}")
except docopt.DocoptExit:
pass
if failures:
+5
View File
@@ -90,10 +90,15 @@ class ConfigurationTests(unittest.TestCase):
def test_environment(self):
c = TestConfig()
self.assertEqual(c.test_str, 'the default')
c.set_environment({'LBRY_TEST_STR': 'from environ'})
self.assertEqual(c.test_str, 'from environ')
self.assertEqual(c.test_int, 9)
c.set_environment({'LBRY_TEST_INT': '1'})
self.assertEqual(c.test_int, 1)
def test_persisted(self):
with tempfile.TemporaryDirectory() as temp_dir:
+24 -22
View File
@@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
await self.ledger.headers.open()
self.wallet = Wallet()
await self.ledger.db.open()
@@ -266,9 +267,10 @@ class TestQueries(AsyncioTestCase):
# SQLite is usually compiled with 999 variables limit: https://www.sqlite.org/limits.html
# This can be removed when there is a better way. See: https://github.com/lbryio/lbry-sdk/issues/2281
fetchall = self.ledger.db.db.execute_fetchall
def check_parameters_length(sql, parameters):
def check_parameters_length(sql, parameters, read_only=False):
self.assertLess(len(parameters or []), 999)
return fetchall(sql, parameters)
return fetchall(sql, parameters, read_only)
self.ledger.db.db.execute_fetchall = check_parameters_length
account = await self.create_account()
@@ -352,37 +354,37 @@ class TestQueries(AsyncioTestCase):
self.assertListEqual([tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
self.assertListEqual([3, 2, 1], [tx.height for tx in txs])
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts)
txs = await self.ledger.db.get_transactions(wallet=wallet1, accounts=wallet1.accounts, include_is_my_output=True)
self.assertListEqual([tx2.id, tx1.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
self.assertEqual(txs[0].inputs[0].is_my_input, True)
self.assertEqual(txs[0].outputs[0].is_my_output, False)
self.assertEqual(txs[1].inputs[0].is_my_input, False)
self.assertEqual(txs[1].outputs[0].is_my_output, True)
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2])
txs = await self.ledger.db.get_transactions(wallet=wallet2, accounts=[account2], include_is_my_output=True)
self.assertListEqual([tx3.id, tx2.id], [tx.id for tx in txs])
self.assertEqual(txs[0].inputs[0].is_my_account, True)
self.assertEqual(txs[0].outputs[0].is_my_account, False)
self.assertEqual(txs[1].inputs[0].is_my_account, False)
self.assertEqual(txs[1].outputs[0].is_my_account, True)
self.assertEqual(txs[0].inputs[0].is_my_input, True)
self.assertEqual(txs[0].outputs[0].is_my_output, False)
self.assertEqual(txs[1].inputs[0].is_my_input, False)
self.assertEqual(txs[1].outputs[0].is_my_output, True)
self.assertEqual(2, await self.ledger.db.get_transaction_count(accounts=[account2]))
tx = await self.ledger.db.get_transaction(txid=tx2.id)
self.assertEqual(tx.id, tx2.id)
self.assertFalse(tx.inputs[0].is_my_account)
self.assertFalse(tx.outputs[0].is_my_account)
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id)
self.assertTrue(tx.inputs[0].is_my_account)
self.assertFalse(tx.outputs[0].is_my_account)
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id)
self.assertFalse(tx.inputs[0].is_my_account)
self.assertTrue(tx.outputs[0].is_my_account)
self.assertIsNone(tx.inputs[0].is_my_input)
self.assertIsNone(tx.outputs[0].is_my_output)
tx = await self.ledger.db.get_transaction(wallet=wallet1, txid=tx2.id, include_is_my_output=True)
self.assertTrue(tx.inputs[0].is_my_input)
self.assertFalse(tx.outputs[0].is_my_output)
tx = await self.ledger.db.get_transaction(wallet=wallet2, txid=tx2.id, include_is_my_output=True)
self.assertFalse(tx.inputs[0].is_my_input)
self.assertTrue(tx.outputs[0].is_my_output)
# height 0 sorted to the top with the rest in descending order
tx4 = await self.create_tx_from_nothing(account1, 0)
txos = await self.ledger.db.get_txos()
self.assertListEqual([0, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertListEqual([tx4.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
self.assertListEqual([0, 3, 2, 2, 1], [txo.tx_ref.height for txo in txos])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx2b.id, tx1.id], [txo.tx_ref.id for txo in txos])
txs = await self.ledger.db.get_transactions(accounts=[account1, account2])
self.assertListEqual([0, 3, 2, 1], [tx.height for tx in txs])
self.assertListEqual([tx4.id, tx3.id, tx2.id, tx1.id], [tx.id for tx in txs])
+70 -27
View File
@@ -1,12 +1,15 @@
import os
import asyncio
import tempfile
from binascii import hexlify, unhexlify
from binascii import unhexlify
from lbry.crypto.hash import sha256
from lbry.wallet.util import ArithUint256
from lbry.testcase import AsyncioTestCase
from lbry.wallet.ledger import Headers
from lbry.wallet.ledger import Headers as _Headers
class Headers(_Headers):
checkpoints = {}
def block_bytes(blocks):
@@ -15,11 +18,12 @@ def block_bytes(blocks):
class TestHeaders(AsyncioTestCase):
def test_deserialize(self):
async def test_deserialize(self):
self.maxDiff = None
h = Headers(':memory:')
h.io.write(HEADERS)
self.assertEqual(h[0], {
await h.open()
await h.connect(0, HEADERS)
self.assertEqual(await h.get(0), {
'bits': 520159231,
'block_height': 0,
'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001',
@@ -29,7 +33,7 @@ class TestHeaders(AsyncioTestCase):
'timestamp': 1446058291,
'version': 1
})
self.assertEqual(h[10], {
self.assertEqual(await h.get(10), {
'bits': 509349720,
'block_height': 10,
'merkle_root': b'f4d8fded6a181d4a8a2817a0eb423cc0f414af29490004a620e66c35c498a554',
@@ -42,13 +46,18 @@ class TestHeaders(AsyncioTestCase):
async def test_connect_from_genesis(self):
headers = Headers(':memory:')
await headers.open()
self.assertEqual(headers.height, -1)
await headers.connect(0, HEADERS)
self.assertEqual(headers.height, 19)
async def test_connect_from_middle(self):
h = Headers(':memory:')
h.io.write(HEADERS[:block_bytes(10)])
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS[:block_bytes(10)])
h = Headers(headers_temporary_file)
await h.open()
self.assertEqual(h.height, 9)
await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)])
self.assertEqual(h.height, 19)
@@ -109,17 +118,19 @@ class TestHeaders(AsyncioTestCase):
async def test_bounds(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS)
self.assertEqual(19, headers.height)
with self.assertRaises(IndexError):
_ = headers[3001]
_ = await headers.get(3001)
with self.assertRaises(IndexError):
_ = headers[-1]
self.assertIsNotNone(headers[19])
self.assertIsNotNone(headers[0])
_ = await headers.get(-1)
self.assertIsNotNone(await headers.get(19))
self.assertIsNotNone(await headers.get(0))
async def test_repair(self):
headers = Headers(':memory:')
await headers.open()
await headers.connect(0, HEADERS[:block_bytes(11)])
self.assertEqual(10, headers.height)
await headers.repair()
@@ -137,21 +148,52 @@ class TestHeaders(AsyncioTestCase):
self.assertEqual(7, headers.height)
await headers.connect(len(headers), HEADERS[block_bytes(8):])
self.assertEqual(19, headers.height)
# verify from middle
await headers.repair(start_height=10)
self.assertEqual(19, headers.height)
async def test_checkpointed_writer(self):
async def test_do_not_estimate_unconfirmed(self):
headers = Headers(':memory:')
getblocks = lambda start, end: HEADERS[block_bytes(start):block_bytes(end)]
headers.checkpoint = 10, hexlify(sha256(getblocks(10, 11)))
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(0, 10))
self.assertEqual(len(headers), 10)
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(10, 19))
self.assertEqual(len(headers), 19)
await headers.open()
self.assertIsNone(headers.estimated_timestamp(-1))
self.assertIsNone(headers.estimated_timestamp(0))
self.assertIsNotNone(headers.estimated_timestamp(1))
async def test_dont_estimate_whats_there(self):
headers = Headers(':memory:')
async with headers.checkpointed_connector() as buff:
buff.write(getblocks(0, 19))
self.assertEqual(len(headers), 19)
await headers.open()
estimated = headers.estimated_timestamp(10)
await headers.connect(0, HEADERS)
real_time = (await headers.get(10))['timestamp']
after_downloading_header_estimated = headers.estimated_timestamp(10)
self.assertNotEqual(estimated, after_downloading_header_estimated)
self.assertEqual(after_downloading_header_estimated, real_time)
async def test_misalignment_triggers_repair_on_open(self):
headers_temporary_file = tempfile.mktemp()
self.addCleanup(os.remove, headers_temporary_file)
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.write(HEADERS)
headers = Headers(headers_temporary_file)
with self.assertLogs(level='WARN') as cm:
await headers.open()
await headers.close()
self.assertEqual(cm.output, [])
with open(headers_temporary_file, 'w+b') as headers_file:
headers_file.seek(0)
headers_file.truncate()
headers_file.write(HEADERS[:block_bytes(10)])
headers_file.write(b'ops')
headers_file.write(HEADERS[block_bytes(10):])
await headers.open()
self.assertEqual(
cm.output, [
'WARNING:lbry.wallet.header:Reader file size doesnt match header size. '
'Repairing, might take a while.',
'WARNING:lbry.wallet.header:Header file corrupted at height 9, truncating '
'it.'
]
)
async def test_concurrency(self):
BLOCKS = 19
@@ -164,12 +206,13 @@ class TestHeaders(AsyncioTestCase):
await headers.connect(block_index, HEADERS[block_bytes(block_index):block_bytes(block_index + 1)])
async def reader():
for block_index in range(BLOCKS):
while len(headers) < block_index:
while len(headers) <= block_index:
await asyncio.sleep(0.000001)
assert headers[block_index]['block_height'] == block_index
assert (await headers.get(block_index))['block_height'] == block_index
reader_task = asyncio.create_task(reader())
await writer()
await reader_task
await headers.close()
HEADERS = unhexlify(
+35 -18
View File
@@ -33,6 +33,13 @@ class MockNetwork:
self.get_transaction_called.append(tx_hash)
return self.transaction[tx_hash]
async def get_transaction_and_merkle(self, tx_hash, known_height=None):
tx = await self.get_transaction(tx_hash)
merkle = {}
if known_height:
merkle = await self.get_merkle(tx_hash, known_height)
return tx, merkle
class LedgerTestCase(AsyncioTestCase):
@@ -41,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase):
'db': Database(':memory:'),
'headers': Headers(':memory:')
})
self.ledger.headers.checkpoints = {}
await self.ledger.headers.open()
self.account = Account.generate(self.ledger, Wallet(), "lbryum")
await self.ledger.db.open()
@@ -67,12 +76,17 @@ class LedgerTestCase(AsyncioTestCase):
serialized = self.make_header(**kwargs)
self.ledger.headers.io.seek(0, os.SEEK_END)
self.ledger.headers.io.write(serialized)
self.ledger.headers._size = None
self.ledger.headers._size = self.ledger.headers.io.seek(0, os.SEEK_END) // self.ledger.headers.header_size
class TestSynchronization(LedgerTestCase):
async def test_update_history(self):
txid1 = '252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792'
txid2 = 'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9'
txid3 = 'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0'
txid4 = '047cf1d53ef68f0fd586d46f90c09ff8e57a4180f67e7f4b8dd0135c3741e828'
account = Account.generate(self.ledger, Wallet(), "torba")
address = await account.receiving.get_or_create_usable_address()
address_details = await self.ledger.db.get_address(address=address)
@@ -83,46 +97,49 @@ class TestSynchronization(LedgerTestCase):
self.add_header(block_height=2, merkle_root=b'abcd04')
self.add_header(block_height=3, merkle_root=b'abcd04')
self.ledger.network = MockNetwork([
{'tx_hash': 'abcd01', 'height': 0},
{'tx_hash': 'abcd02', 'height': 1},
{'tx_hash': 'abcd03', 'height': 2},
{'tx_hash': txid1, 'height': 0},
{'tx_hash': txid2, 'height': 1},
{'tx_hash': txid3, 'height': 2},
], {
'abcd01': hexlify(get_transaction(get_output(1)).raw),
'abcd02': hexlify(get_transaction(get_output(2)).raw),
'abcd03': hexlify(get_transaction(get_output(3)).raw),
txid1: hexlify(get_transaction(get_output(1)).raw),
txid2: hexlify(get_transaction(get_output(2)).raw),
txid3: hexlify(get_transaction(get_output(3)).raw),
})
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd01', 'abcd02', 'abcd03'])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid1, txid2, txid3])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
address_details['history'],
'252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792:0:'
'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9:1:'
'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0:2:'
f'{txid1}:0:'
f'{txid2}:1:'
f'{txid3}:2:'
)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
for cache_item in self.ledger._tx_cache.values():
cache_item.tx.is_verified = True
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, [])
self.ledger.network.history.append({'tx_hash': 'abcd04', 'height': 3})
self.ledger.network.transaction['abcd04'] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.history.append({'tx_hash': txid4, 'height': 3})
self.ledger.network.transaction[txid4] = hexlify(get_transaction(get_output(4)).raw)
self.ledger.network.get_history_called = []
self.ledger.network.get_transaction_called = []
await self.ledger.update_history(address, '')
self.assertListEqual(self.ledger.network.get_history_called, [address])
self.assertListEqual(self.ledger.network.get_transaction_called, ['abcd04'])
self.assertListEqual(self.ledger.network.get_transaction_called, [txid4])
address_details = await self.ledger.db.get_address(address=address)
self.assertEqual(
address_details['history'],
'252bda9b22cc902ca2aa2de3548ee8baf06b8501ff7bfb3b0b7d980dbd1bf792:0:'
'ab9c0654dd484ac20437030f2034e25dcb29fc507e84b91138f80adc3af738f9:1:'
'a2ae3d1db3c727e7d696122cab39ee20a7f81856dab7019056dd539f38c548a0:2:'
'047cf1d53ef68f0fd586d46f90c09ff8e57a4180f67e7f4b8dd0135c3741e828:3:'
f'{txid1}:0:'
f'{txid2}:1:'
f'{txid3}:2:'
f'{txid4}:3:'
)
+14 -14
View File
@@ -21,9 +21,9 @@ def get_tx():
return Transaction().add_inputs([get_input()])
def get_channel(claim_name='@foo'):
async def get_channel(claim_name='@foo'):
channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc')
channel_txo.generate_channel_private_key()
await channel_txo.generate_channel_private_key()
get_tx().add_outputs([channel_txo])
return channel_txo
@@ -36,32 +36,32 @@ def get_stream(claim_name='foo'):
class TestSigningAndValidatingClaim(AsyncioTestCase):
def test_successful_create_sign_and_validate(self):
channel = get_channel()
async def test_successful_create_sign_and_validate(self):
channel = await get_channel()
stream = get_stream()
stream.sign(channel)
self.assertTrue(stream.is_signed_by(channel))
def test_fail_to_validate_on_wrong_channel(self):
async def test_fail_to_validate_on_wrong_channel(self):
stream = get_stream()
stream.sign(get_channel())
self.assertFalse(stream.is_signed_by(get_channel()))
stream.sign(await get_channel())
self.assertFalse(stream.is_signed_by(await get_channel()))
def test_fail_to_validate_altered_claim(self):
channel = get_channel()
async def test_fail_to_validate_altered_claim(self):
channel = await get_channel()
stream = get_stream()
stream.sign(channel)
self.assertTrue(stream.is_signed_by(channel))
stream.claim.stream.title = 'hello'
self.assertFalse(stream.is_signed_by(channel))
def test_valid_private_key_for_cert(self):
channel = get_channel()
async def test_valid_private_key_for_cert(self):
channel = await get_channel()
self.assertTrue(channel.is_channel_private_key(channel.private_key))
def test_fail_to_load_wrong_private_key_for_cert(self):
channel = get_channel()
self.assertFalse(channel.is_channel_private_key(get_channel().private_key))
async def test_fail_to_load_wrong_private_key_for_cert(self):
channel = await get_channel()
self.assertFalse(channel.is_channel_private_key((await get_channel()).private_key))
class TestValidatingOldSignatures(AsyncioTestCase):
+12 -12
View File
@@ -82,14 +82,14 @@ class TestSizeAndFeeEstimation(AsyncioTestCase):
class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
def test_is_my_account_not_set(self):
def test_is_my_output_not_set(self):
tx = get_transaction()
with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"):
_ = tx.net_account_balance
tx.inputs[0].txo_ref.txo.is_my_account = True
tx.inputs[0].txo_ref.txo.is_my_output = True
with self.assertRaisesRegex(ValueError, "Cannot access net_account_balance"):
_ = tx.net_account_balance
tx.outputs[0].is_my_account = True
tx.outputs[0].is_my_output = True
# all inputs/outputs are set now so it should work
_ = tx.net_account_balance
@@ -98,9 +98,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_account = True
tx.outputs[0].is_my_account = False
tx.outputs[1].is_my_account = True
tx.inputs[0].txo_ref.txo.is_my_output = True
tx.outputs[0].is_my_output = False
tx.outputs[1].is_my_output = True
self.assertEqual(tx.net_account_balance, -200*CENT)
def test_paying_from_other_account_to_my_account(self):
@@ -108,9 +108,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_account = False
tx.outputs[0].is_my_account = True
tx.outputs[1].is_my_account = False
tx.inputs[0].txo_ref.txo.is_my_output = False
tx.outputs[0].is_my_output = True
tx.outputs[1].is_my_output = False
self.assertEqual(tx.net_account_balance, 190*CENT)
def test_paying_from_my_account_to_my_account(self):
@@ -118,9 +118,9 @@ class TestAccountBalanceImpactFromTransaction(unittest.TestCase):
.add_inputs([get_input(300*CENT)]) \
.add_outputs([get_output(190*CENT, NULL_HASH),
get_output(100*CENT, NULL_HASH)])
tx.inputs[0].txo_ref.txo.is_my_account = True
tx.outputs[0].is_my_account = True
tx.outputs[1].is_my_account = True
tx.inputs[0].txo_ref.txo.is_my_output = True
tx.outputs[0].is_my_output = True
tx.outputs[1].is_my_output = True
self.assertEqual(tx.net_account_balance, -10*CENT) # lost to fee
+1
View File
@@ -11,6 +11,7 @@ commands =
--global-option=fetch \
--global-option=--version --global-option=3.30.1 --global-option=--all \
--global-option=build --global-option=--enable --global-option=fts5
pip install lbry-libtorrent
orchstr8 download
blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs}
datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs}