Compare commits

..
901 Commits
Author SHA1 Message Date
Jack 9c82689f65 Bump version: 0.3.19 -> 0.3.20 2016-08-21 04:04:11 -04:00
Jack Robison ce2755467d Merge pull request #133 from sonatagreen/patch-1
spelling fix
2016-08-21 03:44:10 -04:00
Jack Robison 02b19402ae Merge pull request #116 from lbryio/reflector
Backend for pushing newly created files to a designated server
2016-08-21 03:43:03 -04:00
Jack 80af66e852 Merge branch 'master' into reflector
# Conflicts:
#	lbrynet/core/log_support.py
2016-08-21 01:12:29 -04:00
Jack 15cfa45646 remove duplicate function 2016-08-21 00:58:25 -04:00
Jack Robison ac63849b49 Merge pull request #132 from lbryio/fix-get-history
fix get_history in lbrycrdwallet
2016-08-21 00:07:41 -04:00
Jack Robison 02142e352b Merge pull request #127 from lbryio/disable-more-noisy-loggers
reduce logging to sub fire hazard levels
2016-08-21 00:07:19 -04:00
Jack 3d92413372 debug logging 2016-08-20 23:47:41 -04:00
Jack 5022ed2acc blob_completed before moving to next one
otherwise blob files download to the blobfiles directory, but they are
never recorded in blobs.db
2016-08-20 23:47:12 -04:00
Sonata Green 31b9d22649 spelling fix 2016-08-19 21:12:02 -05:00
Jack 7a54894015 debugging stuff 2016-08-19 02:41:23 -04:00
Jack 7100345a09 Merge branch 'master' into reflector 2016-08-18 22:42:27 -04:00
Jack 929db15214 fix get_history in lbrycrdwallet 2016-08-18 22:15:49 -04:00
Jack f6473e3383 Bump version: 0.3.18 -> 0.3.19 2016-08-18 06:41:17 -04:00
Jack Robison 3cd9d6a604 Merge pull request #129 from kimihiro64/patch-2
Proper no-cache for web-ui index.html
2016-08-18 06:39:11 -04:00
Jack 142281777e stop heartbeat looping call on shutdown 2016-08-18 06:37:57 -04:00
Jack 8e7b8af180 call blob_completed when is_validated is true 2016-08-18 06:23:27 -04:00
Jack c046cd02d5 call blob_completed after receiving blob 2016-08-18 05:58:13 -04:00
Jack f5508fcdf1 typo 2016-08-18 05:36:17 -04:00
Jack adc2eab6da log request dict 2016-08-18 05:36:01 -04:00
Jack 702698d8aa stop heartbeat looping call on shutdown 2016-08-18 05:25:23 -04:00
kimihiro64 e7158e5a4b Fix editor spacing again 2016-08-17 23:02:38 -05:00
kimihiro64 c66df259a6 fix editor spacing 2016-08-17 23:02:00 -05:00
kimihiro64 0b8437fa12 Proper no-cache for web-ui index.html 2016-08-17 22:34:20 -05:00
Jack a32b6ebf82 add reflect jsonrpc command 2016-08-17 21:33:41 -04:00
Jack edbda1696e logging fire hazard 2016-08-17 20:24:48 -04:00
Jack ece218785c reduce logging to sub fire hazard levels 2016-08-17 20:20:03 -04:00
Jack cc1b3609a1 Merge branch 'master' into reflector
# Conflicts:
#	lbrynet/__init__.py
2016-08-17 19:01:38 -04:00
Alex Grintsvayg 0841aad07a fix double-tab bug on linux 2016-08-17 14:30:56 -04:00
Jack 975e83a015 Bump version: 0.3.17 -> 0.3.18 2016-08-17 02:39:48 -04:00
Jack 1ddb02dbb7 speed up search
get results for get_est_cost from lighthouse instead of manually trying
unless force=True
2016-08-17 02:38:59 -04:00
Jack Robison 1159204ed8 Merge pull request #126 from lbryio/pretty-dmg
Make a pretty dmg
2016-08-17 01:36:44 -04:00
Jack Robison a5212e7741 Merge pull request #121 from lbryio/save-settings
Save settings
2016-08-17 01:34:50 -04:00
Jack Robison 1932fd72e3 Merge pull request #123 from lbryio/add-heartbeat
Add basic analytics api and heartbeat event
2016-08-17 01:34:31 -04:00
Jack Robison c061fe091b Merge pull request #122 from lbryio/add-debug-logging
Add debug logging
2016-08-17 01:33:07 -04:00
Jack Robison 3621983a4d Merge pull request #119 from lbryio/update-protobuf
Update protobuf
2016-08-17 01:31:48 -04:00
Job Evers-Meltzer f5dc213e53 need to not be in build dir 2016-08-16 15:16:55 -05:00
Job Evers-Meltzer 138e9c1486 throw more things at wall 2016-08-16 15:03:41 -05:00
Job Evers-Meltzer 7053c0dc93 argh. so much hate 2016-08-16 14:52:08 -05:00
Job Evers-Meltzer ee8a653b13 argh. so much hate 2016-08-16 14:43:49 -05:00
Job Evers-Meltzer 71fad7e97f also need wheel 2016-08-16 13:41:04 -05:00
Job Evers-Meltzer 04740eb928 need to detect if ON_TRAVIS earlier 2016-08-16 13:24:38 -05:00
Job Evers-Meltzer 8c63d58c96 tmp: debug travis 2016-08-16 13:21:38 -05:00
Job Evers-Meltzer 991672d423 move pyobj building earlier 2016-08-16 13:16:25 -05:00
Job Evers-Meltzer 55ce38cc87 temp: debug travis 2016-08-16 12:52:55 -05:00
Job Evers‐Meltzer 96f3390407 Merge pull request #120 from lbryio/defer-later
replace callLater with deferLater
2016-08-16 12:36:43 -05:00
Job Evers-Meltzer aa2426d4e4 ignore DS_Store files 2016-08-16 12:26:19 -05:00
Job Evers-Meltzer d0e3e13852 ignore osx build artifacts 2016-08-16 12:25:35 -05:00
Job Evers-Meltzer a2a1e30d68 remove unnecessary libgmp 2016-08-16 12:25:18 -05:00
Job Evers-Meltzer a6f08b9e0c Prettifies the osx dmg
Adds a new background image, along with precisely located icon files.
Also updates the icon files to have white backgrounds. The icon files were
created using the 256px icon used in the ubuntu build.
2016-08-16 12:22:14 -05:00
Job Evers 9a738c3783 Add basic analytics api and heartbeat event 2016-08-12 13:37:26 -05:00
Jack 6b4519e959 squelch very verbose dht error log 2016-08-12 10:46:05 -05:00
Job Evers-Meltzer 9b1d5213a6 improve logging 2016-08-12 10:44:37 -05:00
Job Evers-Meltzer f9ed9a4505 fixup logging some more 2016-08-12 10:42:31 -05:00
Job Evers-Meltzer 3ed7ce5630 Ensure settings are actually committed
Without committing or closing the db, settings were not being saved.

Also, adds a run_in_thread decorator to simplify the code.
2016-08-12 10:08:21 -05:00
Job Evers 63cd8343c4 replace callLater with deferLater 2016-08-11 13:41:42 -05:00
Jack 2d5f8aed24 whitespace 2016-08-11 12:38:10 -04:00
Jack b8d23d0965 add announce_all_blobs_to_dht 2016-08-11 12:36:13 -04:00
Job Evers-Meltzer f4ef92e653 fixup logging some more 2016-08-11 05:37:45 +00:00
Job Evers-Meltzer c7430f4ae9 better more logging on the server side 2016-08-11 05:25:45 +00:00
Job Evers-Meltzer 623fda3087 improve logging 2016-08-11 05:16:50 +00:00
Job Evers-Meltzer 80f27f2b29 Revert "more logging on the server side"
This reverts commit f83daa5a22.
2016-08-11 05:11:18 +00:00
Jack defbd73b4f squelch very verbose dht error log 2016-08-11 01:07:51 -04:00
Jack 5285982515 resolve reflector ip 2016-08-11 01:07:51 -04:00
Job Evers-Meltzer f83daa5a22 more logging on the server side 2016-08-11 04:45:25 +00:00
Job Evers-Meltzer 0a07c8f131 log on closed/lost connection 2016-08-10 19:39:28 -05:00
Job Evers-Meltzer 35481a92f5 actually cleanup patched time.time 2016-08-10 19:36:52 -05:00
Job Evers-Meltzer 0995d864e8 connectTCP wants an ip address, not a url 2016-08-10 19:09:20 -05:00
Job Evers-Meltzer 1a42e43573 log all the things 2016-08-10 19:06:26 -05:00
Jack 132aa569fe move run_reflector_server into settings file and add debug lines 2016-08-10 16:51:46 -04:00
Job Evers-Meltzer e93a787bd6 update protobuf out of beta 2016-08-10 09:28:13 -05:00
Job Evers-Meltzer d6f902653f put import in alphabetical order 2016-08-10 08:35:15 -05:00
Job Evers-Meltzer d172d43ddf add reflector functional test to travis 2016-08-10 08:29:44 -05:00
Job Evers-Meltzer 67909724e7 fix up import paths 2016-08-10 07:44:41 -05:00
Jack 1c88cbc5f6 Merge branch 'master' into reflector 2016-08-10 02:31:28 -04:00
Jack 3fb4831804 raise exception when curr_pos > 100 and json fails to decode 2016-08-10 02:30:41 -04:00
Jack b65570fb16 drop connection on response longer than 100 bytes that can't' be decoded 2016-08-09 19:53:13 -04:00
Jack 871e6e6f64 remove files that didn't fix test 2016-08-09 17:53:34 -04:00
Jack 0cd92a96c0 add reflector files to tests directory 2016-08-09 17:46:25 -04:00
Jack ff31ddd2d5 Bump version: 0.3.16 -> 0.3.17 2016-08-09 14:40:18 -04:00
Jack e9f5b43785 fix exceptions 2016-08-09 14:29:59 -04:00
Jack 36b41081ab Bump version: 0.3.15 -> 0.3.16 2016-08-09 12:41:11 -04:00
Jack Robison 1092313892 Merge pull request #118 from lbryio/add-LBRYDaemonRequest
add LBRYDaemonRequest to os x app
2016-08-09 12:35:08 -04:00
Jack Robison ce605cb8f3 Merge pull request #117 from lbryio/update-lbrycrdd
use most recent lbrycrdd release on github
2016-08-09 12:32:40 -04:00
Jack 0733d885fb start reflector upload in publish 2016-08-09 12:26:34 -04:00
Jack ef6fe3d1d9 add variable to enable running reflector server 2016-08-09 12:18:46 -04:00
Jack 82a4fea81a fix log statement 2016-08-09 12:12:40 -04:00
Jack 14a0252819 hooking up reflector plumbing 2016-08-09 12:07:26 -04:00
Job Evers-Meltzer 849aeb65f0 use most recent lbrycrdd release on github 2016-08-09 11:03:40 -05:00
Jack a99dc6e93d add LBRYDaemonRequest to os x app 2016-08-09 11:19:01 -04:00
Alex Liebowitz 1a5fcc1348 Add requests_futures dependency 2016-08-09 07:59:27 -04:00
Jack c9d88f5628 Bump version: 0.3.14 -> 0.3.15 2016-08-09 03:29:51 -04:00
Jack Robison 0910711aca Merge pull request #111 from lbryio/update-claim
claim updates
2016-08-09 03:29:12 -04:00
Jack 6a1a9ef6bc encode claim id to be shown in log 2016-08-09 03:25:20 -04:00
Jack ef1c059079 clean up log line 2016-08-09 03:18:44 -04:00
Jack a05ab37373 Merge branch 'master' into update-claim
# Conflicts:
#	lbrynet/lbrynet_daemon/LBRYDaemon.py
2016-08-09 03:09:06 -04:00
Jack 7ea88cfd22 crudely fix loggly import problem 2016-08-09 03:01:33 -04:00
Jack Robison 8d14fe6a90 Merge pull request #113 from lbryio/log-to-loggly
Log to loggly
2016-08-09 02:30:55 -04:00
Jack Robison 49a05b42f9 Merge pull request #114 from lbryio/no-unused-imports
Remove all unused imports; add check to pylint
2016-08-09 02:30:49 -04:00
Jack b6da1d7769 decode and reverse claim id for updateclaim in lbryum 2016-08-09 01:26:55 -04:00
Jimmy Kiselak 7e2ad58edd get reflector client and server working; add func test to prove it 2016-08-09 00:59:50 -04:00
Jack 7ad0d98404 Merge branch 'master' into update-claim 2016-08-08 21:39:41 -04:00
Jack b7052bba95 rename variable 2016-08-08 21:34:05 -04:00
Jack 4a5618bcc4 get update claim working with lbryumwallet 2016-08-08 21:32:39 -04:00
Job Evers-Meltzer fccdf32c9f Remove all unused imports; add check to pylint 2016-08-08 15:42:35 -05:00
Jack c474411e1b delete bad records that got into name_metadata 2016-08-08 16:35:55 -04:00
Job Evers-Meltzer 9033eac5b1 Remove commented code 2016-08-08 15:07:25 -05:00
Job Evers-Meltzer 0ba0426591 Refactor lbrycrdd path function for osx 2016-08-08 15:06:48 -05:00
Job Evers-Meltzer 9a9b540701 Move some ServerRequestHandling logging to debug level 2016-08-08 15:04:50 -05:00
Job Evers-Meltzer b8ddbb9a26 Add loggly logger
It configured twice, once right at startup without the lbry id and then
after the settings database is available it gets reconfigured with the
lbry id and a session id.
2016-08-08 15:02:36 -05:00
Job Evers-Meltzer bc3c2b96ed disable noisy loggers 2016-08-08 14:59:30 -05:00
Job Evers-Meltzer 622dc87d84 Remove log handlers with the same name
Also renames functions to follow underscore convention
2016-08-08 14:57:49 -05:00
Jack 18edbdf404 squelch resolve_name logging 2016-08-08 14:58:31 -04:00
Job Evers-Meltzer 02a3eab693 add loggly dependency 2016-08-08 11:39:11 -05:00
Job Evers-Meltzer dded868a80 remove leveldb requirement 2016-08-08 11:37:01 -05:00
Jack 6fe878bdb9 make load_from_hex a class method 2016-08-08 03:18:32 -04:00
Jack 505ece9e36 add txid param to get_claim_info and update todo note 2016-08-08 02:37:08 -04:00
Jack 72acfa1e0c add support_claim 2016-08-08 02:32:56 -04:00
Jack 02cd8b9a44 fix callback 2016-08-08 02:08:09 -04:00
Jack f32250f431 whoops 2016-08-08 01:54:41 -04:00
Jack 1523cad044 refactor get_claim_info 2016-08-08 01:46:19 -04:00
Jack ed55353700 bug fix 2016-08-08 01:43:28 -04:00
Jack 8d3bd45102 get_claims_for_name daemon function 2016-08-08 01:43:10 -04:00
Jack f69753f6a0 add get_claims_for_name 2016-08-08 01:42:37 -04:00
Jimmy Kiselak b7e2e87ac1 fill in much of the skeleton for the reflector protocol 2016-08-07 22:33:40 -04:00
Jack 3b98508986 clean up claim_name function 2016-08-07 16:13:47 -04:00
Jack 36ebfc42d9 update from master 2016-08-06 21:44:14 -04:00
Jack ae5f55ff65 Bump version: 0.3.13 -> 0.3.14 2016-08-06 20:07:28 -04:00
Jack Robison 01131da5b7 Merge pull request #99 from lbryio/refactor-refresh
Add DownloadNameHelper and ResolveNameHelper
2016-08-06 20:06:43 -04:00
Jack 66ae865de0 don't make new lbry file if publish is given a full metadata dict, remove update_claim() 2016-08-06 20:02:51 -04:00
Job Evers-Meltzer 8c50f08fec refactor _start_download 2016-08-05 11:37:24 -05:00
Job Evers-Meltzer e69de47daf refactor wait_on_write stopping condition 2016-08-05 11:37:24 -05:00
Job Evers-Meltzer acef10e848 fix merge oversight 2016-08-05 11:37:24 -05:00
Job Evers edb9d50d9e fix typo in docstring 2016-08-05 11:37:24 -05:00
Job Evers b989d4f6be Bug fix: correct the check on wait_or_get_stream
I reversed the check on lbry_file. Compare to the original code:
https://github.com/lbryio/lbry/blob/e67f55bd7447cbee730d895b7f35cf53cf69dba5/lbrynet/lbrynet_daemon/LBRYDaemon.py#L1119
2016-08-05 11:37:24 -05:00
Job Evers 84dd5c0eca Revert "revert get() refactor"
This reverts commit 5eb7c64ec3.
2016-08-05 11:37:24 -05:00
Job Evers-Meltzer db8daf20b7 quiet noisy log statement 2016-08-05 05:50:24 -05:00
Jack cb985b5a03 log conditions of lbry_file failing to start 2016-08-04 23:27:44 -04:00
Jack 28c4e38272 make line neater 2016-08-04 23:27:25 -04:00
Jack b29377dc2b keep claim id history 2016-08-04 23:27:10 -04:00
Jack ca09592a72 make logs easier to scroll through 2016-08-04 23:10:15 -04:00
Jack b0e14fae43 update over existing claim in publish() 2016-08-04 23:08:54 -04:00
Jack dfa68f126b get update_name in lbrycrdwallet working 2016-08-04 20:43:39 -04:00
Jack 4570f68727 add claim_id to lbry_file and to get_claim_info 2016-08-04 18:44:12 -04:00
Jack 4705acba55 use lbryum updateclaim 2016-08-04 02:22:10 -04:00
Jimmy Kiselak 1dea20a358 include sd blob hashes in list of blob hashes to send via reflector 2016-08-03 23:11:29 -04:00
Jimmy Kiselak 191f661b35 skeleton for reflector protocol 2016-08-03 23:03:14 -04:00
Jack f3ab38ba8f Bump version: 0.3.12 -> 0.3.13 2016-08-03 18:01:18 -04:00
Jack Robison 2d0a892e70 Merge pull request #110 from lbryio/lighthouse-client
Lighthouse client
2016-08-03 18:00:04 -04:00
Jack Robison 00eefe87d7 Merge pull request #109 from lbryio/download-descriptor
add download_descriptor
2016-08-03 17:59:52 -04:00
Jack f68fd96555 break out sd timeout into a variable 2016-08-03 17:57:26 -04:00
Jack fc50a3242f add note about request.notifyFinish 2016-08-03 17:49:13 -04:00
Jack 64e9ce2bfa comments about the upnp redirects 2016-08-03 17:44:41 -04:00
Jack e8d04b1921 specify a lighthouse server 2016-08-03 17:44:32 -04:00
Jack 9fedf14e55 request.notifyFinish 2016-08-03 12:46:46 -04:00
Jack 85d610bcdf add get_peers_for_hash 2016-08-03 03:16:06 -04:00
Jack 3af8b7bd0c use upnp redirects if they're already set 2016-08-03 01:27:40 -04:00
Jack 6215d85581 add LighthouseClient, announce sd hash after publishing 2016-08-02 18:34:48 -04:00
Jack 308d9f5abd add todo 2016-08-02 17:33:57 -04:00
Jack bcfabba038 fast search
use cost from lighthouse
2016-08-02 16:45:06 -04:00
Jack 69139b1987 already called error 2016-08-02 16:03:47 -04:00
Jack 3f8f9d9d73 add timeout to download_sd_blob 2016-08-02 02:54:04 -04:00
Jack 2283579109 reorganize download_descriptor 2016-08-01 23:15:32 -04:00
Jack 1e0af94ba2 add download_descriptor 2016-08-01 22:31:49 -04:00
Jack bbf7940e45 Bump version: 0.3.11 -> 0.3.12 2016-07-29 23:44:04 -04:00
Jack Robison f5af71f4e6 Merge pull request #107 from lbryio/metadata-version
Metadata versioning, publishing things, and USD denominated key fees
2016-07-29 23:43:11 -04:00
Jack 1ffcd67c9d specify exception 2016-07-29 23:05:01 -04:00
Jack cf8592ba12 docstring and todo 2016-07-29 22:49:50 -04:00
Jack 1e1d036590 clean log line 2016-07-29 22:44:38 -04:00
Jack 64afb6d305 clean up get_est_costs in search 2016-07-29 22:43:50 -04:00
Jack 20a9e018f7 revert updates to git 2016-07-29 22:36:30 -04:00
Jack a574c33ac6 multiple search servers 2016-07-29 21:37:44 -04:00
Jack d21e458fe1 remove debug line 2016-07-29 02:42:58 -04:00
Jack bb3ba53c6e fix search results 2016-07-28 22:39:34 -04:00
Jack ba2e4df76b remove debug stuff 2016-07-28 21:31:19 -04:00
Jack 33e7e4bee3 fancy search 2016-07-28 21:25:37 -04:00
Jack b83dce2cb3 no tricky characters 2016-07-28 18:48:29 -04:00
Jack 6821676853 handle decimals in jsonrpc return 2016-07-28 18:12:54 -04:00
Jack 52024986ee derp 2016-07-28 16:32:59 -04:00
Jack 2e3342d20d fix test 2016-07-28 16:27:16 -04:00
Jack 419aa1d78f fix est data cost / search 2016-07-28 16:12:20 -04:00
Jack de4ddd578f improve readability of metadata and fee requirements 2016-07-28 15:24:20 -04:00
Jack 5dd68a7ed4 add get_claim_info, clean up _get_stream_info_from_value 2016-07-28 14:55:17 -04:00
Jack 881168dc7a fix too many args 2016-07-28 13:16:33 -04:00
Job Evers-Meltzer 295e427abe bug fix: check if fee is in metadata before loading 2016-07-28 12:04:25 -05:00
Job Evers 67c5d15f51 stupid typo 2016-07-28 11:46:33 -05:00
Job Evers 39a7f37f35 muck with pending_claims... still not sure this is correct 2016-07-28 11:43:47 -05:00
Job Evers 15d276b0bc fix case where to and from currencies are the same 2016-07-28 11:43:20 -05:00
Jack 4387025c48 another still 2016-07-28 06:07:33 -04:00
Jack 52c20f02c7 one more 2016-07-28 06:03:47 -04:00
Jack 413dab8094 whoops 2016-07-28 05:59:10 -04:00
Jack 6b1d11a04d fix rate in test 2016-07-28 05:53:31 -04:00
Jack 5441379f38 fix LBRYExchangeRateManager test 2016-07-28 05:49:31 -04:00
Jack 52d954c64e don't spam the market apis 2016-07-28 05:34:20 -04:00
Jack 415495fc16 LBRYExchangeRateManager 2016-07-28 05:30:13 -04:00
Jack 4223298634 test wrong metadata version 2016-07-28 02:49:35 -04:00
Jack e058c71019 test data 2016-07-28 02:46:46 -04:00
Jack 12b2bbfa26 make sure claimed ver is true 2016-07-28 02:36:07 -04:00
Jack 7d22bfffa5 fee tests 2016-07-28 02:30:37 -04:00
Jack 4ade80c8a9 clean up metadata and fee 2016-07-28 02:19:45 -04:00
Jack 9674381fc2 Merge remote-tracking branch 'origin/metadata-version-avec-tests' into metadata-version 2016-07-27 20:18:00 -04:00
Jack f6d91280d7 remove debug print line 2016-07-27 15:03:03 -04:00
Jack e4ebbd8f69 neater force param in resolve_name 2016-07-27 15:02:54 -04:00
Jack 7f5560de6d comment spacing and removing setLevel 2016-07-27 15:02:19 -04:00
Job Evers-Meltzer 0925171e07 bug fix: use txid attribute, not key 2016-07-27 13:59:26 -05:00
Job Evers-Meltzer 900d3ad01d Fixes errback not being callable
Small code refactoring to make it more obvious what
both the callback and errback are doing
2016-07-27 13:12:20 -05:00
Job Evers-Meltzer 6a1efedc1f some example tests. please expand on these 2016-07-27 12:05:29 -05:00
Job Evers-Meltzer 0f602e0e62 allow bittrex fee to be changed 2016-07-27 11:59:57 -05:00
Jack 3a425fc47d Merge branch 'master' into metadata-version 2016-07-27 05:07:23 -04:00
Jack 36722cbbd8 Bump version: 0.3.10 -> 0.3.11 2016-07-27 03:58:25 -04:00
Jack 9572336e96 Merge branch 'master' into metadata-version
# Conflicts:
#	lbrynet/core/utils.py
#	lbrynet/lbrynet_daemon/LBRYDaemon.py
2016-07-27 03:54:10 -04:00
Jack 4895cde474 fix conflicts 2016-07-27 03:48:32 -04:00
Jack Robison 21ca1a73cf Merge pull request #100 from lbryio/change-ui-location
web-ui files are on S3 now
2016-07-27 03:44:28 -04:00
Jack Robison 457c1c6ae6 Merge pull request #101 from lbryio/version-cmp
Properly Compare Version Strings
2016-07-27 03:43:13 -04:00
Jack Robison c1ff127101 Merge pull request #102 from lbryio/log-configure
Fix: Log configuration should not happen when a module is imported
2016-07-27 03:42:51 -04:00
Jack 5e815a5832 merge master 2016-07-27 03:22:03 -04:00
Jack 4604e9cd54 check pending claims after one is made 2016-07-27 03:00:44 -04:00
Jack ef580a7680 errback for pending of date lbry_file 2016-07-27 01:51:09 -04:00
Jack 2102776be8 another errback 2016-07-27 01:44:01 -04:00
Jack f3af7a6b77 add errback for unknown name 2016-07-27 01:39:23 -04:00
Jack 5abeceac1f abandon previous claim in update 2016-07-27 01:13:41 -04:00
Jack 22f73a0817 add force parameter to resolve_name 2016-07-26 21:46:04 -04:00
Jack 29b5aef73a call restore() after claiming name 2016-07-26 21:24:58 -04:00
Jack 3e602bce52 fix publish callback 2016-07-26 20:58:37 -04:00
Jack fe39901885 update claim when publishing to an already claimed name
-also save name_metadata for each unique claim rather than for each name
2016-07-26 20:52:05 -04:00
Jack ee9e8b365c add debug log line 2016-07-26 18:42:51 -04:00
Jack bd6fa35d4a disable verbose dht error 2016-07-26 18:19:40 -04:00
Jack 5e1fdf66b0 fix download problem 2016-07-26 18:18:34 -04:00
Jack d86e709b90 reduce min blob prices 2016-07-26 17:24:59 -04:00
Jack f82cffe882 verbose logging 2016-07-26 17:21:41 -04:00
Jack 3d88eb92c1 change default max fee to $25 2016-07-26 17:21:03 -04:00
Jack 0811ebb52d fix key payment problems 2016-07-26 17:20:51 -04:00
Jack 491d431ea5 remove price updater from daemon 2016-07-26 17:19:26 -04:00
Job Evers‐Meltzer ceeebebcad Merge pull request #106 from lbryio/no-blobs-in-logs
Have TimeoutError hexlify blobs
2016-07-26 14:57:30 -05:00
Job Evers‐Meltzer 04d766aba7 Merge pull request #105 from lbryio/bug-fixes
misc bug fixes and code cleanup
2016-07-26 14:56:50 -05:00
Job Evers‐Meltzer 3b431e180e Merge pull request #98 from lbryio/log-cleanup
Remove datetime.now() from log statements
2016-07-26 14:54:36 -05:00
Job Evers a2eab1577d Have TimeoutError hexlify blobs 2016-07-26 13:54:46 -05:00
Job Evers-Meltzer f8dd3d05dc misc bug fixes and code cleanup 2016-07-26 12:24:25 -05:00
Job Evers-Meltzer 93993e62d6 Fix: Log configuration should not happen when a module is imported
Instead, move the responsibility to the main program. Also,
each module had the same, redundant setup.
2016-07-26 12:01:03 -05:00
Job Evers‐Meltzer 483c88b958 Merge pull request #104 from lbryio/no-print
Remove print line
2016-07-26 11:36:00 -05:00
Job Evers 4a1e846958 Remove print line
This line is responsible for flooding the console with messages like
<_io.BytesIO object at 0x7f9e286e52f0>
2016-07-26 11:34:13 -05:00
Job Evers-Meltzer 927b9afe7c forgot to add the test file 2016-07-26 09:34:00 -05:00
Jack 9e8827f015 resolve name bug 2016-07-25 23:46:18 -04:00
Jack ca45f461ae spelling 2016-07-25 23:42:03 -04:00
Jack 7e5af11c2d fix key fee payment problem 2016-07-25 23:41:26 -04:00
Jack 102436274f USD denominated max_key_fee in settings 2016-07-25 22:46:04 -04:00
Jack 49c26a76b4 remove fee checking from autofetcher script 2016-07-25 22:28:01 -04:00
Jack d469ef7bf2 add googlefinance to requirements 2016-07-25 22:21:38 -04:00
Jack a5374cb876 fix currencies and LBRYFeeFormat 2016-07-25 22:17:31 -04:00
Jack a53e911a0c stop looping call when stop() is called 2016-07-25 21:48:39 -04:00
Jack c4a78a149b LBRYFee things
-move exchange rate updates to wallet, do them every half hour
-add convention checker for fees
-pay BTC or USD denominated key fees in converted LBC amount
2016-07-25 21:45:42 -04:00
Job Evers-Meltzer 3814912adf Move to core.utils and expand to check lbryum too 2016-07-25 16:09:13 -05:00
Jack 75ef652cb3 add googlefinance to setup.py 2016-07-25 17:08:56 -04:00
Job Evers-Meltzer 1eae3fe5dc pylint doesn't like distutils 2016-07-25 14:53:07 -05:00
Job Evers-Meltzer d0bc383a79 bug fix: version comparison is not lexographic.
Use distutils.version to compare version strings.
2016-07-25 12:25:10 -05:00
Job Evers-Meltzer ea5d31b530 OCD: alphabetize and group imports
I'm not the only one:
https://google.github.io/styleguide/pyguide.html?showone=Imports_formatting#Imports_formatting
2016-07-25 12:19:19 -05:00
Job Evers‐Meltzer e56ad19e81 Merge pull request #95 from lbryio/deployment-update
Updates the deployment process and the version check associated with it
2016-07-25 12:10:29 -05:00
Job Evers ed393eb75c Updates the deployment process and the version check associated with it
This commit supports steps 1 and 2 in the new workflow:
1. Change the logic in the daemon to check the github api for the latest release that is not a pre release
2. Change travis to mark all releases as pre release
3. When we are ready to stage a release we push a tag to master. Travis builds the packages and releases them
4. We manually check them
5. Remove the pre release mark when we are happy
2016-07-25 11:13:47 -05:00
Jack 2e2d309d38 BTC/USD denominated fees, metadata revisions 2016-07-25 01:40:26 -04:00
Jack 6bfb33adb2 fix version comparison 2016-07-24 22:03:32 -04:00
Job Evers-Meltzer 05138e1db1 web-ui files are on S3 now 2016-07-22 10:58:37 -05:00
Jack b65402db41 debug line 2016-07-21 16:47:47 -04:00
Jack fbef187400 Merge branch 'master' into metadata-version
# Conflicts:
#	lbrynet/conf.py
#	lbrynet/lbrynet_daemon/LBRYDaemon.py
2016-07-21 16:45:41 -04:00
Jack 25d1b8b9e5 move towards better fees and metadata 2016-07-21 16:11:14 -04:00
Job Evers ec91594083 Remove datetime.now() from log statements 2016-07-21 10:41:26 -05:00
Jack e67f55bd74 disable debug logging 2016-07-20 22:40:55 -04:00
Jack Robison 276a7f9d2a Merge pull request #92 from lbryio/file-uploads
Add support for file uploads (for Publish page)
2016-07-20 22:37:08 -04:00
Jack 4888d7a567 Bump version: 0.3.9 -> 0.3.10 2016-07-20 22:34:16 -04:00
Jack Robison 767f78315b Merge pull request #97 from lbryio/lbryum-catchup
lbryum fixes
2016-07-20 22:33:16 -04:00
Jack 4311b218fb fix auto_connect 2016-07-20 22:13:22 -04:00
Jack 89892042dd remove unused line 2016-07-20 20:57:05 -04:00
Jack a22dfb3864 cleaner logging
from https://github.com/lbryio/lbry/pull/94
2016-07-20 20:56:56 -04:00
Jack f40c42dcf0 Merge branch 'master' into lbryum-catchup 2016-07-20 20:38:42 -04:00
Jack 1d31aa9da3 Bump version: 0.3.8 -> 0.3.9 2016-07-20 20:34:41 -04:00
Jack f999073fb4 revert download refactor
to be re-merged after fixes
2016-07-20 20:34:02 -04:00
Jack 5eb7c64ec3 revert get() refactor 2016-07-20 20:30:57 -04:00
Jack 7e79818555 Revert "Merge branch 'master' into lbryum-catchup"
This reverts commit a51576b250, reversing
changes made to 0bee2689d1.
2016-07-20 20:24:49 -04:00
Jack a51576b250 Merge branch 'master' into lbryum-catchup 2016-07-20 20:17:08 -04:00
Jack 0bee2689d1 download fixes 2016-07-20 20:17:01 -04:00
Jack c232743963 fix LBRYumWallet._do_send_many 2016-07-20 20:13:44 -04:00
Job Evers‐Meltzer c2d5f092b3 Merge pull request #94 from lbryio/logging-reformat-2
use asctime in logging format
2016-07-20 18:41:54 -05:00
Jack 8e8e186de9 Merge branch 'master' into lbryum-catchup 2016-07-20 19:29:11 -04:00
Jack 55f47a2b1b lbryum catchup 2016-07-20 17:32:15 -04:00
Job Evers-Meltzer 5c27b9eb90 use asctime in logging format
Instead of manually adding datetime.now() use
asctime formatter to do it for us.
2016-07-20 07:21:12 -05:00
Job Evers‐Meltzer 1d018533d1 Merge pull request #81 from lbryio/deprecate-lbry-console
deprecate lbry-console script
2016-07-20 07:07:16 -05:00
Job Evers‐Meltzer 34b8841f5c Merge pull request #73 from lbryio/refactor-dl-name
Refactor jsonrpc_get, _download_name, _resolve_name
2016-07-20 07:07:02 -05:00
Alex Liebowitz a54166a27c Add support for file uploads (for Publish page) 2016-07-20 02:36:55 -04:00
Job Evers 4adec39dce Bump version: 0.3.7 -> 0.3.8 2016-07-18 16:00:52 -05:00
Job Evers‐Meltzer e63ae08638 Merge pull request #86 from lbryio/back-to-lbryum
Revert default wallet back to lbryum
2016-07-18 15:33:03 -05:00
Job Evers‐Meltzer f1b4c6a1ec Cache pyobjc (#87)
* cache pyobjc modules as wheels
2016-07-18 14:50:05 -05:00
Job Evers-Meltzer 78b4b3ffa8 revert default wallet back to lbryum 2016-07-18 11:30:21 -05:00
Job Evers‐Meltzer f51fd48cdb Merge pull request #85 from lbryio/update-protobuf
Update protobuf library
2016-07-18 11:26:47 -05:00
Job Evers-Meltzer 9127a816e3 Update protobuf library
Updating to protobuf-v3.0.0b3 because it
fixes a bug here https://github.com/google/protobuf/blob/e841bac4fcf47f809e089a70d5f84ac37b3883df/python/google/protobuf/internal/python_message.py#L59
where six.moves is not always available.
In particular, the system python on OSX has an old version of six
which is without six.moves.
2016-07-18 10:12:49 -05:00
Job Evers-Meltzer 1a5cd9b062 Forget about trying to set a label.
Setting a label also sets the account, but the LBRY app
only looks at the default "" account.  Marking the imported
keys would have been nice, but it doesn't work.
2016-07-18 09:14:50 -05:00
Job Evers-Meltzer a86b4c4e58 bug fix: use sys.exit() not exit() 2016-07-18 08:52:56 -05:00
Job Evers-Meltzer a384960a7b deprecate lbry-console script 2016-07-18 08:45:28 -05:00
Job Evers‐Meltzer 5817b80c94 script to migrate private keys to lbrycrd (#79)
* script to migrate private keys to lbrycrd
* handle compressed keys too
* validate private key actually gets added
* add lbrycrdd and lbrycrd-cli checks
* exit if no wallet path
* move each imported address into the default account
2016-07-17 20:17:42 -05:00
Job Evers-Meltzer 97efd63c99 deprecate lbry-console script 2016-07-17 19:19:00 -05:00
Job Evers-Meltzer 5ebcc01bcc Bump version: 0.3.6 -> 0.3.7 2016-07-17 14:10:11 -05:00
Job Evers‐Meltzer ddcc008ea6 Merge pull request #78 from lbryio/force-wallet
emergency fix: force lbrycrd to be the default wallet
2016-07-17 14:08:41 -05:00
Job Evers-Meltzer 8ec10e2304 emergency fix: force lbrycrd to be the default wallet
Ignore save settings for wallet type and use the default wallet (lbrycrd)
instead.  lbryum can be used but only if specified on the command line.
2016-07-17 13:45:26 -05:00
Job Evers-Meltzer 172f275bc7 Refactor _resolve_name.
Continue using the delegation/helper pattern for the daemon.
2016-07-16 09:24:27 -05:00
Job Evers‐Meltzer 0dac5943bb Merge pull request #74 from llllIHARWINDERIllll/patch-3
Need to have git installed
2016-07-15 13:42:21 -05:00
Jack 2dcd2b13be add LBRYMetadata.py 2016-07-15 14:09:43 -04:00
Harwinder 8743decd17 Need to have git installed.
You cant execute the code that follows after if you havent installed git already.
2016-07-15 20:01:25 +02:00
Job Evers-Meltzer a15d7ca543 Refactor _download_name
Nested functions are the devil, especially ones that
use variables from the outer scope. Refactoring
_download_name to use a helper class helps make
the scoping more explicit and will undoubtably
prevent bugs in the future.

I think this makes _download_name drastically more readable.

Also cleaned up some duplicated code and
made download_directory respect the passed in parameter
instead of being the default.
2016-07-15 12:42:26 -05:00
Job Evers-Meltzer a90029ec50 update doc string 2016-07-15 11:37:04 -05:00
Job Evers-Meltzer 563896b126 fix bug in reveal code 2016-07-15 11:33:38 -05:00
Job Evers-Meltzer 5aecd02668 Refactor jsonrpc_get.
Move parameter handling into its own function and
better use the `.get()` function for dictionaries.

Early return on the failed checks is more readable.

The lambda function in the callback was long and
hard to read so moved it out.
2016-07-15 11:32:46 -05:00
Jeremy Kauffman eef6312a17 Merge pull request #71 from llllIHARWINDERIllll/patch-2
Cant use git if its not installed
2016-07-14 16:26:23 -04:00
Harwinder 1e48f5c8f4 Cant use git if its not installed
First install needed packages then execute the task. Cant run the whole code if git isnt installed :)
2016-07-14 22:25:32 +02:00
Jeremy Kauffman 7908fa2814 Merge pull request #70 from llllIHARWINDERIllll/patch-1
Adding git to linux install command
2016-07-14 16:16:51 -04:00
Harwinder 7ffb3ea042 Added git to linux install command
Some people dont have git installed so adding it to it solves confusion for people new to linux.
2016-07-14 18:41:23 +02:00
Job Evers‐Meltzer 842dd90780 Merge pull request #66 from lbryio/add-tests-blob-handler
add tests for BlobRequestHandler and run on travis
2016-07-14 09:33:06 -05:00
kimihiro64 ee71a03f96 Revert to 2 params as isinstance cannot support 3+ (#69)
* Revert to 2 params as isinstance cannot support 3+

* Use basestring for maximum efficiency
2016-07-13 22:44:02 -05:00
Jack 5221631b40 Bump version: 0.3.5 -> 0.3.6 2016-07-12 17:30:58 -04:00
Job Evers‐Meltzer ade2b4ac0d Merge pull request #68 from lbryio/switch-slack
switch slack notification token
2016-07-12 15:25:21 -05:00
Job Evers-Meltzer 643b95c0e4 switch slack notification token 2016-07-12 14:48:13 -05:00
Jack Robison ed0f9d4fb5 Merge pull request #67 from lbryio/real-lbryum
Real lbryum
2016-07-12 15:35:54 -04:00
Jack 6f353c31cb Merge branch 'master' into real-lbryum 2016-07-11 22:04:32 -04:00
Jack 404412a950 return publish error message 2016-07-11 21:57:34 -04:00
Job Evers‐Meltzer 0e1778d8da Merge pull request #65 from lbryio/add-slack-notifications
add slack notifications to travis
2016-07-11 13:28:56 -05:00
Job Evers-Meltzer dfc0ca9525 change slack key and encrypt it 2016-07-11 12:05:27 -05:00
Job Evers-Meltzer 53bf1fe4fd add send_blob tests 2016-07-10 21:56:00 -05:00
Jack 5c05daa007 update import to new file name 2016-07-10 22:21:50 -04:00
Job Evers-Meltzer f74f075b4e add tests for BlobRequestHandler and run on travis 2016-07-10 18:48:14 -05:00
Jack 1b53e8e98c re-enable lbryum 2016-07-09 13:31:07 -04:00
Job Evers-Meltzer 4caee284bc add slack notifications to travis 2016-07-07 12:06:07 -05:00
Alex Grintsvayg 5368c91399 lbry replaces old lbrynet 2016-07-07 00:18:29 -04:00
Alex Grintsvayg 00b1881f20 dont forget to run bumpversion instead of manually bumping 2016-07-06 22:56:23 -04:00
Jack b727bd20c8 version bump 2016-07-06 22:48:38 -04:00
Jack 5a703c116f linux lbrycrdd path 2016-07-06 22:46:59 -04:00
Alex Grintsvayg 57f728f40e desc update 2016-07-06 22:29:12 -04:00
Alex Grintsvayg 74bbde0c34 fix "launch as root" bug, update package name and maintainer 2016-07-06 21:54:55 -04:00
Alex Liebowitz c58fc0781c Call xdg-open properly in jsonrpc_reveal() 2016-07-06 15:57:00 -04:00
Alex Grintsvayg 7a266d49c1 fix 2016-07-06 15:33:06 -04:00
Alex Grintsvayg 344eeeb8c0 typo 2016-07-06 15:04:41 -04:00
Alex Grintsvayg 08d50ddfc5 bundle lbrycrdd bins instead of downloading them 2016-07-06 14:58:06 -04:00
Alex Grintsvayg f83c892b13 fix lbry symlink, fix icon path, add .desktop file to bumpversion 2016-07-06 14:13:11 -04:00
Alex Grintsvayg 7dbec9d395 typo 2016-07-06 13:41:47 -04:00
Alex Grintsvayg 5171c801d9 add lbry icons, symlink into user path 2016-07-06 13:34:17 -04:00
Alex Grintsvayg 72b4a778a6 dont forget bumpversion 2016-07-06 10:09:40 -04:00
Jack Robison 0254df63f9 Merge pull request #64 from lbryio/check-for-new-ui
Check for new ui periodically
2016-07-06 04:35:06 -04:00
Jack 03c02ff74f lbrycrdd path wierdness 2016-07-06 03:37:31 -04:00
Jack 9277c3c674 add help for when things go wrong 2016-07-06 03:02:55 -04:00
Jack 34b1259dc6 move check for if daemon is running to inside of function 2016-07-06 02:20:18 -04:00
Jack 257e38de12 add command line wrapper for lbrynet-daemon calls 2016-07-06 02:17:38 -04:00
Jack 1984132eae add update_available keys to version()
as to not require making the version string comparison in whatever is
making the daemon call
2016-07-06 01:46:49 -04:00
Jack 8b1d75ea98 check every 30 minutes for a new ui version
-previously the check for a new ui version was only at startup, which
resulted in bugs persisting after they’d been fixed

-add option to skip requirements check

-try to load an existing ui if it can’t be obtained from github
2016-07-06 01:27:25 -04:00
Job Evers‐Meltzer a79af11371 Merge pull request #60 from lbryio/change-make-deb
Use patched version of make-deb
2016-07-05 15:08:42 -05:00
Job Evers 087a9b70b4 Use patched version of make-deb
Ran into a bug in make-deb that fails when there is utf-8 characters
in the git commit message. https://github.com/nylas/make-deb/issues/13.

I wrote a patch to fix the bug and am installing that version until the
make-deb maintainers incorporate it
2016-07-05 14:57:48 -05:00
Alex Grintsvayg 8b37952128 make-deb doesn't like utf8 arrow in commit message 2016-07-05 15:53:51 -04:00
Alex Grintsvayg 8336930091 Bump version: 0.3.2 → 0.3.3 2016-07-05 15:20:35 -04:00
Alex Grintsvayg d875325cee better non-root fix, restore version var 2016-07-05 15:20:30 -04:00
Alex Grintsvayg f1c5d1cb23 Bump version: 0.3.1 → 0.3.2 2016-07-05 14:30:35 -04:00
Alex Grintsvayg b55752e715 add bumpversion script 2016-07-05 14:29:24 -04:00
Alex Grintsvayg 36a99e6ec8 run lbry as non-root user 2016-07-05 14:06:41 -04:00
Jack df3c5b9a77 open conf for write in app 2016-07-04 20:47:43 -04:00
Jack 4c2ab23c1f write to the file that's supposed to be written to 2016-07-04 20:19:04 -04:00
Jack 0a3f81e364 fix download directory not existing problem 2016-07-04 20:02:38 -04:00
Jack 69c5631b25 make lbrycrddpath.conf if it doesnt exist, point to binary in app 2016-07-04 19:53:01 -04:00
Jack cca4331ac4 create lbrycrddpath.conf if it doesnt exist 2016-07-04 19:34:51 -04:00
Alex Grintsvayg 5972dbb6a7 fix lbry init script 2016-07-04 19:18:22 -04:00
Job Evers-Meltzer 794f2d8576 version bump: 0.3.0.2 2016-07-04 16:51:29 -05:00
Alex Liebowitz e35dce530a Set default download dir to ~/Downloads on Linux 2016-07-04 17:49:48 -04:00
Job Evers-Meltzer 6034bd3cb2 version bump: 0.3.0.1 2016-07-04 15:47:12 -05:00
Job Evers‐Meltzer 42c77bbb67 Merge pull request #51 from lbryio/restart-lbrycrd
add a more useful exception if auth fails on lbrycrd
2016-07-04 15:45:44 -05:00
Job Evers‐Meltzer d504c8f4a3 Merge pull request #50 from lbryio/lbrycrd-in-bundle
Add lbrycrd to app bundle on macos
2016-07-04 15:42:24 -05:00
Job Evers-Meltzer 976c8b3ce0 add a more useful exception if auth fails on lbrycrd 2016-07-04 15:40:52 -05:00
Alex Liebowitz 905e7e41a6 Merge branch 'master' of https://github.com/lbryio/lbry 2016-07-04 16:26:35 -04:00
Job Evers-Meltzer 16f48d0400 Add lbrycrd to app bundle on macos 2016-07-04 15:18:08 -05:00
Alex Liebowitz 6d7608d851 Create lbrycrd.conf if needed 2016-07-04 16:16:34 -04:00
Jack 7cd30efbe9 download lbrycrd binaries and set up lbrycrddpath.conf 2016-07-04 08:41:44 -04:00
Jack Robison 8c3652e1ac Merge pull request #49 from lbryio/real-lbrycrd2
real blockchain
2016-07-04 05:48:49 -04:00
Jack 7b9e3aace9 use lbrycrd 2016-07-04 05:43:26 -04:00
Jack 568737e123 change lbryum version constant name 2016-07-04 05:15:53 -04:00
Jack 808e3ea6e3 rename LBRYcrdWallet.py to LBRYWallet.py and change default wallet to lbrycrd 2016-07-04 04:57:30 -04:00
Jack e4f14d871f update setup.py 2016-07-01 03:36:51 -04:00
Jack 1cedd8ee5b update name 2016-07-01 03:16:08 -04:00
Jack b80fe24179 tanstaafl 2016-07-01 02:55:20 -04:00
Jack 3cea41a854 remove lbrynet_gui 2016-07-01 02:42:34 -04:00
Jack 62b80c13cc version bump 2016-06-30 01:26:33 -04:00
Jack 08b612575c update_claim fix and clean up publish keys
-replace ‘value’ key with ‘metadata’

-update claim metadata in db when an update is published
2016-06-28 23:20:28 -04:00
Jack b3b804bf2d add update_name 2016-06-28 14:28:59 -04:00
Jack 09d8a86e30 rename variable 2016-06-28 02:06:04 -04:00
Jack cc5d917c72 download fix 2016-06-28 01:51:05 -04:00
Jack 7cc3e9d5ef publish updates
-adds a base set of metadata fields required for results to be rendered
in conf.py, including language and content-type

-dont support old style claims on the new blockchain
2016-06-27 17:07:59 -04:00
Job Evers‐Meltzer 87aa9fc6c4 Merge pull request #48 from lbryio/add-cache
add caching for travis
2016-06-27 11:50:23 -05:00
Job Evers 8d5c315817 add caching for travis 2016-06-27 11:21:30 -05:00
Jack f89486f5fe port and import name 2016-06-24 13:12:27 -04:00
Job Evers‐Meltzer cf4d78a24c Merge pull request #47 from lbryio/fix-osx-build
Fix OSX build issue with openssl
2016-06-23 16:35:13 -05:00
Job Evers-Meltzer ee88970075 use the python installer from python.org
The python from brew links to brew's version of openssl
while the python one works with what osx has by default.
We need the later or else the resultiing app bundle
only works on machines that have brew's version of
openssl.
2016-06-23 15:24:19 -05:00
Job Evers‐Meltzer f5fa40feb7 Merge pull request #44 from lbryio/new-osx-container
Use a more up-to-date OSX version
2016-06-23 08:52:03 -05:00
Job Evers-Meltzer da13192cfe Use a more up-to-date OSX version
Also related:
 - install framework version of python
 - ensure unencrypted keys are never accidently added to repo
2016-06-22 23:23:26 -05:00
Job Evers‐Meltzer 8ff65a65db Merge pull request #43 from lbryio/no-libgmp
No longer swap libgmp
2016-06-21 18:07:24 -05:00
Job Evers-Meltzer 2d882a11a5 No longer swap libgmp
The correct libgmp builds on travis so we don't need to swap it out anymore
2016-06-21 16:21:46 -05:00
Jack Robison 79adc677a1 add --log-to-console 2016-06-21 16:58:08 -04:00
Jack Robison a8e2dad6d0 fix link 2016-06-21 16:55:52 -04:00
Jack Robison 16954c5626 add FAQ 2016-06-21 16:54:22 -04:00
Job Evers-Meltzer 56305c4ede allow tags that start with test to pass 2016-06-21 15:05:15 -05:00
Job Evers‐Meltzer 30713cc21d Merge pull request #42 from lbryio/osx-pyobjc
Install pyobjc dependencies
2016-06-21 14:45:43 -05:00
Job Evers-Meltzer 2bac0a25e0 install pyobjc dependencies 2016-06-21 13:51:53 -05:00
Job Evers-Meltzer c3aa5b0515 restore version check 2016-06-21 01:36:03 -05:00
Job Evers-Meltzer 30afa43ed4 how about two providers? 2016-06-21 01:24:34 -05:00
Job Evers-Meltzer f2e66abe67 can there be two deploys? 2016-06-21 01:08:06 -05:00
Job Evers-Meltzer cdb311a501 build dmg from LBRY.app 2016-06-21 00:19:40 -05:00
Job Evers-Meltzer 4c2b3e157c deactivate version check so that I can do a test release 2016-06-20 22:57:04 -05:00
Job Evers-Meltzer 52625b710b trying to release linux and OSX files 2016-06-20 22:55:55 -05:00
Job Evers‐Meltzer b411bb4e85 Merge pull request #41 from lbryio/add-osx-certs
Add osx certs
2016-06-20 21:39:41 -05:00
Job Evers-Meltzer f16111369f Update build script
setup_uri_handler.py has moved, update build script to reflect
 again, setup_uri_handler changed location
 remove pipe to /dev/null
 try updating openssl
 try deep signing
 another file renamed
 install a dependency
 prereqs
 install requirements
 install certifi
 only install requirements on osx
 thin binaries only if they're actually fat
2016-06-20 20:45:28 -05:00
Job Evers-Meltzer 628d052749 move contents of lbry-osx-app into lbry repo 2016-06-20 20:43:48 -05:00
Job Evers-Meltzer 2e26bb0e57 remove submodule 2016-06-20 20:43:47 -05:00
Job Evers 652c7d72f6 Add encrypted certificate needed for code signing
This is a cleaned up version of commit c5dc06962d
  add OSX build to travis matrix
  fix the OSX build not failing
  add encrypted certificate and identity file
  Update submodule to lbry-osx-app/master
2016-06-20 20:43:34 -05:00
Alex Grin 0b046ea8cf fix flipped if statement 2016-06-16 22:41:08 -04:00
Jack Robison 7133f05fea update readme 2016-06-10 18:39:35 -04:00
Jack Robison da1c23d7dd update readme 2016-06-10 18:28:21 -04:00
Job Evers‐Meltzer a329dcd844 Merge pull request #40 from lbryio/fix-travis-build
Fix travis build
2016-06-10 08:47:19 -05:00
Job Evers b57212d5f7 print package size in travis 2016-06-10 01:38:05 -05:00
Job Evers 12c9e16f37 fix typo to actually use ubuntu trusty 2016-06-10 01:22:51 -05:00
Job Evers‐Meltzer 422381ac01 Merge pull request #39 from lbryio/add-tracing
activate tracing flag on build script
2016-06-07 10:59:47 -05:00
Job Evers-Meltzer 4b817cef65 activate tracing flag on build script 2016-06-07 10:52:21 -05:00
Jack Robison 4255a9916d Merge pull request #38 from lbryio/development
0.2.5
2016-06-07 04:54:39 -04:00
Jack Robison b1566e904b Merge pull request #36 from lbryio/add-tests-to-travis
Add tests to travis
2016-06-07 04:51:09 -04:00
Jack 8629076814 use lbryio/lbry-osx-app.git for lbry/packaging/osx/lbry-osx-app 2016-06-07 04:41:26 -04:00
Jack 761e4f2718 change variable name to lbrynet_log 2016-06-07 04:30:22 -04:00
Jack 3441dd4d2d don't mess with logging 2016-06-07 04:19:51 -04:00
Jack 9a8139a8b2 merge conflict 2016-06-07 03:21:18 -04:00
Jack a0f823910c delete accidentally committed file 2016-06-07 03:16:22 -04:00
Jack 755338ded0 import logging.handlers 2016-06-07 03:08:32 -04:00
Jack ecd85a53a1 ui dir for linux 2016-06-07 01:20:44 -04:00
Jack 8faa0d9e01 update lbryum log path 2016-06-06 15:24:29 -04:00
Jack 20ca2eac37 lbryum logging in lbryum.log, add to upload_log() 2016-06-04 17:05:14 -04:00
Jack 85b56247a3 merge add-tests-to-travis updates 2016-06-04 15:11:51 -04:00
Jack febc66fb02 missing keyword args for travis 2016-06-04 15:03:16 -04:00
Jack 3f929aee3c merge travis fixes 2016-06-04 14:28:32 -04:00
Jack 8c46dd194d save command line wallet choice to config 2016-06-04 14:18:58 -04:00
Jack a9f99d05b9 add more known dht nodes
also fix migrator bug
2016-06-03 20:20:16 -04:00
Jack e224c33817 add mime_type to get_lbry_file 2016-06-03 03:45:46 -04:00
Job Evers-Meltzer 4e6afbc8ff Merge remote-tracking branch 'origin/master' into add-tests-to-travis 2016-06-01 22:56:05 -05:00
Job Evers-Meltzer 80b912ea14 pylint: disable incorrect not-callable message 2016-06-01 22:40:01 -05:00
Job Evers-Meltzer 85a571521b disambuguate popup functions 2016-06-01 22:38:24 -05:00
Job Evers-Meltzer 98efa9f466 pylint: ignore incorrect not-callable message 2016-06-01 22:35:30 -05:00
Job Evers-Meltzer a30a9ad763 have pylint skip currently unmaintained files 2016-06-01 22:31:41 -05:00
Jack 36787eeeeb default value 2016-06-01 21:05:38 -04:00
Jack 104cafedb0 --wallet fix 2016-06-01 20:52:15 -04:00
Job Evers-Meltzer 284d307c0f fix logging-too-few-args errors 2016-05-31 22:49:00 -05:00
Jack f50c98e10d revert import
see if this helps travis
2016-05-31 17:45:04 -04:00
Jack a4a5d63da7 fix update settings problem
also move log upload url constant to conf.py
2016-05-31 17:38:00 -04:00
Jack 9f0f9e0928 Merge remote-tracking branch 'origin/better_log_names' into add-tests-to-travis 2016-05-30 17:09:39 -04:00
Jack ef62fd7e34 fix pip install jsonrpc 2016-05-30 16:37:47 -04:00
Jack 0151dd8875 remove test_bot
test bot was moved to keynes repo, and shouldn’t be with the tests
script
2016-05-30 16:14:32 -04:00
Jack 2026024c8a updates from development 2016-05-30 15:49:25 -04:00
Jack 3d56baddd7 fix import error 2016-05-30 15:44:31 -04:00
Jack c10e7b2b41 fix import 2016-05-30 15:39:27 -04:00
Jack ba605e985a no shell=True 2016-05-30 04:40:58 -04:00
Jack 4cd1cdf495 escape spaces in paths given to reveal() 2016-05-30 04:37:34 -04:00
Jack f9c644b964 fix switching between --branch and --ui 2016-05-30 04:00:47 -04:00
Jack da68bcf952 third time's the charm 2016-05-30 03:14:59 -04:00
Jack 2c3b625b6b change full_path to path 2016-05-30 02:56:08 -04:00
Jack 16ef259ae1 add full_path to lbry_file 2016-05-30 02:53:36 -04:00
Jack c56b3e75ea add download_directory to lbry_file 2016-05-30 02:42:33 -04:00
Jack 772866389f ui manager fixes
load a ui when it’s supposed to be loaded
2016-05-30 02:36:13 -04:00
Jack 41f8b5aee2 add reveal() function and delete_target_file param for delete_lbry_file 2016-05-30 02:05:16 -04:00
Jack 75052fc773 remove check_for_new_version vestige 2016-05-30 00:10:43 -04:00
Jack e7a580fd3a add lbry_ui_manager 2016-05-29 23:18:30 -04:00
Job Evers c967230969 restore version consistency check 2016-05-28 11:04:10 -05:00
Job Evers f3e6b2ab57 on travis, move built deb file for easier access 2016-05-28 10:55:15 -05:00
Job Evers 7385cb0c89 try new filename for travis deployment 2016-05-28 09:54:08 -05:00
Job Evers‐Meltzer d1be9e7dc8 Merge pull request #37 from lbryio/tmp-remove-version-check
comment out version check
2016-05-28 09:38:44 -05:00
Job Evers a22052e85b comment out version check 2016-05-28 09:32:41 -05:00
Alex Grintsvayg ab401541f2 Merge branch 'travis-release'
* travis-release:
  add deployment section to .travis.yml
  fail build if version doens't match tag
  fix whitespace
2016-05-28 10:08:24 -04:00
Job Evers-Meltzer dab4427c18 add deployment section to .travis.yml
- not sure if the file name entry will work.
2016-05-28 10:08:18 -04:00
Job Evers-Meltzer ad7d508a3c fail build if version doens't match tag 2016-05-28 10:08:18 -04:00
Job Evers-Meltzer cb30514cbc fix whitespace 2016-05-28 10:08:18 -04:00
Job Evers 53af9db285 add tests and linting 2016-05-27 15:46:00 -05:00
Job Evers-Meltzer b557a14134 fix whitespace 2016-05-27 13:27:44 -05:00
Jack 8922fd6dde add startup scripts
-populate blockchainname.db on first run from older version
2016-05-25 22:28:45 -04:00
Jack 6b39e549f7 add developer id to travis.yml 2016-05-25 21:02:30 -04:00
Jack eb0dd827b1 delete old unused app and move uri handler to lbry-osx-app 2016-05-24 17:54:44 -04:00
Jack a48902d7d3 fix merge conflict 2016-05-24 17:51:00 -04:00
Jack 7fca1f865d update from master 2016-05-24 17:50:01 -04:00
Jack c5dc06962d travis code signing 2016-05-24 17:44:26 -04:00
Alex Grintsvayg 45ecdafd52 i have the best words 2016-05-23 16:25:14 -04:00
Alex Grintsvayg f9e8bf92bd readme rewrite 2016-05-23 16:13:53 -04:00
Alex Grintsvayg de614b293c change description 2016-05-23 16:13:53 -04:00
Jack Robison 8382075147 add divider line 2016-05-23 15:22:42 -04:00
Jack Robison b605630a9c update install instructions 2016-05-23 15:21:47 -04:00
Alex Grintsvayg 0a89fb1928 10 is too long 2016-05-23 00:40:18 -04:00
Alex Grintsvayg 4c81906b50 run lbry instructions in bg 2016-05-23 00:35:44 -04:00
Alex Grintsvayg b6220e985a let user know how to run lbry after install 2016-05-23 00:19:55 -04:00
Alex Grintsvayg 7a97a28e00 fix lbry icon 2016-05-22 23:48:51 -04:00
Alex Grintsvayg dda8f281a1 :shipit: travis build badge :shipit: 2016-05-22 22:45:08 -04:00
Alex Grintsvayg 28b561a810 Merge branch 'travis-ci-integration'
* travis-ci-integration:
  need fakeroot too
  Enabling builds on travis-ci
2016-05-22 22:38:22 -04:00
Alex Grintsvayg 44aba93d37 need fakeroot too 2016-05-22 22:36:56 -04:00
Job Evers efb5157027 Enabling builds on travis-ci
- adds a travis.yml config file
- modifies the ubuntu build script to work on travis
- adds start of osx builds
   - waiting for certs to be added
2016-05-22 20:17:36 -05:00
Alex Grintsvayg f48c1562f3 better log names 2016-05-20 19:59:49 -04:00
Alex Grintsvayg 098b5b28a8 fix web ui branch in install script 2016-05-20 12:13:58 -04:00
Jack Robison 998e8e748c Merge pull request #32 from lbryio/development
more stream and file fixes
2016-05-16 20:28:13 -04:00
Jack be96c77467 add set_miner and get_miner_status
set_miner takes parameter key ‘run’, which is True or False
2016-05-14 17:57:34 -04:00
Jack b5e8aec548 fix --wallet=lbrycrd 2016-05-14 17:36:30 -04:00
Jack 278c7dc836 have get() wait for file to have written data before returning by default
controlled by wait_for_write parameter
2016-05-13 19:35:38 -04:00
Jack 1425ad87ab fix get_lbry_file problem 2016-05-11 04:44:23 -04:00
Jack f985a2a8ad add waiting_for_credits startup code 2016-05-11 02:47:33 -04:00
Jack 20f423a2fc bump version 2016-05-11 02:08:13 -04:00
Jack 210c1215d6 add metadata to lbry_file 2016-05-11 01:48:23 -04:00
Jack 1c507393a9 fix problem with getting a stream where the file has been deleted 2016-05-10 21:01:53 -04:00
Jack b04cfdc091 load uri and txid when stream is started 2016-05-10 20:47:35 -04:00
Jack 42a3006f6e add claim txid and uri to lbry_file 2016-05-10 19:03:18 -04:00
Jack Robison 7d5c618246 Merge pull request #29 from lbryio/development
0.2.3
2016-05-06 18:06:46 -04:00
Alex Grintsvayg 74250982d1 update linux uri handler 2016-05-06 17:38:59 -04:00
Jack 8bd6fb5b4e clean up uri handler
clean up work arounds from when ui didn’t do loading nicely
2016-05-06 17:20:03 -04:00
Jack bb2fb92b0e have uri handler fail silently to prevent misleading error
otherwise first run produced an error while it was downloading the
headers if the app was started by going to lbry://lbry
2016-05-06 16:59:23 -04:00
Jack 10b016312d get rid of Xcode tools and git requirement 2016-05-06 16:16:09 -04:00
Jack b4500a2914 get rid of git HEAD stuff 2016-05-06 15:51:59 -04:00
Jack da9b1b3f58 merge conflicts 2016-05-06 15:42:21 -04:00
Alex Grintsvayg da31520465 use github api instead of git 2016-05-06 15:34:41 -04:00
Jack f3d3a0e57f Don't show 0 while downloading headers from amazon, fix uri handler 2016-05-06 14:31:44 -04:00
Alex Grintsvayg 9a655988dd update script to build against diff web ui branches 2016-05-06 14:30:00 -04:00
Jack 43b541bb7d Merge branch 'master' into development 2016-05-06 02:11:30 -04:00
Jack e32853744a better loading message 2016-05-05 22:26:25 -04:00
Jack b017b63745 status code for previously downloaded file
also fix return for files downloaded manually with stream_info
2016-05-05 21:45:25 -04:00
Jack 29d6d824fc write cryptsd files to data dir instead of cwd 2016-05-05 19:16:36 -04:00
Jack d6983a25b0 fix metadata problem in downloader and publisher 2016-05-04 23:40:05 -04:00
Jack 4cab6726b0 log to file from publisher
also fix sd_hash exception that could happen
2016-05-04 23:27:40 -04:00
Jack 509b8f3a29 add file_name and stream_info fields to get()
-file_name is the name of the file in the downloads folder

-stream_info is a dict of the metadata in a name claim, it can be used
to download streams where the claim hasn’t yet been added to the
nametrie
2016-05-04 21:25:46 -04:00
Jack 461c2f9055 add file download statuses for get_lbry_file
also clean up remaining fetcher stuff
2016-05-04 05:20:38 -04:00
Jack c1d0f9cf1b get rid of autofetcher in daemon
the purpose of the autofetcher was to automatically back up and host
published content, it is simpler to do this in a separate script that
uses existing daemon functions than to have it be built in
2016-05-04 04:12:11 -04:00
Jack 019de08c64 add more transaction functions to daemon
also fix —wallet command line option
2016-05-03 23:13:31 -04:00
Jack 30f88f99b7 use static.File 2016-05-02 15:58:40 -04:00
Jack 04ee9894c9 speed up streaming, off by one errors 2016-05-02 04:10:50 -04:00
Jack 46368f52f5 streaming files
still needs work, still has problems in safari and is otherwise slow
2016-05-01 05:17:59 -04:00
Jack 226e9084c9 return false from get_lbry_file if the file doesnt exist 2016-04-30 23:43:02 -04:00
Jack caf5d2f788 update uri handler to use /watch 2016-04-26 19:48:43 -04:00
Jack 591634f175 cache name claim info
-cache name claim info for an hour rather than looking it up each time
it’s required

-add default thumbnail to search results
2016-04-25 22:35:21 -04:00
Jack dfaf51a432 lbry file improvements
-fix daemon functions to start/stop lbry files

-remove unused stuff in LBRYFileManager

-improve and use new get_lbry_file function instead of _check_history,
which didn’t use the lbry file manager

-use said function to let delete_lbry_file use the same search keys
(sd_hash, name, and file_name)

-logging in LBRYDownloader
2016-04-24 17:51:24 -04:00
Jack c2ec066c85 add LBRYFileProducer
-Add LBRYFileProducer, to host the contents of a download without
having to keep re-opening it as it is added to

-included sd hash in ManagedLBRYFileDownloader, to make comparing the
contents of the file manager against name claims easier

-add get_lbry_file function, which returns information about a LBRY
file found by sd_hash, file name, or lbry uri
2016-04-24 04:42:42 -04:00
Jack 3e7c09bb44 fix duplicate/buffering problem
fix problem where get request takes time to respond, in this time it
can receive more get requests that each manage to start downloading
2016-04-22 18:48:44 -04:00
Jack 28f66e30dd improve timeouts 2016-04-22 18:18:17 -04:00
Jack bac7f097a9 add -x to filename before extension for duplicate downloads 2016-04-22 02:45:41 -04:00
Jack fecd29a967 only load /view page when file isn't empty 2016-04-22 02:45:05 -04:00
Alex Grintsvayg c22eed9109 improved unity description 2016-04-21 17:18:51 -04:00
Jack 2c1ba623a4 add message key to upload_log
log the message which is then uploaded with a special file prefix, and
also send the message to slack
2016-04-20 23:50:13 -04:00
Jack 561cdba7c9 fix jsonrpc import 2016-04-20 22:59:46 -04:00
Jack e5e25012ff move daemon server stuff into its own class 2016-04-20 22:02:52 -04:00
Jack c5d653a51a add bug report page
adds a page to submit a message and upload log at /report
2016-04-20 14:44:26 -04:00
Jack c4620a0b0c add problem_code to daemon_status 2016-04-20 04:24:32 -04:00
Jack dc969f80f3 Merge branch '0.2.3-development' into development 2016-04-20 03:02:57 -04:00
Jack a82f85f058 bump version, increase log size, fix shutdown problem 2016-04-20 02:56:29 -04:00
Jack 3ef85e711d Merge branch 'master' into development 2016-04-19 19:23:56 -04:00
Jack 29f7307f9d periodically check internet connection and for new versions 2016-04-18 13:21:53 -04:00
Jack 24eed71959 upload log function doc string 2016-04-18 03:41:16 -04:00
Jack 9341ccb643 Merge branch 'master' into transaction-explorer
fix conflicts in LBRYDaemonControl.py
2016-04-17 23:28:00 -04:00
Jack 4385b90cca add get transaction and history functions to daemon 2016-04-17 23:23:20 -04:00
Jack Robison 9c10ee12fd Merge pull request #28 from lbryio/lag-and-diagnostics
fix lagging indicator, improve diagnostics, fix lbryum link, make ui easier to launch on linux
2016-04-17 23:05:55 -04:00
Jack 7a622986d6 remove lbryum version from setup.py
version number made dependency link not work
2016-04-17 22:56:51 -04:00
Jack 99b3cdd45d default on launching ui if daemon already running when command is called
-also update deb version
2016-04-17 20:16:35 -04:00
Jack 7b9d84f528 use master branch for ui in .deb 2016-04-17 20:11:10 -04:00
Jack 5f49827bd2 fix lagging indicator and improve diagnostic
-make is_lagging more meaningful - it is set to true after 90 seconds
with no progress, this is to stop slow but steady catchups from
triggering it

-prevent situation where repeated shutdowns before wallet catchup
results in never receiving first run credits

-fix settings to write new defaults that aren’t already in the
configuration file

-report log of startup sequence if upload_log set to true

-redirect /view?name=lbry to the main UI page. This is to make the ui
accessible from a lbry:// link on linux
2016-04-17 19:54:04 -04:00
Jack Robison e1b4fe02d6 Merge pull request #27 from lbryio/development
merge updates from development into master
2016-04-16 01:15:50 -04:00
Jack cdab127b98 have separate handlers for uris on linux and os x 2016-04-15 23:57:52 -04:00
Jack bac7ea5dc0 include platform information in version() 2016-04-15 22:31:06 -04:00
Jack b3b5581f2d don't log to console, add --no-launch flag, fix help()
by default, the ui opens in the lbrynet-daemon startup sequence, the
—no-launch flag stops the ui from launching
2016-04-15 19:37:27 -04:00
Jack 9a26d869fe better first run message 2016-04-14 17:01:05 -04:00
Jack 57138d9629 is_lagging message 2016-04-14 16:50:11 -04:00
Jack 47d897b48d is_lagging indicator and internet connection check
-adds is_lagging indicator as a key from daemon_status, which is set to
True during a phase of the startup sequence, presently only during
loading_wallet if catching up with the blockchain takes longer than a
minute.

-checks if connected to internet before trying to start
2016-04-14 00:29:40 -04:00
Jack 6b9f1d519e remove null handling used to fix a now resolved bug, update daemon_status outputs per alex's suggestions 2016-04-13 23:10:17 -04:00
Jack 8b966c61d8 check version during startup 2016-04-13 22:07:27 -04:00
Jack 87e09efff5 update lbryum version in setup.py 2016-04-13 20:52:51 -04:00
Jack 65a0583c85 add help function and documentation for functions
-calling help() will return the list of functions

-setting the ‘function’ param in the help function will return the doc
string for that function

-setting the ‘callable_on_startup’ in the help function will return the
list of functions callable during the startup sequence
2016-04-13 14:47:34 -04:00
Jack 2000a1b334 replace rawgit with raw.githubusercontent 2016-04-13 02:52:41 -04:00
Jack cdd44af043 fix links in LBRYDaemonControl
https was failing
2016-04-13 02:39:50 -04:00
Jack df99d06d11 fix links in check_for_new_version and log version info
-also handle situation where function is given a null argument
2016-04-13 02:35:31 -04:00
Alex Grintsvayg b32d096be2 add png to setup.py 2016-04-12 20:44:47 -04:00
Alex Grintsvayg b9d5d1041c bump version, better icon 2016-04-12 19:27:26 -04:00
Alex Grintsvayg 994dc9ff33 Merge branch 'linux-install' into development
* linux-install:
  bunch of changes, ready to roll
  add ubuntu package scripts, lbry:// uri handler
2016-04-12 18:53:56 -04:00
Alex Grintsvayg 2ac97736ce bunch of changes, ready to roll 2016-04-12 18:53:36 -04:00
Alex Grintsvayg 925e106c43 add ubuntu package scripts, lbry:// uri handler 2016-04-12 18:53:36 -04:00
Jack 59002413f8 organize daemon init, move more into daemon_settings.json
-also add wallet.catchup_progress to get a rough percentage of how
caught up the daemon is, this is accessed through daemon_status()
2016-04-12 02:03:57 -04:00
Jack Robison 32293ee3bb Merge pull request #26 from lbryio/settings
merge in updates from settings branch
2016-04-11 23:19:40 -04:00
Jack e04f35af4f fix check_for_new_version
returns true if new version is available, otherwise false

check_for_new_version is callable during startup
2016-04-11 23:13:26 -04:00
Jack c50f83400a make version callable during startup 2016-04-11 22:37:42 -04:00
Jack 31fa974379 version function, ui version tracking, blocks behind blockchain during startup, cleaner set_vars
-adds version() function, which can be called after the daemon is
running and returns lbrynet, lbryum, and lbry ui version information.

-daemon_status now returns how far behind its behind the blockchain
when in the loading_wallet phase of startup

-cleans up set_vars by moving to init
2016-04-11 22:28:46 -04:00
Jack aa8c362cb8 add --branch arg to lbrynet-daemon to specify ui branch 2016-04-11 16:15:12 -04:00
Jack e48cc40610 fix incorrect variable 2016-04-09 16:18:06 -04:00
Jack a931f59dda only return first run message until credits arrive 2016-04-09 16:14:28 -04:00
Jack 4140fc4d28 add base58 2016-04-09 15:43:25 -04:00
Jack f0d39e6081 log LBRYDaemonControl and return None from get_start_notice if there is no notice to give 2016-04-09 15:43:10 -04:00
Jack c11872a229 default on no startup message 2016-04-09 04:35:34 -04:00
Jack f4fe730dc9 add get_start_notice to return startup messages 2016-04-09 04:15:58 -04:00
Jack a4c83a08c7 change name of check_first_run to is_first_run 2016-04-09 04:00:31 -04:00
Jack c3277d24c3 use status_message and status_code return fields for daemon_status() 2016-04-09 02:09:30 -04:00
Jack 28a23590ef add message for startup status
-added daemon_status function that returns {'status': startup status
message, 'is_running': true/false}
2016-04-09 01:33:27 -04:00
Jack 162275992b return fail from check_first_run when is_first_run isn't callable 2016-04-08 23:52:43 -04:00
Jack 08e25f4622 convert ints to floats if given as settings params 2016-04-08 23:38:57 -04:00
Jack 9a300d773f optional download_directory parameter for get function
-optional download_directory parameter to choose the directory a file
is saved to, if not given or not a valid directory it uses the default
2016-04-08 22:23:37 -04:00
Jack a5362f3170 daemon settings
-set_settings accepts a dict of settings
    -adds upload_log field, defaulting on true, this uploads the lbry
log file to lbry.io to help figure out if and where things aren’t
working
    -default_download_directory is the key of the path where files are
saved

-gets publish working with sources

-adds check_first_run function, returns True/False

-previously only is_running would work during startup, check_first_run,
 get_time_behind_blockchain, and stop have been added

-requires six 1.9, 1.10.0 produced errors
2016-04-07 03:12:09 -04:00
Jimmy Kiselak 7b653b90a8 Merge branch 'abandonclaimfix' into development 2016-04-04 22:20:29 -04:00
Jimmy Kiselak f464b3ae7c when decoding lbryum transactions, return values as Decimal 2016-04-04 22:20:15 -04:00
Jack d4b7064d2f fix sources problems 2016-04-01 22:11:11 -04:00
Jack 2540b9969f check UI version from command line
-check UI version, if out of date get a fresh copy. overridden by —ui
flag, which is used to specify a ui directory
2016-04-01 22:06:06 -04:00
Jack 4355b799cb before is_running is true all other methods return a failure 2016-03-29 22:02:36 -04:00
Jack a7fb434f9f update uri handler for change in daemon output 2016-03-29 17:04:01 -04:00
Jack b3465f3bbf fix /view, improve is_running, clean function outputs
-Use localhost:5279/view?name=wonderfullife to download and render a
file

-is_running is now the only function that will work during startup

-functions output json, rather than json in your json
2016-03-29 16:42:47 -04:00
Jack 4dfc3ff94c Merge branch 'master' into development 2016-03-29 16:34:47 -04:00
Jack 2670279b0d change access control allow origin to * 2016-03-28 16:49:51 -04:00
Jack Robison f4fd4245be Merge pull request #24 from lbryio/development
fix daemon directories for linux
2016-03-28 14:58:15 -04:00
Jack 200d239e0b log ui directory 2016-03-28 14:54:01 -04:00
Jack 41a0590182 fix daemon directories for linux 2016-03-28 14:53:32 -04:00
Jack 63b2c49354 don't start lbry app if daemon is running in console 2016-03-28 14:21:41 -04:00
Alex Grintsvayg c99d7ce89d fix weird version, add required fields for deb package 2016-03-27 18:20:02 -04:00
Alex Grintsvayg 79e382e064 add requirements.txt so we can have repeatable builds 2016-03-27 18:19:53 -04:00
Jack fa1d4dae0f add --ui parameter to lbrynet-daemon to set ui path
defaults on temp, which will download a .zip of the current ui to a
temp directory, which will be deleted when lbrynet-daemon closes
2016-03-25 21:03:58 -04:00
Jack 35f84b4414 Merge branch 'master' into development 2016-03-25 21:02:51 -04:00
Jack a882bdb08c fix output of get_time_behind_blockchain function 2016-03-25 09:32:24 -04:00
Jimmy Kiselak b5c2e1c95b Merge branch 'development' 2016-03-24 22:16:07 -04:00
Jack cfd6ea5a34 Merge branch 'development' 2016-03-24 21:15:54 -04:00
Jack b3cf5e8698 add bundle identifier to uri handler 2016-03-24 20:46:44 -04:00
Jimmy Kiselak 2d07223995 Merge branch 'fixsetuptools' into development 2016-03-24 20:01:05 -04:00
Jimmy Kiselak 32bb9fd4ec reorder setuptools 2016-03-24 20:00:56 -04:00
Jack Robison 37b5f22c1a Merge pull request #23 from lbryio/jsonrpc
Jsonrpc
2016-03-23 22:37:29 -04:00
Jack e0b00c09ce fix conflicts 2016-03-23 22:36:14 -04:00
Jack 71c4dba34d Merge branch 'development' into jsonrpc
# Conflicts:
#	setup.py
2016-03-23 22:34:27 -04:00
Jack 9a3187e36c merge updates 2016-03-23 22:29:48 -04:00
Jack c714a3f91c merge updates 2016-03-23 22:27:48 -04:00
Jimmy Kiselak 5d4846cf60 Merge branch 'versioning' into development 2016-03-23 22:07:11 -04:00
Jack 543b327d09 use only six 1.9 on os x 2016-03-23 22:06:04 -04:00
Jimmy Kiselak 1ccd2a3cc9 do version better 2016-03-23 22:02:02 -04:00
Jack 94398071eb have uri handler check if app is running
-checks if LBRY.app is running, starts it if it isn’t

-delete unused function in LBRYDaemon
2016-03-23 19:49:14 -04:00
Jimmy Kiselak 143476c76c Merge branch 'showwalletcatchup' into development 2016-03-22 22:43:10 -04:00
Jimmy Kiselak e85683e40f show more informative messages when blockchain is catching up 2016-03-22 22:42:45 -04:00
Alex Grintsvayg 141162e5ae fixes so we can build an sdist 2016-03-22 22:15:07 -04:00
Jack 13acbfb64f update uri handler
-use new jsonrpc daemon, start app if it isn’t running on first lbry://
request
2016-03-22 11:51:27 -04:00
Jack a55e3d8578 use UI downloaded to temp folder at startup 2016-03-22 10:37:13 -04:00
Jack 2486ae6d3f get rid of unused updater function
updater function was hacky and not the most reliable, to be replaced by
app version checking
2016-03-22 09:40:18 -04:00
Jack ba991b47ea add /view to dynamically generate a video containing page
-uses UI stored in Resources folder of app

-going to /view?name=  will download the corresponding stream and
display the file in the browser

-adds default daemon settings, currently they don’t do anything, to be
added to

-adds missing packages to setup.py
2016-03-22 00:03:17 -04:00
Jimmy Kiselak a096d0db37 Merge branch 'versioning' into development 2016-03-21 21:37:26 -04:00
Jimmy Kiselak e4a30c052e put the actual version in __init__.py 2016-03-21 21:37:15 -04:00
Jimmy Kiselak ccfe80ecfa Merge branch 'versioning' into development 2016-03-21 21:34:05 -04:00
Jimmy Kiselak df5b889f7d add a version to lbrynet 2016-03-21 21:33:55 -04:00
Jack 55f25c2613 consolidate search output
consolidate search output into a list of dicts
2016-03-21 12:06:19 -04:00
Jack 34eaa822a0 handle null parameter 2016-03-21 10:12:01 -04:00
Jack 5072f503c4 fix cross scripting problem
add access-control-allow-origin: http://localhost:5279
2016-03-20 22:53:01 -04:00
Jimmy Kiselak 8e6b5a5e55 Merge branch 'fewerlogstatements' into development 2016-03-19 22:39:59 -04:00
Jimmy Kiselak 246647641a remove some log statements that are useless, instead log whenever balance changes 2016-03-19 22:39:47 -04:00
Jimmy Kiselak 06a1bdbcba Merge branch 'lbcpatience' into development 2016-03-19 22:33:56 -04:00
Jimmy Kiselak a0b75d7ada Tell user to be patient if it's the first run and user has no funds, in two more places 2016-03-19 22:30:55 -04:00
Jimmy Kiselak d36036a9fe Tell user to be patient if it's the first run and user has no funds 2016-03-19 21:50:38 -04:00
Jimmy Kiselak a68d891770 enable checking first run status multiple times 2016-03-19 21:24:44 -04:00
Jimmy Kiselak f272a9ead2 Merge branch 'badunicode' into development 2016-03-19 18:56:21 -04:00
Jimmy Kiselak 8db2e10696 Merge branch 'master' of github.com:lbryio/lbry 2016-03-19 18:55:50 -04:00
Jimmy Kiselak bea4f7da98 Merge branch 'badunicode' 2016-03-19 18:55:37 -04:00
Jimmy Kiselak 3cd74c06d6 convert description in ControlHandlers to str so that the prompt does not get converted to unicode 2016-03-19 18:55:17 -04:00
Jack Robison 053681e4a5 Merge pull request #21 from lbryio/detect-if-daemon-is-running
check if daemon is running before starting console
2016-03-17 23:08:12 -04:00
Jack e3225f7123 add missing commits 2016-03-17 22:50:15 -04:00
Jack ef62b0e746 fix conflicts 2016-03-17 22:38:48 -04:00
Jack 7756d12993 check if daemon is running before starting console 2016-03-17 22:03:56 -04:00
Jimmy Kiselak 3b92d12a44 Merge branch 'walletbalancerefresh' into development 2016-03-17 21:55:20 -04:00
Jimmy Kiselak 4449bf3c75 change manage timer back to 60 seconds 2016-03-17 21:55:06 -04:00
Jimmy Kiselak 0200ed1341 refresh wallet balance every 3 seconds instead of 60 2016-03-17 20:19:13 -04:00
Jack 02bde900d6 update daemon and console
-don’t start console if daemon is running, display a message alerting
user

-move os x data directory to match app

-add timeout parameter to get function
2016-03-17 10:19:36 -04:00
Jack 36e299b4cf fix files
-don’t automatically remove files that can’t be found from
LBRYFileManager

-add timeout to GetStream
2016-03-17 00:44:04 -04:00
Jack 4e34f861c9 Merge branch 'development' into jsonrpc 2016-03-15 23:02:06 -04:00
Jack 42df2fe242 update daemon
convert to jsonrpc, fix a few issues, remove hacky functions
2016-03-14 12:30:22 -04:00
Jimmy Kiselak 7f11da8650 Merge branch 'walletcatchup' into development 2016-03-12 14:59:23 -05:00
Jimmy Kiselak dff0fcb1fd ensure the lbryum wallet is caught up before giving the prompt 2016-03-12 14:58:58 -05:00
Jimmy Kiselak c31d22f996 Merge branch 'metadata' into development 2016-03-12 14:46:31 -05:00
Jimmy Kiselak 14ab2b52e9 create fee field in metadata 2016-03-12 14:42:57 -05:00
Jimmy Kiselak e91f34f09b change metadata structure to use sources key 2016-03-12 14:25:46 -05:00
Jimmy Kiselak 6c733f84e9 Merge branch 'stopexceptions' into development 2016-03-12 14:08:42 -05:00
Jimmy Kiselak 71c2cd5065 Merge branch 'stopexceptions' 2016-03-12 14:08:32 -05:00
Jimmy Kiselak 45f9a10bfb hide exceptions that show up in the dht and due to bad metadata in the blockchain 2016-03-12 14:08:15 -05:00
Jimmy Kiselak 6ad203f767 Merge branch 'docsupdate' into development 2016-03-09 12:26:27 -05:00
Jimmy Kiselak 5bf895eefe update documentation to reflect using lbryum by default 2016-03-09 12:26:07 -05:00
Jack 5a8f5b14c1 Merge branch 'master' into development 2016-03-08 12:16:34 -05:00
Jack 27ccb37646 update daemon
check that managed files still exist, if they don’t remove them from
the manager
2016-03-08 12:15:49 -05:00
Jack Robison b4c528f6b5 Merge pull request #18 from lbryio/development
merge in commit that got left out
2016-03-03 14:05:25 -05:00
Jack Robison 517a255d5a Merge pull request #17 from lbryio/development
merge lbryum updates
2016-03-03 14:00:34 -05:00
Jack d76feeb320 use master branches 2016-03-03 13:55:03 -05:00
Jack e4f23b3fce better organize apps 2016-03-03 00:16:51 -05:00
Jack 02d8dc22e9 update daemon
-notify user when connected to lbrynet
-check for updates from status bar app
2016-03-02 18:32:48 -05:00
Jack 2fc3ec7224 update status bar app 2016-03-02 02:00:45 -05:00
Jack a243ac7a01 fix bug starting daemon from status bar 2016-02-29 15:04:20 -05:00
Jack 6f1ee78623 update daemon 2016-02-29 13:25:47 -05:00
Jack 6633370a34 Merge branch 'development' of https://github.com/lbryio/lbry into development 2016-02-28 23:51:09 -05:00
Jack a22e29532c start daemon from status bar 2016-02-28 23:49:37 -05:00
Jimmy Kiselak 96c8d1f68b Merge branch 'development' of github.com:lbryio/lbry into development 2016-02-28 12:46:29 -05:00
Jimmy Kiselak 4aa3bd56c4 Merge branch 'win32build' into development 2016-02-28 12:45:39 -05:00
Jimmy Kiselak 7d1cf7975a update win32 setup script to include lbryum 2016-02-28 12:45:21 -05:00
Jack 59d08a92d0 update status bar app 2016-02-27 17:49:49 -05:00
Jack 33c1a002da update daemon
only let local user connect to lbrynet-daemon
2016-02-27 13:19:26 -05:00
Jack 8f04d3418a get channel id from slack for test bot
in case i accidentally delete the bot conf file again and don’t want to
look up the channel id
2016-02-26 11:20:31 -05:00
Jimmy Kiselak f5b556c6cb Merge branch 'development' 2016-02-26 11:02:41 -05:00
Jack a18f3bf08a disable status bar 2016-02-26 10:59:09 -05:00
Jack Robison 41a0a0c64c Merge pull request #15 from lbryio/lbryum
merge updates
2016-02-26 10:41:07 -05:00
Jack b4c8a317d0 switch branch 2016-02-26 10:39:56 -05:00
Jack Robison 30ac075555 Merge pull request #14 from lbryio/development
merge updates
2016-02-26 10:29:38 -05:00
Jimmy Kiselak fcb317ca40 Merge branch 'electrumparity' into development 2016-02-26 01:46:21 -05:00
Jimmy Kiselak 9991f79326 add support for claiming names, abandoning names, and listing claimed names to the LBRYumWallet 2016-02-26 01:45:52 -05:00
Jack 7cf3f8c7cd fix typo 2016-02-25 17:18:18 -05:00
Jack eeef183cc6 update lbrynet-daemon 2016-02-25 17:17:07 -05:00
Jack 96bc02c8cb Merge branch 'development' into lbryum 2016-02-24 16:19:19 -05:00
Jimmy Kiselak af276a24ed Merge branch 'development' 2016-02-23 15:48:00 -05:00
Jimmy Kiselak a92dbae459 Merge branch 'electrum' into development 2016-02-23 15:17:52 -05:00
Jimmy Kiselak b0bfb5e443 use electrum as default 2016-02-23 15:13:34 -05:00
Jimmy Kiselak 671e6da104 show when the wallet is starting/started 2016-02-22 23:31:07 -05:00
Jack 485be533c9 make lbryum default wallet for daemon
with optional command line argument —wallet=
2016-02-22 22:32:07 -05:00
Jimmy Kiselak 53ff4cd651 Merge branch 'electruminstall' into development 2016-02-22 18:22:08 -05:00
Jimmy Kiselak a98c760d93 Merge branch 'electruminstall' 2016-02-22 18:22:00 -05:00
Jimmy Kiselak 9b0c95e37d make sure the correct version of six gets installed 2016-02-22 18:21:39 -05:00
Jimmy Kiselak b419b8f642 enable getting the whole claim trie 2016-02-22 14:24:49 -05:00
Jimmy Kiselak c32f8d0998 implement getblock 2016-02-22 12:18:28 -05:00
Jimmy Kiselak c6d253cc25 Merge branch 'updateinstructions' into development 2016-02-22 10:29:14 -05:00
Jimmy Kiselak c05cb62cdb Merge branch 'updateinstructions' 2016-02-22 10:29:00 -05:00
Jimmy Kiselak ddd213dda3 update manual build instructions to check out lbrycrd alpha 2016-02-22 10:28:33 -05:00
Jack 8aab844a48 add bdb 2016-02-22 10:05:55 -05:00
Jack 8fc1cbfc54 check for six 1.9.0 2016-02-20 15:30:30 -05:00
Jack 82ea330dda Merge remote-tracking branch 'origin/master' 2016-02-19 23:27:29 -05:00
Jack c63aa3033a install lbryum 2016-02-19 23:26:24 -05:00
Jimmy Kiselak 7837738c17 Merge branch 'master' of github.com:lbryio/lbry 2016-02-19 00:44:35 -05:00
Jimmy Kiselak bdb0ad4836 enable lbryum, the lightweight lbrycrd client 2016-02-19 00:44:08 -05:00
Jack 0dbf810cfe move status bar stuff 2016-02-19 00:07:19 -05:00
Jack 58aa4887ad update daemon
Start status bar if possible
2016-02-18 20:55:59 -05:00
Jack 3ff0916f49 remove icon 2016-02-18 20:48:43 -05:00
Jack cc9fb3ff41 fix icon path 2016-02-18 20:45:16 -05:00
Jack 2c51928b9d Status bar for lbrynet daemon 2016-02-18 20:41:23 -05:00
Jimmy Kiselak 4451978af6 make wallet method public and move common functions back into parent wallet class 2016-02-17 11:47:39 -05:00
Jack d4193c55ae update fetcher 2016-02-17 10:45:33 -05:00
Jimmy Kiselak ec3e365b3c fix up LBRYcrdWallet and split it into two classes 2016-02-16 23:10:26 -05:00
Jack c245c69389 Add check-for-new-version function to daemon 2016-02-16 14:48:10 -05:00
Jack 9a9a7c9d7a update daemon 2016-02-16 13:55:46 -05:00
Jack 20b516b5fa automatic updates
Adds automatic updates to lbrynet-daemon, this will check if the
installed versions of LBRY applications as well as the lbrynet python
installation are up to date. If they aren’t, it downloads the new
versions, installs them, and restarts the daemon.
2016-02-16 13:39:08 -05:00
Jack 4dcfeca092 speed up update scripts
don’t download repo history, only what’s current.
2016-02-16 10:25:15 -05:00
Jack 8ca8c73868 update scripts 2016-02-16 09:09:50 -05:00
Jack 1a1c101b9a Add lbry app update script 2016-02-15 23:00:24 -05:00
Jack a0a0e2d324 add daemon update script 2016-02-15 21:31:29 -05:00
Jack d544199eff Fix search issue and move some settings to conf.py 2016-02-15 10:12:49 -05:00
Jack 407ebeb699 Update daemon 2016-02-11 08:32:48 -05:00
Jimmy Kiselak 9ba12b3e3f Merge branch 'master' of github.com:lbryio/lbry 2016-02-04 15:19:47 -05:00
Jimmy Kiselak 97976cbee9 lower the minimum rate 2016-02-04 15:18:47 -05:00
Jack cb5c772233 Have autofetcher use managed downloader 2016-01-27 11:05:40 -05:00
Jimmy Kiselak 2bfbb824bf Merge branch 'master' of github.com:lbryio/lbry 2016-01-27 02:56:41 -05:00
Jimmy Kiselak 05189651ee update path for close file name on windows 2016-01-27 02:55:45 -05:00
James Kiselak 0c80134523 Merge pull request #12 from DaveA50/master
Enable building and installing of GUI for windows
2016-01-27 00:47:32 -05:00
David Amrhein 6c686520c4 Add instructions for setting up windows environment 2016-01-26 22:52:55 -05:00
Jack 49ae029572 Don't try to process blank IPs 2016-01-26 16:07:33 -05:00
David Amrhein 10e43203ed Enable building and installing of GUI for windows 2016-01-26 11:56:28 -05:00
Jack e25be5a522 Update lbrynet-daemon
Use PaymentRateManager
2016-01-25 23:59:57 -05:00
Jack 47ff64a9e9 Update lbrynet-daemon
Adds the following:

-Abandon claim from its txid
-Get your name claims
-Get time behind blockchain
-Fixes upload
2016-01-25 20:28:05 -05:00
Jack 35f3887d79 Don't try to download a blank name 2016-01-24 00:26:41 -05:00
Jack dacfc8b00a Update lbrynet-daemon
-Add optional content_license to name claim

-Get a better cost estimate, if the sd blob takes longer than 5 seconds
to download just use the key fee

-Adds the free speech flag as the default thumbnail if none is provided
2016-01-24 00:10:39 -05:00
Jimmy Kiselak 0840ac7d06 Merge branch 'master' of github.com:lbryio/lbry 2016-01-22 18:16:14 -05:00
Jimmy Kiselak f34b96f565 print a message when a download finishes 2016-01-22 18:16:00 -05:00
jackrobison 808fbed4f3 Don't open links while having a 0.0 balance 2016-01-22 16:46:35 -05:00
jackrobison b48800049b fix url 2016-01-22 16:19:20 -05:00
jackrobison 7e5d4166d8 Use locally hosted gui 2016-01-22 16:04:23 -05:00
Jimmy Kiselak 6b724ba8db Merge branch 'master' of github.com:lbryio/lbry 2016-01-22 15:56:01 -05:00
Jimmy Kiselak 25e1427a84 fix error that occurred when one connection's download was stopped due to another download finishing 2016-01-22 15:50:18 -05:00
jackrobison 82944bc103 Merge pull request #11 from jackrobison/master
Update daemon and uri handler
2016-01-22 09:45:34 -05:00
Jack 1e73b2437d Update daemon and uri handler
Add command to stop daemon from the terminal

If lbrynet-daemon isn’t running the uri handler will now redirect to
lbry.io/get
2016-01-22 09:44:38 -05:00
jackrobison 574cd304e3 Test ability to download from lbrynet
This tries to download a small image with a five minute timeout. After this, the bot waits ten minutes and tries again. Any failures get posted to slack.
2016-01-20 22:52:58 -05:00
jackrobison 2edcd1f5b2 Merge pull request #10 from jackrobison/master
Update lbrynet-daemon
2016-01-20 22:10:15 -05:00
Jack e144b26619 Merge remote-tracking branch 'lbryio/master' 2016-01-20 22:05:16 -05:00
Jack 9ef9220689 Add thumbnail option to name claim
Add thumbnail option to name claim
2016-01-20 22:01:03 -05:00
Jack 23b6a8c495 Add publish
Add publish function
2016-01-20 22:00:28 -05:00
jackrobison 4b9552fdf5 Fix output from shutdown function 2016-01-20 12:55:15 -05:00
jackrobison a249097170 Merge pull request #9 from jackrobison/master
Merge updates
2016-01-19 21:36:41 -05:00
jackrobison 6f4b2d7967 Delete x.sh 2016-01-19 21:13:29 -05:00
Jack 888d77c96c Update lbrynet-daemon
-Get rid of separate db for daemon, instead use already existing files
-Simplify the get function
-Add start/stop functions for lbry files
2016-01-19 21:07:16 -05:00
Jack e2ae1ca866 Remove file handle 2016-01-17 01:49:17 -05:00
Jack 7ef9a0f0bf Merge remote-tracking branch 'lbryio/master' 2016-01-16 23:16:21 -05:00
Jack 885a0882f2 Update daemon and uri handler 2016-01-16 23:06:24 -05:00
Jimmy Kiselak 2ddeca2976 fix bug caused by downloading file twice and deleting one 2016-01-16 01:16:37 -05:00
Jack 2085012874 Get managed lbry files 2016-01-13 09:42:30 -05:00
Jack ebb495ab8b Fix error at shutdown
Fixed messy error message that sometimes appeared at shutdown
2016-01-12 13:41:20 -05:00
Jack 905274d6d4 Use same downloader for daemon as is used for console
Use
lbrynet.lbryfilemanager.LBRYFileDownloader.ManagedLBRYFileDownloaderFact
ory instead of
lbrynet.lbryfile.client.LBRYFileDownloader.LBRYFileSaverFactory
2016-01-12 13:40:51 -05:00
Jack 120d7edf12 Better handle bad names, improve browser uri handler
Play videos in browser rather than in a separate application window
2016-01-12 12:42:32 -05:00
Jack 12cd06cae5 Add download history logging
Logs downloads as to not make duplicate files
2016-01-11 18:01:46 -05:00
Jack 42c26102d8 Make daemon shut down properly
Makes the daemon shutdown properly and adds better start/stop messages.
2016-01-11 15:52:38 -05:00
jackrobison e906564f70 Merge pull request #8 from jackrobison/master
Fix daemon paths and add settings
2016-01-07 10:28:59 -05:00
Jack b8b2dc3b6d Fix daemon paths and add settings
Fixed lbrycrdd path in lbrynet-daemon, added configurable max key fee
and data payment rate
2016-01-07 10:24:55 -05:00
Jack d69f76e68a Merge remote-tracking branch 'lbryio/master' 2016-01-07 10:20:46 -05:00
Jimmy Kiselak ec1ba02cca show where the log file is, and log failure to open file better 2016-01-06 13:08:20 -05:00
Jimmy Kiselak 49869d071a show when lbrycrdd is being stopped and when it stops 2016-01-06 00:56:45 -05:00
Jimmy Kiselak 39275682d9 show the estimated cost of a download and format the download size better 2016-01-06 00:50:50 -05:00
jackrobison 1dbc3a049b Merge pull request #7 from jackrobison/master
Fix paths for os x binaries
2015-12-30 23:56:26 -05:00
Jack 0a7b2683ff Fix paths for os x
Fix paths to use binaries in the app bundle
2015-12-20 03:29:13 -05:00
Jimmy Kiselak 9714985855 Merge branch 'master' of github.com:lbryio/lbry 2015-12-18 15:48:16 -05:00
Jimmy Kiselak d809b8eee0 increase usefulness of error messages and change the download directory back to pwd for linux 2015-12-18 15:48:02 -05:00
jackrobison 52e2ed2aa7 Update download directory for os x gui 2015-12-16 23:42:38 -05:00
jackrobison ab3cc3d2b6 Merge pull request #6 from jackrobison/master
Add 'get' function to daemon
2015-12-16 09:26:43 -05:00
Jack 0a33324321 Merge remote-tracking branch 'lbryio/master' 2015-12-16 09:16:42 -05:00
Jimmy Kiselak a948335cb9 show if the application is behind on the blockchain if an error occurs that could be caused by that 2015-12-15 14:42:29 -05:00
Jimmy Kiselak 6d2e5af6bc Inform user that they are behind on the blockchain if they get certain errors 2015-12-14 23:49:27 -05:00
Jimmy Kiselak 9a7c71394e make welcome messages more friendly 2015-12-14 22:35:16 -05:00
Jack 1ebf184e9f Add get function for daemon
Adds a ‘get’ function to the daemon, which downloads a file and returns
its file path.
2015-12-14 21:49:01 -05:00
jackrobison 48b013fa85 Delete autodownloader.py
File still exists in autodownloader branch
2015-12-14 15:56:19 -05:00
Jack e96dee478d Remove autofetcher from lbrynet-console
Disabled autofetcher in the console, moved getstream and fetcherdaemon
to their own file in lbrynet-daemon
2015-12-14 15:54:33 -05:00
Jimmy Kiselak 25d2a12a92 Merge branch 'master' of github.com:lbryio/lbry 2015-12-14 14:12:14 -05:00
Jimmy Kiselak b0af31a71c use version of unqlite that does not require cython 2015-12-14 14:12:01 -05:00
Jack 9fdd766429 Fix import error 2015-12-14 13:24:29 -05:00
jackrobison 28ce23350d Merge pull request #5 from lbryio/autodownloader
lbrynet-daemon
2015-12-14 13:19:01 -05:00
Jack 1abe890067 Disable console autofetcher 2015-12-14 13:14:04 -05:00
Jack 75fbedeeac Merge branch 'master' into autodownloader 2015-12-14 13:03:17 -05:00
Jack a1861a2f84 Update lbrynet-daemon
Add auto fetcher to fetch new name claims
2015-12-14 12:59:44 -05:00
Jack 0eecfb52ed Update lbrynet-daemon
Daemon creates data directory if it doesn’t already exist
2015-12-14 10:22:52 -05:00
Jimmy Kiselak ce1c28ffb9 allow the thumbnail link to get to the gui 2015-12-10 00:37:43 -05:00
Jack 9667b81c8d Update url handler 2015-12-09 23:59:19 -05:00
Jack afac596a7b Update url handler 2015-12-09 23:57:58 -05:00
Jimmy Kiselak 636692ba6a show a thumbnail if one is included in the blockchain 2015-12-09 02:33:50 -05:00
Jack 203f7e7ea6 Update lbry daemon 2015-12-08 00:37:49 -05:00
Jack 8aee19b0ef Fix error
Fixes an error if you don’t have a conf file set up, previously it
would fail because the AutoFetcher object doesn’t have a console object
until it’s called for the first time
2015-12-07 18:02:29 -05:00
Jimmy Kiselak 49ed31c895 stop showing options, show estimated cost 2015-12-07 15:54:51 -05:00
Jack 526075cd5e Fix get_downloads function
get_downloads would throw an error when it tried to open a file that
had already been downloaded, now it just ignores these duplicates
2015-12-07 09:17:45 -05:00
Jimmy Kiselak 71340805ec Merge branch 'master' of github.com:lbryio/lbry 2015-12-07 00:51:54 -05:00
Jimmy Kiselak 654585aa68 require version of requests that supports json parameter 2015-12-07 00:51:40 -05:00
Jack 441d3c1220 LBRYNet daemon
Adds a daemon for lbrynet. Currently commands are limited the following:

download_name(name)
resolve_name(name)
get_downloads
stop

This allows other programs to easily interact with lbrynet, such as
LBRYURIHandler. LBRYURIHandler can be built with py2app, the resulting
plist file must be edited in the same way the committed plist file has
been edited. When built and installed to the /Applications folder
lbry:// domain names will download and open the corresponding file so
long as the daemon is running.
2015-12-06 17:32:17 -05:00
James Kiselak fc2d7258fb Merge pull request #4 from teran-mckinney/master
Fix LICENSE link
2015-12-06 16:23:33 -05:00
Teran McKinney 8195cedcf4 Fix LICENSE link
I broke it, oops.
2015-12-06 19:36:41 +00:00
Jeremy Kauffman 3e98477707 Merge pull request #2 from teran-mckinney/master
Markdown cleanups
2015-12-06 11:26:35 -05:00
Teran McKinney 7b9479eca9 Markdown cleanups 2015-12-05 21:28:29 +00:00
Jack f15b67d386 Merge branch 'master' into autodownloader 2015-12-03 21:09:26 -05:00
Jack f3141727d7 Update from master and fix a path problem
Update from the master branch and a fix to a path problem in lbrynet-gui
2015-12-03 20:56:56 -05:00
Jimmy Kiselak c7e4638736 update gui to work with updated LBRYcrdWallet 2015-12-02 14:53:39 -05:00
Jimmy Kiselak d04981969f increase the time limit for waiting for lbrycrdd to start up 2015-11-25 11:16:27 -05:00
Jimmy Kiselak a148d4f446 use specific version of python-bitcoinrpc 2015-11-25 10:31:43 -05:00
Jack a9e105f9eb Make autofetcher ignore key payments 2015-11-24 01:17:47 -05:00
Jimmy Kiselak c8afce2a3a stop console from crashing on input before it is started 2015-11-23 15:11:34 -05:00
Jack ebf9d11f4d Rename a class 2015-11-20 10:42:19 -05:00
Jimmy Kiselak 7a3bdf93e9 make sure upnp is neither set nor unset if upnp is disabled 2015-11-19 23:20:17 -05:00
Jack 35d5c585ec Added a max key price set in autofetcher.conf 2015-11-19 16:20:12 -05:00
Jack 45eb4f7201 Improved autofetcher
Added commands:

start-autofetcher
stop-autofetcher
autofetcher-status
2015-11-19 14:43:36 -05:00
Jimmy Kiselak e4c2585e23 pass console object to all control handlers 2015-11-17 17:28:17 -05:00
jackrobison a05f2fe3fb Check blocks for new claims more frequently 2015-11-16 12:55:01 -05:00
jackrobison 94cdd6829c Delete workspace.xml 2015-11-14 21:59:57 -05:00
Jack 34aa51e98a remove debugging messages 2015-11-14 21:57:58 -05:00
Jack 6e93c68a77 Run AutoFetcher in the background 2015-11-14 21:53:16 -05:00
Jack f16eec2ad2 Automatic downloading now works 2015-11-14 20:36:53 -05:00
Jack 657c1a4626 Adds a command to automatically download files as names are claimed
Defaults on only trying to download wonderfullife
2015-11-12 00:03:49 -05:00
Jimmy Kiselak 6bf6e943cc Merge branch 'master' of github.com:lbryio/lbry 2015-11-11 15:06:31 -05:00
Jimmy Kiselak 6252f903ee automatically give LBC to users on the first run and update RUNNING.md for prompt interface 2015-11-11 15:06:14 -05:00
Jack a1e827fb3b LBRY autodownloader
Aquire LBRY files as they are published
2015-11-10 11:33:35 -05:00
Jack 40218e02e1 fixed a problem with publish 2015-11-07 00:15:42 -05:00
Jack 0891b0d294 Changed LBRYcrdWallet to retrieve the rpc config 2015-10-28 20:59:07 -04:00
Jimmy Kiselak 756c43c087 Merge branch 'master' of github.com:lbryio/lbry 2015-10-28 01:38:13 -04:00
Jimmy Kiselak 6488e66088 show lbry:// name of downloaded files if applicable as well as the status of the claim 2015-10-28 01:38:01 -04:00
jackrobison 31ee3fb12b Update LBRYConsole.py
Forgetting to add a new line at the end of lbrycrd.conf will no longer mess up the rpc login.
2015-10-27 12:07:09 -04:00
Jimmy Kiselak b80d45ad45 Prompt for file path instead of file name, then show file name and file path 2015-10-26 13:15:32 -04:00
jackrobison ea2a3c3b70 Update LBRYSettings.py 2015-10-26 11:00:07 -04:00
Jimmy Kiselak c0312159eb update help strings, reorder main commands, and take application status info out of 'status' command 2015-10-23 18:00:10 -04:00
Jimmy Kiselak e124f582aa make a single publish step and consolidate status 2015-10-23 14:51:26 -04:00
Jimmy Kiselak 54150a8f2a move some stream descriptor functions to the appropriate file, start creating a publish command 2015-10-22 20:36:03 -04:00
Jimmy Kiselak 385c221ef8 add some status messages when starting up and break commands into common and debug 2015-10-22 02:23:12 -04:00
Jimmy Kiselak 7df6e99e28 fix typo and formatting errors, stop console's shutdown from running twice on exit, give more informative message if ports can't be bound, only set upnp if there isn't already a redirect on that port/protocol, and only unset upnp if the redirect was set by the program 2015-10-19 15:05:17 -04:00
Jimmy Kiselak 76a5f2f516 show and log when an error occurs during shutdown 2015-10-18 23:56:13 -04:00
Jimmy Kiselak 936c1f2ada fix typo and make sure objects exist before trying to stop them 2015-10-18 22:36:09 -04:00
Jimmy Kiselak 5787802077 change the console ui to command based rather than option based 2015-10-15 14:12:22 -04:00
Jimmy Kiselak c902d25524 start upgroading the console interface 2015-10-12 13:52:47 -04:00
Jimmy Kiselak d152336f42 initial commit of auto-launch lbrycrdd 2015-10-08 11:01:50 -04:00
Jimmy Kiselak 7ab0ae18be fix typo 2015-10-02 18:32:20 -04:00
Jimmy Kiselak 6189eddecf more formatting 2015-10-02 18:30:16 -04:00
Jimmy Kiselak cc7471a9af fix formatting more 2015-10-02 18:23:11 -04:00
Jimmy Kiselak eabaca361c fix formatting 2015-10-02 17:08:15 -04:00
Jimmy Kiselak 141ca2498c rename RUNNING to RUNNING.md 2015-10-02 16:40:46 -04:00
Jimmy Kiselak 964db2a779 enable setting lbrycrdd path in gui conf, disable launching lbrycrdd by default in linux gui, change running instructions, put linux gui conf in users home directory 2015-10-02 16:38:57 -04:00
Jimmy Kiselak f2867542b1 remove length from metadata stored in blockchain 2015-09-22 13:06:20 -04:00
Jimmy Kiselak 8f6e886821 show metadata found in blockchain in console 2015-09-22 12:21:16 -04:00
Jimmy Kiselak dfca7bc2c4 make console pay decryption key fee 2015-09-22 12:08:17 -04:00
Jimmy Kiselak f94b04bb19 make class to encompass stream metadata 2015-09-21 22:06:04 -04:00
Jimmy Kiselak be1e97c616 show name claim options better, and fix bug when length is entered manually 2015-09-20 23:05:10 -04:00
Jimmy Kiselak 9790e9f843 put more fields into the value of a name claim, and fix bug where creating a plain stream descriptor file fails if no file name is given 2015-09-20 00:49:29 -04:00
Jimmy Kiselak e339d3857f make saving plain stream descriptor file work when no file name is given 2015-09-18 00:49:06 -04:00
Jimmy Kiselak 0e4f21842c when creating a new lbry file, make sure its status is set to completed 2015-09-18 00:20:07 -04:00
Jimmy Kiselak d4ea9a1ba3 rename downloader gui to simply gui 2015-09-18 00:09:53 -04:00
Jimmy Kiselak 5300fa929b make sure blobfiles directory is created in console, and make sure blobfiles dir is always inside the db dir for the gui 2015-09-17 23:47:36 -04:00
Jimmy Kiselak 8a5a66a06a in console disable putting blob dir outside of the db dir, and rename conf_dir to db_dir/data_dir 2015-09-17 23:10:14 -04:00
Jimmy Kiselak 43ed4427b4 stop the console from upgrading the db on a brand new conf dir 2015-09-17 12:05:08 -04:00
Jimmy Kiselak 22b0d92e6b disable the blind repeater 2015-09-17 11:03:57 -04:00
Jimmy Kiselak 7eb528232e enable getting a new address via the console 2015-09-17 10:59:57 -04:00
Jimmy Kiselak 33b6b85e72 better error messages in console when funds are exhausted 2015-09-17 01:43:41 -04:00
Jimmy Kiselak 8177119671 Fix how some errors are presented to the user
If an invalid stream descriptor hash is entered, abort and
give a useful error.

If a name doesn't resolve, abort and give a useful error.

If a name resolves to something invalid, abort and give a useful error.
2015-09-16 16:27:46 -04:00
Jimmy Kiselak 4b407a9dd2 shut down session last, actually save LBRY file metadata 2015-09-15 00:30:49 -04:00
kauffj 9b904a7b4a Rename INSTALL to INSTALL.md 2015-09-11 15:07:25 -04:00
kauffj e0424a1e38 Or it needs to be .md? 2015-09-11 15:06:59 -04:00
kauffj 00c08e05f5 github doesn't like setext markdown headers 2015-09-11 15:06:28 -04:00
Jimmy Kiselak 0836d9c5ee fix bug pulling blob length out of the database 2015-09-10 21:43:46 -04:00
Jimmy Kiselak ff5b6c4eab if lbrycrd server fails to launch, don't stop it at shutdown 2015-09-09 12:17:46 -04:00
Jimmy Kiselak bb5cd49327 Use named loggers, fix some pep8 formatting mistakes 2015-09-08 15:42:56 -04:00
Jimmy Kiselak 6f40ef6015 fix typo in migrator path, set up logging and running as a script for migrator, change args to windows migrator exe so they are all strings 2015-09-07 23:59:59 -04:00
Jimmy Kiselak c7758506ac remove leveldb, use sqlite and unqlite. create migrator tool 2015-09-04 16:22:02 -04:00
Jimmy Kiselak 192ac6959a more specific exceptions, and change when blobs are deleted
More specific exceptions are raised when problems are encountered
looking up metadata and validating stream descriptor files, and
on the GUI those more specific exceptions are used to prevent
errors from being presented to the user.

If the user has selected the option to re-upload data for some
stream, blobs downloaded for that purpose will not be deleted
when they are finished being output. Instead, by default, for
the GUI they will be deleted when the stream is removed from
the GUI. That can be changed so they are not deleted at all,
using the lbry.conf file.
2015-09-01 17:49:26 -04:00
Jimmy Kiselak 465a4643a8 change how the windows client finds pictures 2015-09-01 00:08:53 -04:00
Jimmy Kiselak 1bf0065d9d Show download progress in terms of percent, console startup errors now cause the application to abort, and a friendlier error is shown when no lbrycrd server is running 2015-08-31 22:05:50 -04:00
Jimmy Kiselak c14efc843c disable the URI entry bar until the app has finished starting up 2015-08-31 11:25:48 -04:00
Jimmy Kiselak 0506dfba10 when looking up peers, only return one instance of each peer 2015-08-31 11:14:25 -04:00
Jimmy Kiselak 7e4f36e8b4 Make sure errors resulting from connections failing don't make it to the user interface, as they are expected 2015-08-31 10:46:22 -04:00
Jimmy Kiselak 55dc482a61 enable running a server within lbrydownloader, and create a configuration file for lbrydownloader 2015-08-31 01:14:11 -04:00
Jimmy Kiselak 05b9f9cae6 refactor the downloading app a bit 2015-08-28 15:17:28 -04:00
Jimmy Kiselak c79868de3e check for specific vlc paths on windows 2015-08-28 11:47:21 -04:00
Jimmy Kiselak 7eeac060ab don't use distutils to check if vlc is available 2015-08-27 23:32:45 -04:00
Jimmy Kiselak c8b2b7b279 Downloader options in its own class, show options in gui downloader
Put stream downloader options into its own class, and make stream
downloader options global to the stream type rather than specific
to each factory.

Show downloader options in the lbrynet-downloader-gui.

Make a class for downloader option choices, so that the descriptions
can be displayed.

In the console, if there are multiple choices for the download
option, make it a list selected by its index.

Make sure that the ConnectionManager closes properly when some of
the connections fail to open (e.g. due to a host being down)
2015-08-27 15:41:17 -04:00
Jimmy Kiselak f4b3187494 use lbrycrd.conf in LBRYConsole to get rpc credentials 2015-08-24 13:19:03 -04:00
169 changed files with 14471 additions and 3817 deletions

No files matched your search

+10
View File
@@ -0,0 +1,10 @@
[bumpversion]
current_version = 0.3.20
commit = True
tag = True
message = Bump version: {current_version} -> {new_version}
[bumpversion:file:lbrynet/__init__.py]
[bumpversion:file:packaging/ubuntu/lbry.desktop]
+31
View File
@@ -1 +1,32 @@
*.pyc
*.xml
.idea/.name
*.iml
*.log
*.txt
lbrynet.egg-info/PKG-INFO
*.egg
/build
/dist
*.so
*.pem
*.decTest
.coverage
# temporary files from the twisted.trial test runner
_trial_temp/
.DS_Store
View File
Whitespace-only changes.
+379
View File
@@ -0,0 +1,379 @@
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
reports=yes
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=_$|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[BASIC]
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,input
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
[ELIF]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=100
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=twisted.internet.reactor,leveldb
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
ignored-classes=twisted.internet,RequestMessage
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
+71
View File
@@ -0,0 +1,71 @@
matrix:
include:
- os: linux
sudo: required
dist: trusty
# dh-virtualenv requires that we use the same python interpreter
# that comes with the system, so we don't want to use anything that
# travis would try to set-up for us in python
language: generic
- os: osx
# Use generic language for osx
# python 2.7 is broken on osx on travis, so follow we have to specify the installation ourselves
# https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
language: generic
osx_image: xcode7.3
notifications:
slack:
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
cache:
directories:
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $TRAVIS_BUILD_DIR/cache/wheel
before_install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi
install:
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi
before_script:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.cer.enc -d -a -out packaging/osx/certs/dist.cer; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.p12.enc -d -a -out packaging/osx/certs/dist.p12; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/osx/add-key.sh; fi
script:
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash packaging/ubuntu/ubuntu_package_setup.sh -t; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cd packaging/osx/lbry-osx-app && ./setup_app.sh && cd $TRAVIS_BUILD_DIR; fi
# fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass
- if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi
deploy:
- provider: releases
file: "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb"
skip_cleanup: true
prerelease: true
on:
tags: true
condition: "$TRAVIS_OS_NAME = linux"
# this is the oauth token for the lbry-ci user
api_key:
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
- provider: releases
file: "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg"
skip_cleanup: true
prerelease: true
on:
tags: true
condition: "$TRAVIS_OS_NAME = osx"
# this is the oauth token for the lbry-ci user
api_key:
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
env:
global:
# needed to unlock the identity file when its on the keychain
- secure: "aqgVNHfeh6JUqTTWG5+W+tTsIJkePS6HyLkcZlq6ODYrfdGKa90EeV4q4wdGfP5IkrBbf+WBFACDGVp574E7vfMLMNKUDuUtgnGVPwqOIhjG82ud8Xa7qF7lsw50QOnRYYVd2GLlCIzk8sffT5ncjSPN2ClNVM5iTwCkC8TNSAVnEJxu6bG6hcaT4uCWWjs0m+39O1xJwxJ4vTjpE/gy+j2FTSaUR5cNavOVCyJqpeKlga9aoBVsQHWvxlYurdWbwRLwVIV7bDE+sYPnwn1nMFQpx5RZ1AX1Z2UxNFKcYzLJgcWe85OjxLyT4udX+XZ9SLsdOjm1n201OLEKsmTxmHS8yqpbu6+pKQ1rLMEVpgPGfS8DdtqZyb3z0u6q4jztpm+uBe8hnFgwQFGXO63nOQsI0n1PMR3evAnVx7jYt7y/UALs2A9yjosMwDqgql1FMhyd0OJGo8Ky8YpDsr2J5zGVvIqt2/N+lP+SOe2D1J/5EhGlY0o4tqIAFskh3q8/GR6UGm70KT2l4LJqrU1WGrGxPJ+HoOEmvG6eqLesk03fAX3v5+DgXZWErnzXMIOGDPaFVpmx+G9VZTIlmf+3Wbu9TnZE5PRwFTdP1rqjGpjUhHeF0VTc2qgNq1OfL98CBO1wLgA195+em58cELalnwDMWUTmY1Jt1kUuAtCc11U="
# used to decypt the aes decrypt the certificate and identity files
- secure: "FzftiEBFMngQIci5KZ+tpKs/BubalhWXJ9f8yogGNVa3XplLheUXYgcsNU8sYT2MJaKAHN9fSjHWb67UtKT7yXddXxeOPnW6wJ3ua/FXnpynsmeGKTfh3stvgjpzdXous67uHmCWMMklfb6z7UuDohjUMAe5n5HKw5tq1RzTKpc2kJacOC6qUT5laUOvULyCaO9E9HmbHeR5ZeXAC6pnzX2ccsSrcXvPozHzBIZ9RyothKs+CZw8PEuJo07RRL8meboegqYOUrOYuz6A2gS2mZJoy59ivZKOFxS5shEuv2Jt80RyfyxBoUpKFq8OG3Am5nAEzDiTIGzmIoKDEGKeTagk/sEtSZXiMDkzDT4GX/j1rUNLCBU87bXEFS2zfRsrfg8c1XZPIzDYBT1PY2QtLBdddF5zzDoKdPLJ3sjN+fZFE5RlnwfwnMHriVRZZlzjcdk0Z06gKTBCxUg5BZamnOOK+K8qunMJXVS+Vmi5u4RoTZZiCosUlYKnSKJ8suO9C0+znxoViusPqP4ONprNHgDoZw+UKio84QW3PrZv9h4zH/D+msDgJRZ0ceqDD+6Wz1J8Mm5ptW1GOLh/IU12TPXjxteqh0Um2vv5eIPmjK9uEK666kK7PqtPDkYhAfWvF+nmOOyPMJfbP4MW/i9WHNF4ghsIMbPKfqNhgSmfrYw="
+85
View File
@@ -0,0 +1,85 @@
#### Getting LBRY for development
Q: How do I get lbry for command line?
A: In order to run lbry from command line, you need more than the packaged app/deb.
######On OS X
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
######On Linux
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
git clone https://github.com/lbryio/lbry.git
cd lbry
sudo python setup.py install
#### Using LBRY
Q: How do I run lbry from command line?
A: The command is `lbrynet-daemon`
***********
Q: How do I stop lbry from the command line?
A: You can ctrl-c or run `stop-lbrynet-daemon`
***********
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
***********
Q: Where are all the behind the scenes files?
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
***********
Q: How can I see the log in the console?
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
***********
Q: How do I specify a web-UI to use?
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
***********
Q: How do I see the list of API functions I can call, and how do I call them?
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
import sys
from jsonrpc.proxy import JSONRPCProxy
try:
from lbrynet.conf import API_CONNECTION_STRING
except:
print "You don't have lbrynet installed!"
sys.exit(0)
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
if not api.is_running():
print api.daemon_status()
else:
for func in api.help():
print "%s:\n%s" % (func, api.help({'function': func}))
-34
View File
@@ -1,34 +0,0 @@
Prerequisites
-------------
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
this is not desired, the testing client can be used to simulate trading points, which is
built into LBRYnet.
on Ubuntu:
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
Getting the source
------------------
Don't you already have it?
Setting up the environment
--------------------------
It's recommended that you use a virtualenv
sudo apt-get install python-virtualenv
cd <source base directory>
virtualenv .
source bin/activate
(to deactivate the virtualenv, enter 'deactivate')
python setup.py install
this will install all of the libraries and a few applications
For running the file sharing application, see RUNNING
+67
View File
@@ -0,0 +1,67 @@
#### Installing the LBRY app
--------------------------
Installing LBRY is simple. You can get a dmg installer for OS X (Mavericks and up) or a .deb for linux [here](https://lbry.io/get).
##### OS X
Just drag and drop LBRY.app into your applications folder (replacing any older versions). When it's running you'll have a LBRY icon in your status bar.
##### Linux
Double click the .deb file and follow the prompts. The app can be started by searching "LBRY", and it can be turned off by clicking the red 'x' in the browser interface.
On both systems you can also open the UI while the app is running by going to lbry://lbry in Firefox or Safari, or localhost:5279 in Chrome.
#### Installing LBRY command line
--------------------------
##### OS X
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app. You can start LBRY by either starting the app or by running `lbrynet-daemon` from a terminal.
##### Linux
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
```
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
git clone https://github.com/lbryio/lbry.git
cd lbry
sudo python setup.py install
```
To start LBRY, run `lbrynet-daemon` in a terminal.
#### On windows:
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
Add C:\MinGW\bin to the windows PATH.
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing
```
[build]
compiler = mingw32
```
If using virtualenv, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
Currently, available binaries include:
- Cython
- Twisted
- Zope.interface
- pywin32
- Yapsy
- cx_Freeze
- requests
- gmpy
Install the each of the preceding binaries with `pip install *.whl`
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
Install the rest of the required modules with the standard `pip install module` command
The one module which requires additional work is [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9).
Download the source and compile with MinGW by running `mingw32make.bat`
Then install the module by running `python setupmingw32.py install`
+2
View File
@@ -0,0 +1,2 @@
include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin
recursive-include lbrynet/lbrynet_gui *.gif *.ico *.xbm *.conf
+39 -24
View File
@@ -1,12 +1,16 @@
LBRYnet
=======
[![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry)
# LBRYnet
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
table, used by peers to discover other peers.
Overview
--------
## Installation
Download the [latest release](https://github.com/lbryio/lbry/releases/latest) or see [INSTALL.md](INSTALL.md) for manual installation.
## Overview
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
guarantees that peers can verify the correctness of each chunk without having to know anything
@@ -20,38 +24,49 @@ help peers find each other. For example, an application for which clients don't
necessary chunks may use some identifier, chosen by the application, to find clients which do
know all of the necessary chunks.
Running
-------
## For Developers
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
The bundled LBRY application uses the lbrynet JSONRPC api found in `lbrynet.lbrynet_daemon.LBRYDaemon`. This api allows for applications and web services like the lbry browser UI to interact with lbrynet. If you've installed lbrynet, you can run `lbrynet-daemon` without running the app. While the app or `lbrynet-daemon` is running, you can use the following to show the help for all the available commands:
```
from jsonrpc.proxy import JSONRPCProxy
try:
from lbrynet.conf import API_CONNECTION_STRING
except:
print "You don't have lbrynet installed!"
API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
if not api.is_running():
print api.daemon_status()
else:
for func in api.help():
print "%s:\n%s" % (func, api.help({'function': func}))
```
If you've installed lbrynet, it comes with a file sharing application, called `lbrynet-daemon`, which breaks
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
and makes the chunks available for download by other peers. A peer wishing to download the file
must first obtain the 'stream descriptor' and then may open it with his 'lbrynet-console' client,
must first obtain the 'stream descriptor' and then may open it with his `lbrynet-daemon` client,
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
chunks into the original file, according to the metadata included in the 'stream descriptor'.
To install and use this client, see INSTALL and RUNNING
For detailed instructions, see [INSTALL.md](INSTALL.md) and [RUNNING.md](RUNNING.md).
Installation
------------
Documentation: doc.lbry.io (may be out of date)
See INSTALL
Source code: https://github.com/lbryio/lbry
Developers
----------
To contribute, [join us on Slack](https://lbry-slackin.herokuapp.com/) or contact josh@lbry.io. Pull requests are also welcome.
Documentation: doc.lbry.io
Source code: trac.lbry.io/browser
## Support
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
Please open an issue and describe your situation in detail. We will respond as soon as we can.
Support
-------
For private issues, contact josh@lbry.io.
Send all support requests to jimmy@lbry.io
## License
License
-------
See LICENSE
See [LICENSE](LICENSE)
-52
View File
@@ -1,52 +0,0 @@
To install LBRYnet and lbrynet-console, see INSTALL
lbrynet-console is a console application which makes use of the LBRYnet to share files.
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
others wishing to obtain the file, or can be itself turned into a chunk compatible with
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
specified by the hash sums found in the stream descriptor file, decrypts them according to
metadata found in the stream, and reconstructs the original file. lbrynet-console features
a server so that clients can connect to it and download the chunks and other data gotten
from files created locally and files that have been downloaded from LBRYnet.
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
concept which is currently far behind the development of the rest of the application and
therefore will not run, and a plugin which attempts to determine which chunks on the
network should be downloaded in order for the application to turn a profit. It will run,
but its usefulness is extremely limited.
Passing '--help' to lbrynet-console will cause it to print out a quick help message
describing other command line options to the application.
Once the application has been started, the user is presented with a numbered list of
actions which looks something like this:
...
[2] Toggle whether an LBRY File is running
[3] Create an LBRY File from file
[4] Publish a stream descriptor file to the DHT for an LBRY File
...
To perform an action, type the desired number and then hit enter. For example, if you wish
to create an LBRY file from a file as described in the beginning of this document, type 3 and
hit enter.
If the application needs more input in order to for the action to be taken, the application
will continue to print prompts for input until it has received what it needs.
For example, when creating an LBRY file from a file, the application needs to know which file
it's supposed to use to create the LBRY file, so the user will be prompted for it:
File name:
The user should input the desired file name and hit enter, at which point the application
will go about splitting the file and making it available on the network.
Some actions will produce sub-menus of actions, which work the same way.
A more detailed user guide is available at doc.lbry.io
Any issues may be reported to jimmy@lbry.io
+146
View File
@@ -0,0 +1,146 @@
How to watch It's a Wonderful Life via LBRY
## Quickest quick guide
Create a directory called lbry, and go into that directory
Download the file https://raw.githubusercontent.com/lbryio/lbry-setup/master/lbry_setup.sh and run it in that directory
Once it's done building, type:
```
lbrynet-console
```
A console application will load, and after a moment you will be presented with a ">" signifying
that the console is ready for commands.
If it's your first time running lbrynet-console, you should be given some credits to test the
network. If they don't show up within a minute or two, let us know, and we'll send you some.
After your credits have shown up, type
```
get wonderfullife
```
into the prompt and hit enter.
You will be asked if you want to cancel, change options, save, and perhaps stream the file.
Enter 's' for save and then hit enter.
The file should start downloading. Enter the command 'status' to check on the status of files
that are in the process of being downloaded.
To stop lbrynet-console, enter the command 'exit'.
## Slightly longer install guide
### Installing lbrycrd, the full blockchain client
Note: this process takes upwards of an hour and is not necessary to use lbrynet.
```
sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler git
git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git
cd lbrycrd
./autogen.sh
./configure --with-incompatible-bdb --without-gui
make
```
When make has completed, create the directory where LBRYcrd data will be stored. ~/.lbrycrd is where LBRYcrd will look by default and so is recommended.
```
mkdir ~/.lbrycrd
echo 'rpcuser=rpcuser
rpcpassword=rpcpassword' > ~/.lbrycrd/lbrycrd.conf
# (use a long random password if your computer is on a network anyone else has access to. e.g, pwgen -s 20)
cd ..
```
### Installing lbrynet from source
Acquire the LBRYnet source code from https://github.com/lbryio/lbry
```
cd lbry
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
```
(with virtualenv)
```
python-virtualenv
virtualenv .
source bin/activate
python setup.py install
```
to deactivate the virtualenv later:
```
deactivate
```
to reactivate it, go to the directory in which you created it and:
```
source bin/activate
```
(without virtualenv)
```
python setup.py build bdist_egg
sudo python setup.py install
```
## Slightly longer running guide
### lbrynet-console can be set to use lbrycrdd instead of the built in lightweight client.
To run lbrynet-console with lbrycrdd:
```
lbrynet-console </path/to/lbrycrdd>
```
If lbrycrdd is not already running, lbrynet will launch it at that path, and will shut it down
when lbrynet exits. If lbrycrdd is already running, lbrynet will not launch it and will not
shut it down, but it will connect to it and use it.
### Running lbrycrdd manually
From the lbrycrd directory, run:
```
./src/lbrycrdd -server -daemon
```
If you want to mine LBC, also use the flag '-gen', so:
```
./src/lbrycrdd -server -daemon -gen
```
Warning: This will put a heavy load on your CPU
It will take a few minutes for your client to download the whole block chain.
To shut lbrycrdd down: from the lbrycrd directory, run:
```
./src/lbrycrd-cli stop
```
Any questions or problems, email jimmy@lbry.io
BIN
View File
Binary file not shown.
+2
View File
@@ -0,0 +1,2 @@
__version__ = "0.3.20"
version = tuple(__version__.split('.'))
+2
View File
@@ -0,0 +1,2 @@
from events import *
from api import AnalyticsApi as Api
+71
View File
@@ -0,0 +1,71 @@
import functools
import json
import logging
from requests import auth
from requests_futures import sessions
from lbrynet import conf
from lbrynet.analytics import utils
log = logging.getLogger(__name__)
def log_response(fn):
def _log(future):
if future.cancelled():
log.warning('Request was unexpectedly cancelled')
else:
response = future.result()
log.debug('Response (%s): %s', response.status_code, response.content)
@functools.wraps(fn)
def wrapper(*args, **kwargs):
future = fn(*args, **kwargs)
future.add_done_callback(_log)
return future
return wrapper
class AnalyticsApi(object):
def __init__(self, session, url, write_key):
self.session = session
self.url = url
self.write_key = write_key
@property
def auth(self):
return auth.HTTPBasicAuth(self.write_key, '')
@log_response
def batch(self, events):
"""Send multiple events in one request.
Each event needs to have its type specified.
"""
data = json.dumps({
'batch': events,
'sentAt': utils.now(),
})
log.debug('sending %s events', len(events))
log.debug('Data: %s', data)
return self.session.post(self.url + '/batch', json=data, auth=self.auth)
@log_response
def track(self, event):
"""Send a single tracking event"""
log.debug('Sending track event: %s', event)
import base64
return self.session.post(self.url + '/track', json=event, auth=self.auth)
@classmethod
def load(cls, session=None):
"""Initialize an instance using values from lbry.io."""
if not session:
session = sessions.FuturesSession()
return cls(
session,
conf.ANALYTICS_ENDPOINT,
utils.deobfuscate(conf.ANALYTICS_TOKEN)
)
+47
View File
@@ -0,0 +1,47 @@
from lbrynet.analytics import utils
class Events(object):
def __init__(self, context, lbry_id, session_id):
self.context = context
self.lbry_id = lbry_id
self.session_id = session_id
def heartbeat(self):
return {
'userId': 'lbry',
'event': 'Heartbeat',
'properties': {
'lbry_id': self.lbry_id,
'session_id': self.session_id
},
'context': self.context,
'timestamp': utils.now()
}
def make_context(platform, wallet, is_dev=False):
# TODO: distinguish between developer and release instances
return {
'is_dev': is_dev,
'app': {
'name': 'lbrynet',
'version': platform['lbrynet_version'],
'ui_version': platform['ui_version'],
'python_version': platform['python_version'],
'wallet': {
'name': wallet,
# TODO: add in version info for lbrycrdd
'version': platform['lbryum_version'] if wallet == 'lbryum' else None
},
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
'library': {
'name': 'lbrynet-analytics',
'version': '1.0.0'
},
}
+8
View File
@@ -0,0 +1,8 @@
import datetime
from lbrynet.core.utils import *
def now():
"""Return utc now in isoformat with timezone"""
return datetime.datetime.utcnow().isoformat() + 'Z'
+50 -8
View File
@@ -3,9 +3,6 @@ Some network wide and also application specific parameters
"""
import os
MAX_HANDSHAKE_SIZE = 2**16
MAX_REQUEST_SIZE = 2**16
MAX_BLOB_REQUEST_SIZE = 2**16
@@ -13,13 +10,58 @@ MAX_RESPONSE_INFO_SIZE = 2**16
MAX_BLOB_INFOS_TO_REQUEST = 20
BLOBFILES_DIR = ".blobfiles"
BLOB_SIZE = 2**21
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte
MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos
MAX_CONNECTIONS_PER_STREAM = 5
KNOWN_DHT_NODES = [('104.236.42.182', 4000),
('lbrynet1.lbry.io', 4444),
('lbrynet2.lbry.io', 4444),
('lbrynet3.lbry.io', 4444)]
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005",
"http://lighthouse2.lbry.io:50005",
"http://lighthouse3.lbry.io:50005"]
CRYPTSD_FILE_EXTENSION = ".cryptsd"
REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)]
LOG_FILE_NAME = "lbrynet.log"
LOG_POST_URL = "https://lbry.io/log-upload"
CRYPTSD_FILE_EXTENSION = ".cryptsd"
API_INTERFACE = "localhost"
API_ADDRESS = "lbryapi"
API_PORT = 5279
ICON_PATH = "app.icns"
APP_NAME = "LBRY"
API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT)
PROTOCOL_PREFIX = "lbry"
DEFAULT_WALLET = "lbryum"
WALLET_TYPES = ["lbryum", "lbrycrd"]
DEFAULT_TIMEOUT = 30
DEFAULT_MAX_SEARCH_RESULTS = 25
DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}}
DEFAULT_SEARCH_TIMEOUT = 3.0
DEFAULT_SD_DOWNLOAD_TIMEOUT = 3
DEFAULT_CACHE_TIME = 3600
DEFAULT_UI_BRANCH = "master"
SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
CURRENCIES = {
'BTC': {'type': 'crypto'},
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
+189 -81
View File
@@ -1,14 +1,19 @@
import logging
import os
import leveldb
import time
import json
from twisted.internet import threads, defer, reactor, task
import sqlite3
from twisted.internet import threads, defer
from twisted.python.failure import Failure
from twisted.enterprise import adbapi
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
from lbrynet.core.utils import is_valid_blobhash
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.core.Error import NoSuchBlobError
from lbrynet.core.sqlite_helpers import rerun_if_locked
log = logging.getLogger(__name__)
class BlobManager(DHTHashSupplier):
@@ -68,8 +73,8 @@ class DiskBlobManager(BlobManager):
def __init__(self, hash_announcer, blob_dir, db_dir):
BlobManager.__init__(self, hash_announcer)
self.blob_dir = blob_dir
self.db_dir = db_dir
self.db = None
self.db_file = os.path.join(db_dir, "blobs.db")
self.db_conn = None
self.blob_type = BlobFile
self.blob_creator_type = BlobFileCreator
self.blobs = {}
@@ -77,15 +82,19 @@ class DiskBlobManager(BlobManager):
self._next_manage_call = None
def setup(self):
d = threads.deferToThread(self._open_db)
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
str(self.db_file))
d = self._open_db()
d.addCallback(lambda _: self._manage())
return d
def stop(self):
log.info("Stopping the DiskBlobManager")
if self._next_manage_call is not None and self._next_manage_call.active():
self._next_manage_call.cancel()
self._next_manage_call = None
self.db = None
#d = self.db_conn.close()
self.db_conn = None
return defer.succeed(True)
def get_blob(self, blob_hash, upload_allowed, length=None):
@@ -101,7 +110,7 @@ class DiskBlobManager(BlobManager):
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
self.blobs[blob_hash] = blob
d = threads.deferToThread(self._completed_blobs, [blob_hash])
d = self._completed_blobs([blob_hash])
def check_completed(completed_blobs):
@@ -110,7 +119,7 @@ class DiskBlobManager(BlobManager):
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
blob.verified = True
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
inner_d = self._get_blob_length(blob_hash)
inner_d.addCallback(set_length)
inner_d.addCallback(lambda _: blob)
else:
@@ -123,18 +132,18 @@ class DiskBlobManager(BlobManager):
def blob_completed(self, blob, next_announce_time=None):
if next_announce_time is None:
next_announce_time = time.time()
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
time.time(), next_announce_time)
return self._add_completed_blob(blob.blob_hash, blob.length,
time.time(), next_announce_time)
def completed_blobs(self, blobs_to_check):
return threads.deferToThread(self._completed_blobs, blobs_to_check)
return self._completed_blobs(blobs_to_check)
def hashes_to_announce(self):
next_announce_time = time.time() + self.hash_reannounce_time
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
return self._get_blobs_to_announce(next_announce_time)
def creator_finished(self, blob_creator):
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
assert blob_creator.blob_hash is not None
assert blob_creator.blob_hash not in self.blobs
assert blob_creator.length is not None
@@ -155,18 +164,18 @@ class DiskBlobManager(BlobManager):
self.blob_hashes_to_delete[blob_hash] = False
def update_all_last_verified_dates(self, timestamp):
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
return self._update_all_last_verified_dates(timestamp)
def immediate_announce_all_blobs(self):
d = threads.deferToThread(self._get_all_verified_blob_hashes)
d = self._get_all_verified_blob_hashes()
d.addCallback(self.hash_announcer.immediate_announce)
return d
def get_blob_length(self, blob_hash):
return threads.deferToThread(self._get_blob_length, blob_hash)
return self._get_blob_length(blob_hash)
def check_consistency(self):
return threads.deferToThread(self._check_consistency)
return self._check_consistency()
def _manage(self):
from twisted.internet import reactor
@@ -185,19 +194,19 @@ class DiskBlobManager(BlobManager):
return b_h
def set_not_deleting(err, b_h):
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
self.blob_hashes_to_delete[b_h] = False
return err
def delete_from_db(result):
b_hs = [r[1] for r in result if r[0] is True]
if b_hs:
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
d = self._delete_blobs_from_db(b_hs)
else:
d = defer.succeed(True)
def log_error(err):
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
log.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
d.addErrback(log_error)
return d
@@ -221,72 +230,127 @@ class DiskBlobManager(BlobManager):
######### database calls #########
def _open_db(self):
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
return self.db_conn.runQuery("create table if not exists blobs (" +
" blob_hash text primary key, " +
" blob_length integer, " +
" last_verified_time real, " +
" next_announce_time real"
")")
@rerun_if_locked
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
if next_announce_time is None:
next_announce_time = timestamp
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
d = self.db_conn.runQuery("insert into blobs values (?, ?, ?, ?)",
(blob_hash, length, timestamp, next_announce_time))
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
return d
@rerun_if_locked
def _completed_blobs(self, blobs_to_check):
blobs = []
for b in blobs_to_check:
if is_valid_blobhash(b):
try:
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
except KeyError:
continue
blobs_to_check = filter(is_valid_blobhash, blobs_to_check)
def get_blobs_in_db(db_transaction):
blobs_in_db = [] # [(blob_hash, last_verified_time)]
for b in blobs_to_check:
result = db_transaction.execute("select last_verified_time from blobs where blob_hash = ?",
(b,))
row = result.fetchone()
if row is not None:
blobs_in_db.append((b, row[0]))
return blobs_in_db
def get_valid_blobs(blobs_in_db):
def check_blob_verified_date(b, verified_time):
file_path = os.path.join(self.blob_dir, b)
if os.path.isfile(file_path):
if verified_time > os.path.getctime(file_path):
blobs.append(b)
return blobs
return True
return False
def return_valid_blobs(results):
valid_blobs = []
for (b, verified_date), (success, result) in zip(blobs_in_db, results):
if success is True and result is True:
valid_blobs.append(b)
return valid_blobs
ds = []
for b, verified_date in blobs_in_db:
ds.append(threads.deferToThread(check_blob_verified_date, b, verified_date))
dl = defer.DeferredList(ds)
dl.addCallback(return_valid_blobs)
return dl
d = self.db_conn.runInteraction(get_blobs_in_db)
d.addCallback(get_valid_blobs)
return d
@rerun_if_locked
def _get_blob_length(self, blob):
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
return length
d = self.db_conn.runQuery("select blob_length from blobs where blob_hash = ?", (blob,))
d.addCallback(lambda r: r[0][0] if len(r) else Failure(NoSuchBlobError(blob)))
return d
#length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
#return length
@rerun_if_locked
def _update_blob_verified_timestamp(self, blob, timestamp):
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
(blob, timestamp))
@rerun_if_locked
def _get_blobs_to_announce(self, next_announce_time):
# TODO: See if the following would be better for handling announce times:
# TODO: Have a separate db for them, and read the whole thing into memory
# TODO: on startup, and then write changes to db when they happen
blobs = []
batch = leveldb.WriteBatch()
current_time = time.time()
for blob_hash, blob_info in self.db.RangeIter():
length, verified_time, announce_time = json.loads(blob_info)
if announce_time < current_time:
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
blobs.append(blob_hash)
self.db.Write(batch, sync=True)
return blobs
def get_and_update(transaction):
timestamp = time.time()
r = transaction.execute("select blob_hash from blobs " +
"where next_announce_time < ? and blob_hash is not null",
(timestamp,))
blobs = [b for b, in r.fetchall()]
transaction.execute("update blobs set next_announce_time = ? where next_announce_time < ?",
(next_announce_time, timestamp))
return blobs
return self.db_conn.runInteraction(get_and_update)
@rerun_if_locked
def _update_all_last_verified_dates(self, timestamp):
batch = leveldb.WriteBatch()
for blob_hash, blob_info in self.db.RangeIter():
length, verified_time, announce_time = json.loads(blob_info)
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
self.db.Write(batch, sync=True)
return self.db_conn.runQuery("update blobs set last_verified_date = ?", (timestamp,))
@rerun_if_locked
def _delete_blobs_from_db(self, blob_hashes):
batch = leveldb.WriteBatch()
for blob_hash in blob_hashes:
batch.Delete(blob_hash)
self.db.Write(batch, sync=True)
def delete_blobs(transaction):
for b in blob_hashes:
transaction.execute("delete from blobs where blob_hash = ?", (b,))
return self.db_conn.runInteraction(delete_blobs)
@rerun_if_locked
def _check_consistency(self):
batch = leveldb.WriteBatch()
ALREADY_VERIFIED = 1
NEWLY_VERIFIED = 2
INVALID = 3
current_time = time.time()
for blob_hash, blob_info in self.db.RangeIter():
length, verified_time, announce_time = json.loads(blob_info)
d = self.db_conn.runQuery("select blob_hash, blob_length, last_verified_time from blobs")
def check_blob(blob_hash, blob_length, verified_time):
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
if verified_time < os.path.getctime(file_path):
if verified_time >= os.path.getctime(file_path):
return ALREADY_VERIFIED
else:
h = get_lbry_hash_obj()
len_so_far = 0
f = open(file_path)
@@ -296,19 +360,63 @@ class DiskBlobManager(BlobManager):
break
h.update(data)
len_so_far += len(data)
if len_so_far == length and h.hexdigest() == blob_hash:
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
self.db.Write(batch, sync=True)
if len_so_far == blob_length and h.hexdigest() == blob_hash:
return NEWLY_VERIFIED
return INVALID
def do_check(blobs):
already_verified = []
newly_verified = []
invalid = []
for blob_hash, blob_length, verified_time in blobs:
status = check_blob(blob_hash, blob_length, verified_time)
if status == ALREADY_VERIFIED:
already_verified.append(blob_hash)
elif status == NEWLY_VERIFIED:
newly_verified.append(blob_hash)
else:
invalid.append(blob_hash)
return already_verified, newly_verified, invalid
def update_newly_verified(transaction, blobs):
for b in blobs:
transaction.execute("update blobs set last_verified_time = ? where blob_hash = ?",
(current_time, b))
def check_blobs(blobs):
@rerun_if_locked
def update_and_return(status_lists):
already_verified, newly_verified, invalid = status_lists
d = self.db_conn.runInteraction(update_newly_verified, newly_verified)
d.addCallback(lambda _: status_lists)
return d
d = threads.deferToThread(do_check, blobs)
d.addCallback(update_and_return)
return d
d.addCallback(check_blobs)
return d
@rerun_if_locked
def _get_all_verified_blob_hashes(self):
blob_hashes = []
for blob_hash, blob_info in self.db.RangeIter():
length, verified_time, announce_time = json.loads(blob_info)
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
if verified_time > os.path.getctime(file_path):
blob_hashes.append(blob_hash)
return blob_hashes
d = self.db_conn.runQuery("select blob_hash, last_verified_time from blobs")
def get_verified_blobs(blobs):
verified_blobs = []
for blob_hash, verified_time in blobs:
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
if verified_time > os.path.getctime(file_path):
verified_blobs.append(blob_hash)
return verified_blobs
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
return d
class TempBlobManager(BlobManager):
@@ -389,7 +497,7 @@ class TempBlobManager(BlobManager):
if blob_hash in self.blobs:
if self.blobs[blob_hash].length is not None:
return defer.succeed(self.blobs[blob_hash].length)
return defer.fail(ValueError("No such blob hash is known"))
return defer.fail(NoSuchBlobError(blob_hash))
def immediate_announce_all_blobs(self):
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
@@ -400,7 +508,7 @@ class TempBlobManager(BlobManager):
d = self._delete_blobs_marked_for_deletion()
def set_next_manage_call():
logging.info("Setting the next manage call in %s", str(self))
log.info("Setting the next manage call in %s", str(self))
self._next_manage_call = reactor.callLater(1, self._manage)
d.addCallback(lambda _: set_next_manage_call())
@@ -409,11 +517,11 @@ class TempBlobManager(BlobManager):
def remove_from_list(b_h):
del self.blob_hashes_to_delete[b_h]
logging.info("Deleted blob %s", blob_hash)
log.info("Deleted blob %s", blob_hash)
return b_h
def set_not_deleting(err, b_h):
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
self.blob_hashes_to_delete[b_h] = False
return b_h
@@ -422,7 +530,7 @@ class TempBlobManager(BlobManager):
if being_deleted is False:
if blob_hash in self.blobs:
self.blob_hashes_to_delete[blob_hash] = True
logging.info("Found a blob marked for deletion: %s", blob_hash)
log.info("Found a blob marked for deletion: %s", blob_hash)
blob = self.blobs[blob_hash]
d = blob.delete()
@@ -432,7 +540,7 @@ class TempBlobManager(BlobManager):
ds.append(d)
else:
remove_from_list(blob_hash)
d = defer.fail(Failure(ValueError("No such blob known")))
logging.warning("Blob %s cannot be deleted because it is unknown")
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
log.warning("Blob %s cannot be deleted because it is unknown")
ds.append(d)
return defer.DeferredList(ds)
+17 -2
View File
@@ -1,6 +1,21 @@
class DownloadOptionChoice(object):
"""A possible choice that can be picked for some option.
An option can have one or more choices that can be picked from.
"""
def __init__(self, value, short_description, long_description, bool_options_description=None):
self.value = value
self.short_description = short_description
self.long_description = long_description
self.bool_options_description = bool_options_description
class DownloadOption(object):
def __init__(self, option_types, long_description, short_description, default):
"""An option for a user to select a value from several different choices."""
def __init__(self, option_types, long_description, short_description, default_value,
default_value_description):
self.option_types = option_types
self.long_description = long_description
self.short_description = short_description
self.default = default
self.default_value = default_value
self.default_value_description = default_value_description
+42
View File
@@ -22,15 +22,45 @@ class ConnectionClosedBeforeResponseError(Exception):
pass
class KeyFeeAboveMaxAllowed(Exception):
pass
class UnknownNameError(Exception):
def __init__(self, name):
self.name = name
def __str__(self):
return repr(self.name)
class InvalidNameError(Exception):
def __init__(self, name):
self.name = name
def __str__(self):
return repr(self.name)
class UnknownStreamTypeError(Exception):
def __init__(self, stream_type):
self.stream_type = stream_type
def __str__(self):
return repr(self.stream_type)
class InvalidStreamDescriptorError(Exception):
pass
class InvalidStreamInfoError(Exception):
def __init__(self, name):
self.name = name
def __str__(self):
return repr(self.name)
class MisbehavingPeerError(Exception):
pass
@@ -45,4 +75,16 @@ class NoResponseError(MisbehavingPeerError):
class InvalidResponseError(MisbehavingPeerError):
pass
class NoSuchBlobError(Exception):
pass
class NoSuchStreamHashError(Exception):
pass
class InvalidBlobHashError(Exception):
pass
+13 -7
View File
@@ -13,6 +13,9 @@ from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class HashBlobReader(object):
implements(interfaces.IConsumer)
@@ -56,6 +59,9 @@ class HashBlobWriter(object):
" %s to %s" % (str(self.len_so_far),
str(self.length_getter())))))
else:
if self.write_handle is None:
log.debug("Tried to write to a write_handle that was None.")
return
self.write_handle.write(data)
if self.len_so_far == self.length_getter():
self.finished_cb(self)
@@ -84,7 +90,7 @@ class HashBlob(object):
if self.length is None and 0 <= length <= BLOB_SIZE:
self.length = length
return True
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
return False
def get_length(self):
@@ -126,8 +132,8 @@ class HashBlob(object):
finished_deferred.callback(self)
del self.writers[p]
return True
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
str(writer))
log.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
str(writer))
return False
def errback_finished_deferred(err):
@@ -199,14 +205,14 @@ class BlobFile(HashBlob):
def open_for_writing(self, peer):
if not peer in self.writers:
logging.debug("Opening %s to be written by %s", str(self), str(peer))
log.debug("Opening %s to be written by %s", str(self), str(peer))
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
finished_deferred = defer.Deferred()
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
self.writers[peer] = (writer, finished_deferred)
return finished_deferred, writer.write, writer.cancel
logging.warning("Tried to download the same file twice simultaneously from the same peer")
log.warning("Tried to download the same file twice simultaneously from the same peer")
return None, None, None
def open_for_reading(self):
@@ -232,7 +238,7 @@ class BlobFile(HashBlob):
d = threads.deferToThread(delete_from_file_system)
def log_error(err):
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
log.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
return err
d.addErrback(log_error)
@@ -247,7 +253,7 @@ class BlobFile(HashBlob):
def _close_writer(self, writer):
if writer.write_handle is not None:
logging.debug("Closing %s", str(self))
log.debug("Closing %s", str(self))
name = writer.write_handle.name
writer.write_handle.close()
threads.deferToThread(os.remove, name)
+135
View File
@@ -0,0 +1,135 @@
import json
from copy import deepcopy
from lbrynet.conf import CURRENCIES
from lbrynet.core import utils
import logging
log = logging.getLogger(__name__)
BITTREX_FEE = 0.0025
# Metadata version
SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-'
BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources']
OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey']
MV001 = "0.0.1"
MV002 = "0.0.2"
CURRENT_METADATA_VERSION = MV002
METADATA_REVISIONS = {}
METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS}
METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']}
# Fee version
BASE_FEE_FIELDS = ['amount', 'address']
FV001 = "0.0.1"
CURRENT_FEE_REVISION = FV001
FEE_REVISIONS = {}
FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []}
def verify_name_characters(name):
for c in name:
assert c in NAME_ALLOWED_CHARSET, "Invalid character"
return True
class LBRYFeeValidator(dict):
def __init__(self, fee_dict):
dict.__init__(self)
assert len(fee_dict) == 1
self.fee_version = None
self.currency_symbol = None
fee_to_load = deepcopy(fee_dict)
for currency in fee_dict:
self._verify_fee(currency, fee_to_load)
self.amount = self._get_amount()
self.address = self[self.currency_symbol]['address']
def _get_amount(self):
amt = self[self.currency_symbol]['amount']
if isinstance(amt, float):
return amt
else:
try:
return float(amt)
except TypeError:
log.error('Failed to convert %s to float', amt)
raise
def _verify_fee(self, currency, f):
# str in case someone made a claim with a wierd fee
assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency)
self.currency_symbol = currency
self.update({currency: {}})
for version in FEE_REVISIONS:
self._load_revision(version, f)
if not f:
self.fee_version = version
break
assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys())
def _load_revision(self, version, f):
for k in FEE_REVISIONS[version]['required']:
assert k in f[self.currency_symbol], "Missing required fee field: %s" % k
self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
for k in FEE_REVISIONS[version]['optional']:
if k in f[self.currency_symbol]:
self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)})
class Metadata(dict):
@classmethod
def load_from_hex(cls, metadata):
return cls(json.loads(metadata.decode('hex')))
def __init__(self, metadata):
dict.__init__(self)
self.meta_version = None
metadata_to_load = deepcopy(metadata)
self._verify_sources(metadata_to_load)
self._verify_metadata(metadata_to_load)
def _load_revision(self, version, metadata):
for k in METADATA_REVISIONS[version]['required']:
assert k in metadata, "Missing required metadata field: %s" % k
self.update({k: metadata.pop(k)})
for k in METADATA_REVISIONS[version]['optional']:
if k == 'fee':
self._load_fee(metadata)
elif k in metadata:
self.update({k: metadata.pop(k)})
def _load_fee(self, metadata):
if 'fee' in metadata:
self['fee'] = LBRYFeeValidator(metadata.pop('fee'))
def _verify_sources(self, metadata):
assert "sources" in metadata, "No sources given"
for source in metadata['sources']:
assert source in SOURCE_TYPES, "Unknown source type"
def _verify_metadata(self, metadata):
for version in METADATA_REVISIONS:
self._load_revision(version, metadata)
if not metadata:
self.meta_version = version
if utils.version_is_greater_than(self.meta_version, "0.0.1"):
assert self.meta_version == self['ver'], "version mismatch"
break
assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys())
def serialize(self):
return json.dumps(self).encode("hex")
def as_json(self):
return json.dumps(self)
File diff suppressed because it is too large. Load diff
-468
View File
@@ -1,468 +0,0 @@
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
from twisted.internet import threads, reactor, defer, task
from twisted.python.failure import Failure
from collections import defaultdict, deque
from zope.interface import implements
from decimal import Decimal
import datetime
import logging
import json
import subprocess
import socket
import time
import os
class ReservedPoints(object):
def __init__(self, identifier, amount):
self.identifier = identifier
self.amount = amount
class LBRYcrdWallet(object):
"""This class implements the LBRYWallet interface for the LBRYcrd payment system"""
implements(ILBRYWallet)
def __init__(self, rpc_user, rpc_pass, rpc_url, rpc_port, start_lbrycrdd=False,
wallet_dir=None, wallet_conf=None):
self.rpc_conn_string = "http://%s:%s@%s:%s" % (rpc_user, rpc_pass, rpc_url, str(rpc_port))
self.next_manage_call = None
self.wallet_balance = Decimal(0.0)
self.total_reserved_points = Decimal(0.0)
self.peer_addresses = {} # {Peer: string}
self.queued_payments = defaultdict(Decimal) # {address(string): amount(Decimal)}
self.expected_balances = defaultdict(Decimal) # {address(string): amount(Decimal)}
self.current_address_given_to_peer = {} # {Peer: address(string)}
self.expected_balance_at_time = deque() # (Peer, address(string), amount(Decimal), time(datetime), count(int),
# incremental_amount(float))
self.max_expected_payment_time = datetime.timedelta(minutes=3)
self.stopped = True
self.start_lbrycrdd = start_lbrycrdd
self.started_lbrycrdd = False
self.wallet_dir = wallet_dir
self.wallet_conf = wallet_conf
self.lbrycrdd = None
self.manage_running = False
def start(self):
def make_connection():
if self.start_lbrycrdd is True:
self._start_daemon()
logging.info("Trying to connect to %s", self.rpc_conn_string)
self.rpc_conn = AuthServiceProxy(self.rpc_conn_string)
logging.info("Connected!")
def start_manage():
self.stopped = False
self.manage()
return True
d = threads.deferToThread(make_connection)
d.addCallback(lambda _: start_manage())
return d
def stop(self):
self.stopped = True
# If self.next_manage_call is None, then manage is currently running or else
# start has not been called, so set stopped and do nothing else.
if self.next_manage_call is not None:
self.next_manage_call.cancel()
self.next_manage_call = None
d = self.manage()
if self.start_lbrycrdd is True:
d.addBoth(lambda _: self._stop_daemon())
return d
def manage(self):
logging.info("Doing manage")
self.next_manage_call = None
have_set_manage_running = [False]
def check_if_manage_running():
d = defer.Deferred()
def fire_if_not_running():
if self.manage_running is False:
self.manage_running = True
have_set_manage_running[0] = True
d.callback(True)
else:
task.deferLater(reactor, 1, fire_if_not_running)
fire_if_not_running()
return d
d = check_if_manage_running()
d.addCallback(lambda _: self._check_expected_balances())
d.addCallback(lambda _: self._send_payments())
d.addCallback(lambda _: threads.deferToThread(self._get_wallet_balance))
def set_wallet_balance(balance):
self.wallet_balance = balance
d.addCallback(set_wallet_balance)
def set_next_manage_call():
if not self.stopped:
self.next_manage_call = reactor.callLater(60, self.manage)
d.addCallback(lambda _: set_next_manage_call())
def log_error(err):
logging.error("Something went wrong during manage. Error message: %s", err.getErrorMessage())
return err
d.addErrback(log_error)
def set_manage_not_running(arg):
if have_set_manage_running[0] is True:
self.manage_running = False
return arg
d.addBoth(set_manage_not_running)
return d
def get_info_exchanger(self):
return LBRYcrdAddressRequester(self)
def get_wallet_info_query_handler_factory(self):
return LBRYcrdAddressQueryHandlerFactory(self)
def get_balance(self):
d = threads.deferToThread(self._get_wallet_balance)
return d
def reserve_points(self, peer, amount):
"""
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
@param peer: The peer to which the payment will ultimately be sent
@param amount: The amount of points to reserve
@return: A ReservedPoints object which is given to send_points once the service has been rendered
"""
rounded_amount = Decimal(str(round(amount, 8)))
#if peer in self.peer_addresses:
if self.wallet_balance >= self.total_reserved_points + rounded_amount:
self.total_reserved_points += rounded_amount
return ReservedPoints(peer, rounded_amount)
return None
def cancel_point_reservation(self, reserved_points):
"""
Return all of the points that were reserved previously for some ReservedPoints object
@param reserved_points: ReservedPoints previously returned by reserve_points
@return: None
"""
self.total_reserved_points -= reserved_points.amount
def send_points(self, reserved_points, amount):
"""
Schedule a payment to be sent to a peer
@param reserved_points: ReservedPoints object previously returned by reserve_points
@param amount: amount of points to actually send, must be less than or equal to the
amount reserved in reserved_points
@return: Deferred which fires when the payment has been scheduled
"""
rounded_amount = Decimal(str(round(amount, 8)))
peer = reserved_points.identifier
assert(rounded_amount <= reserved_points.amount)
assert(peer in self.peer_addresses)
self.queued_payments[self.peer_addresses[peer]] += rounded_amount
# make any unused points available
self.total_reserved_points -= (reserved_points.amount - rounded_amount)
logging.info("ordering that %s points be sent to %s", str(rounded_amount),
str(self.peer_addresses[peer]))
peer.update_stats('points_sent', amount)
return defer.succeed(True)
def add_expected_payment(self, peer, amount):
"""Increase the number of points expected to be paid by a peer"""
rounded_amount = Decimal(str(round(amount, 8)))
assert(peer in self.current_address_given_to_peer)
address = self.current_address_given_to_peer[peer]
logging.info("expecting a payment at address %s in the amount of %s", str(address), str(rounded_amount))
self.expected_balances[address] += rounded_amount
expected_balance = self.expected_balances[address]
expected_time = datetime.datetime.now() + self.max_expected_payment_time
self.expected_balance_at_time.append((peer, address, expected_balance, expected_time, 0, amount))
peer.update_stats('expected_points', amount)
def update_peer_address(self, peer, address):
self.peer_addresses[peer] = address
def get_new_address_for_peer(self, peer):
def set_address_for_peer(address):
self.current_address_given_to_peer[peer] = address
return address
d = threads.deferToThread(self._get_new_address)
d.addCallback(set_address_for_peer)
return d
def get_stream_info_for_name(self, name):
def get_stream_info_from_value(result):
r_dict = {}
if 'value' in result:
value = result['value']
try:
value_dict = json.loads(value)
except ValueError:
return Failure(InvalidStreamInfoError(name))
if 'stream_hash' in value_dict:
r_dict['stream_hash'] = value_dict['stream_hash']
if 'name' in value_dict:
r_dict['name'] = value_dict['name']
if 'description' in value_dict:
r_dict['description'] = value_dict['description']
return r_dict
return Failure(UnknownNameError(name))
d = threads.deferToThread(self._get_value_for_name, name)
d.addCallback(get_stream_info_from_value)
return d
def claim_name(self, stream_hash, name, amount):
value = json.dumps({"stream_hash": stream_hash})
d = threads.deferToThread(self._claim_name, name, value, amount)
return d
def get_available_balance(self):
return float(self.wallet_balance - self.total_reserved_points)
def get_new_address(self):
return threads.deferToThread(self._get_new_address)
def _start_daemon(self):
if os.name == "nt":
si = subprocess.STARTUPINFO
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = subprocess.SW_HIDE
self.lbrycrdd = subprocess.Popen(["lbrycrdd.exe", "-datadir=%s" % self.wallet_dir,
"-conf=%s" % self.wallet_conf], startupinfo=si)
else:
self.lbrycrdd = subprocess.Popen(["./lbrycrdd", "-datadir=%s" % self.wallet_dir,
"-conf=%s" % self.wallet_conf])
self.started_lbrycrdd = True
tries = 0
while tries < 5:
try:
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
rpc_conn.getinfo()
break
except (socket.error, JSONRPCException):
tries += 1
logging.warning("Failed to connect to lbrycrdd.")
if tries < 5:
time.sleep(2 ** tries)
logging.warning("Trying again in %d seconds", 2 ** tries)
else:
logging.warning("Giving up.")
else:
self.lbrycrdd.terminate()
raise ValueError("Couldn't open lbrycrdd")
def _stop_daemon(self):
if self.lbrycrdd is not None and self.started_lbrycrdd is True:
d = threads.deferToThread(self._rpc_stop)
return d
return defer.succeed(True)
def _check_expected_balances(self):
now = datetime.datetime.now()
balances_to_check = []
try:
while self.expected_balance_at_time[0][3] < now:
balances_to_check.append(self.expected_balance_at_time.popleft())
except IndexError:
pass
ds = []
for balance_to_check in balances_to_check:
d = threads.deferToThread(self._check_expected_balance, balance_to_check)
ds.append(d)
dl = defer.DeferredList(ds)
def handle_checks(results):
from future_builtins import zip
for balance, (success, result) in zip(balances_to_check, results):
peer = balance[0]
if success is True:
if result is False:
if balance[4] <= 1: # first or second strike, give them another chance
new_expected_balance = (balance[0],
balance[1],
balance[2],
datetime.datetime.now() + self.max_expected_payment_time,
balance[4] + 1,
balance[5])
self.expected_balance_at_time.append(new_expected_balance)
peer.update_score(-5.0)
else:
peer.update_score(-50.0)
else:
if balance[4] == 0:
peer.update_score(balance[5])
peer.update_stats('points_received', balance[5])
else:
logging.warning("Something went wrong checking a balance. Peer: %s, account: %s,"
"expected balance: %s, expected time: %s, count: %s, error: %s",
str(balance[0]), str(balance[1]), str(balance[2]), str(balance[3]),
str(balance[4]), str(result.getErrorMessage()))
dl.addCallback(handle_checks)
return dl
def _check_expected_balance(self, expected_balance):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
logging.info("Checking balance of address %s", str(expected_balance[1]))
balance = rpc_conn.getreceivedbyaddress(expected_balance[1])
logging.debug("received balance: %s", str(balance))
logging.debug("expected balance: %s", str(expected_balance[2]))
return balance >= expected_balance[2]
def _send_payments(self):
logging.info("Trying to send payments, if there are any to be sent")
def do_send(payments):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
rpc_conn.sendmany("", payments)
payments_to_send = {}
for address, points in self.queued_payments.items():
logging.info("Should be sending %s points to %s", str(points), str(address))
payments_to_send[address] = float(points)
self.total_reserved_points -= points
self.wallet_balance -= points
del self.queued_payments[address]
if payments_to_send:
logging.info("Creating a transaction with outputs %s", str(payments_to_send))
return threads.deferToThread(do_send, payments_to_send)
logging.info("There were no payments to send")
return defer.succeed(True)
def _get_wallet_balance(self):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
return rpc_conn.getbalance("")
def _get_new_address(self):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
return rpc_conn.getnewaddress()
def _get_value_for_name(self, name):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
return rpc_conn.getvalueforname(name)
def _claim_name(self, name, value, amount):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
return str(rpc_conn.claimname(name, value, amount))
def _rpc_stop(self):
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
rpc_conn.stop()
self.lbrycrdd.wait()
class LBRYcrdAddressRequester(object):
implements([IRequestCreator])
def __init__(self, wallet):
self.wallet = wallet
self._protocols = []
######### IRequestCreator #########
def send_next_request(self, peer, protocol):
if not protocol in self._protocols:
r = ClientRequest({'lbrycrd_address': True}, 'lbrycrd_address')
d = protocol.add_request(r)
d.addCallback(self._handle_address_response, peer, r, protocol)
d.addErrback(self._request_failed, peer)
self._protocols.append(protocol)
return defer.succeed(True)
else:
return defer.succeed(False)
######### internal calls #########
def _handle_address_response(self, response_dict, peer, request, protocol):
assert request.response_identifier in response_dict, \
"Expected %s in dict but did not get it" % request.response_identifier
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
address = response_dict[request.response_identifier]
self.wallet.update_peer_address(peer, address)
def _request_failed(self, err, peer):
if not err.check(RequestCanceledError):
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
err.getErrorMessage(), str(peer))
#return err
class LBRYcrdAddressQueryHandlerFactory(object):
implements(IQueryHandlerFactory)
def __init__(self, wallet):
self.wallet = wallet
######### IQueryHandlerFactory #########
def build_query_handler(self):
q_h = LBRYcrdAddressQueryHandler(self.wallet)
return q_h
def get_primary_query_identifier(self):
return 'lbrycrd_address'
def get_description(self):
return "LBRYcrd Address - an address for receiving payments via LBRYcrd"
class LBRYcrdAddressQueryHandler(object):
implements(IQueryHandler)
def __init__(self, wallet):
self.wallet = wallet
self.query_identifiers = ['lbrycrd_address']
self.address = None
self.peer = None
######### IQueryHandler #########
def register_with_request_handler(self, request_handler, peer):
self.peer = peer
request_handler.register_query_handler(self, self.query_identifiers)
def handle_queries(self, queries):
def create_response(address):
self.address = address
fields = {'lbrycrd_address': address}
return fields
if self.query_identifiers[0] in queries:
d = self.wallet.get_new_address_for_peer(self.peer)
d.addCallback(create_response)
return d
if self.address is None:
logging.warning("Expected a request for an address, but did not receive one")
return defer.fail(Failure(ValueError("Expected but did not receive an address request")))
else:
return defer.succeed({})
+30 -22
View File
@@ -1,7 +1,7 @@
from collections import defaultdict
import logging
import leveldb
import os
import unqlite
import time
from Crypto.Hash import SHA512
from Crypto.PublicKey import RSA
@@ -12,7 +12,10 @@ from lbrynet.pointtraderclient import pointtraderclient
from twisted.internet import defer, threads
from zope.interface import implements
from twisted.python.failure import Failure
from lbrynet.core.LBRYcrdWallet import ReservedPoints
from lbrynet.core.LBRYWallet import ReservedPoints
log = logging.getLogger(__name__)
class PTCWallet(object):
@@ -71,7 +74,7 @@ class PTCWallet(object):
def save_key(success, private_key):
if success is True:
threads.deferToThread(self.save_private_key, private_key.exportKey())
self._save_private_key(private_key.exportKey())
return True
return False
@@ -95,8 +98,8 @@ class PTCWallet(object):
def start_manage():
self.manage()
return True
d = threads.deferToThread(self._open_db)
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
d = self._open_db()
d.addCallback(lambda _: self._get_wallet_private_key())
d.addCallback(ensure_private_key_exists)
d.addCallback(lambda _: start_manage())
return d
@@ -160,8 +163,8 @@ class PTCWallet(object):
ds.append(d)
del self.queued_payments[peer]
else:
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
str(peer), str(points))
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
str(peer), str(points))
return defer.DeferredList(ds)
def get_balance(self):
@@ -207,20 +210,25 @@ class PTCWallet(object):
if self.peer_pub_keys[peer] in self.received_payments:
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
if min_expected_balance > received_balance:
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
log.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
def _open_db(self):
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
def open_db():
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
return threads.deferToThread(open_db)
def save_private_key(self, private_key):
self.db.Put("private_key", private_key)
def _save_private_key(self, private_key):
def save_key():
self.db['private_key'] = private_key
return threads.deferToThread(save_key)
def get_wallet_private_key(self):
try:
return self.db.Get("private_key")
except KeyError:
def _get_wallet_private_key(self):
def get_key():
if 'private_key' in self.db:
return self.db['private_key']
return None
return threads.deferToThread(get_key)
class PointTraderKeyExchanger(object):
@@ -256,9 +264,9 @@ class PointTraderKeyExchanger(object):
def _request_failed(self, err, peer):
if not err.check(RequestCanceledError):
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
err.getErrorMessage(), str(peer))
#return err
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
err.getErrorMessage(), str(peer))
return err
class PointTraderKeyQueryHandlerFactory(object):
@@ -301,15 +309,15 @@ class PointTraderKeyQueryHandler(object):
try:
RSA.importKey(new_encoded_pub_key)
except (ValueError, TypeError, IndexError):
logging.warning("Client sent an invalid public key.")
log.warning("Client sent an invalid public key.")
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
self.public_key = new_encoded_pub_key
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
logging.debug("Received the client's public key: %s", str(self.public_key))
log.debug("Received the client's public key: %s", str(self.public_key))
fields = {'public_key': self.wallet.encoded_public_key}
return defer.succeed(fields)
if self.public_key is None:
logging.warning("Expected a public key, but did not receive one")
log.warning("Expected a public key, but did not receive one")
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
else:
return defer.succeed({})
+32 -91
View File
@@ -1,5 +1,6 @@
from zope.interface import implements
from lbrynet.interfaces import IRateLimiter
from twisted.internet import task
class DummyRateLimiter(object):
@@ -10,22 +11,20 @@ class DummyRateLimiter(object):
self.total_ul_bytes = 0
self.target_dl = 0
self.target_ul = 0
self.ul_delay = 0.00
self.dl_delay = 0.00
self.next_tick = None
self.tick_call = None
def start(self):
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(1)
def tick(self):
from twisted.internet import reactor
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
self.next_tick = reactor.callLater(1.0, self.tick)
def stop(self):
if self.next_tick is not None:
self.next_tick.cancel()
self.next_tick = None
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
pass
@@ -33,12 +32,6 @@ class DummyRateLimiter(object):
def set_ul_limit(self, limit):
pass
def ul_wait_time(self):
return self.ul_delay
def dl_wait_time(self):
return self.dl_delay
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_second += num_bytes
self.total_dl_bytes += num_bytes
@@ -58,66 +51,32 @@ class RateLimiter(object):
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
self.max_dl_bytes = max_dl_bytes
self.max_ul_bytes = max_ul_bytes
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.total_dl_bytes = 0
self.total_ul_bytes = 0
self.next_tick = None
self.next_unthrottle_dl = None
self.next_unthrottle_ul = None
self.next_dl_check = None
self.next_ul_check = None
self.dl_check_interval = 1.0
self.ul_check_interval = 1.0
self.tick_call = None
self.tick_interval = 0.1
self.dl_throttled = False
self.ul_throttled = False
self.protocols = []
def start(self):
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(self.tick_interval)
def tick(self):
from twisted.internet import reactor
# happens once per second
if self.next_dl_check is not None:
self.next_dl_check.cancel()
self.next_dl_check = None
if self.next_ul_check is not None:
self.next_ul_check.cancel()
self.next_ul_check = None
if self.max_dl_bytes is not None:
if self.dl_bytes_this_second == 0:
self.dl_check_interval = 1.0
else:
self.dl_check_interval = min(1.0, self.dl_check_interval *
self.max_dl_bytes / self.dl_bytes_this_second)
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
if self.max_ul_bytes is not None:
if self.ul_bytes_this_second == 0:
self.ul_check_interval = 1.0
else:
self.ul_check_interval = min(1.0, self.ul_check_interval *
self.max_ul_bytes / self.ul_bytes_this_second)
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.unthrottle_dl()
self.unthrottle_ul()
self.next_tick = reactor.callLater(1.0, self.tick)
def stop(self):
if self.next_tick is not None:
self.next_tick.cancel()
self.next_tick = None
if self.next_dl_check is not None:
self.next_dl_check.cancel()
self.next_dl_check = None
if self.next_ul_check is not None:
self.next_ul_check.cancel()
self.next_ul_check = None
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
self.max_dl_bytes = limit
@@ -129,27 +88,15 @@ class RateLimiter(object):
def check_dl(self):
from twisted.internet import reactor
self.next_dl_check = None
if self.dl_bytes_this_second > self.max_dl_bytes:
self.throttle_dl()
else:
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
if self.max_dl_bytes is not None and self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_dl)
def check_ul(self):
from twisted.internet import reactor
self.next_ul_check = None
if self.ul_bytes_this_second > self.max_ul_bytes:
self.throttle_ul()
else:
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
if self.max_ul_bytes is not None and self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_ul)
def throttle_dl(self):
if self.dl_throttled is False:
@@ -175,23 +122,17 @@ class RateLimiter(object):
protocol.unthrottle_upload()
self.ul_throttled = False
#deprecated
def ul_wait_time(self):
return 0
def dl_wait_time(self):
return 0
#called by protocols
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_second += num_bytes
self.dl_bytes_this_interval += num_bytes
self.total_dl_bytes += num_bytes
self.check_dl()
def report_ul_bytes(self, num_bytes):
self.ul_bytes_this_second += num_bytes
self.ul_bytes_this_interval += num_bytes
self.total_ul_bytes += num_bytes
self.check_ul()
def register_protocol(self, protocol):
if protocol not in self.protocols:
+71 -32
View File
@@ -13,6 +13,9 @@ from lbrynet.core.PaymentRateManager import BasePaymentRateManager
from twisted.internet import threads, defer
log = logging.getLogger(__name__)
class LBRYSession(object):
"""This class manages all important services common to any application that uses the network:
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
@@ -25,7 +28,7 @@ class LBRYSession(object):
def __init__(self, blob_data_payment_rate, db_dir=None, lbryid=None, peer_manager=None, dht_node_port=None,
known_dht_nodes=None, peer_finder=None, hash_announcer=None,
blob_dir=None, blob_manager=None, peer_port=None, use_upnp=True,
rate_limiter=None, wallet=None):
rate_limiter=None, wallet=None, dht_node_class=node.Node):
"""
@param blob_data_payment_rate: The default payment rate for blob data
@@ -92,11 +95,11 @@ class LBRYSession(object):
self.rate_limiter = rate_limiter
self.external_ip = '127.0.0.1'
self.upnp_handler = None
self.upnp_redirects_set = False
self.upnp_redirects = []
self.wallet = wallet
self.dht_node_class = dht_node_class
self.dht_node = None
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
@@ -104,7 +107,7 @@ class LBRYSession(object):
def setup(self):
"""Create the blob directory and database if necessary, start all desired services"""
logging.debug("Setting up the lbry session")
log.debug("Setting up the lbry session")
if self.lbryid is None:
self.lbryid = generate_id()
@@ -124,7 +127,7 @@ class LBRYSession(object):
d.addCallback(lambda _: self._setup_dht())
else:
if self.hash_announcer is None and self.peer_port is not None:
logging.warning("The server has no way to advertise its available blobs.")
log.warning("The server has no way to advertise its available blobs.")
self.hash_announcer = DummyHashAnnouncer()
d.addCallback(lambda _: self._setup_other_components())
@@ -135,41 +138,63 @@ class LBRYSession(object):
ds = []
if self.dht_node is not None:
ds.append(defer.maybeDeferred(self.dht_node.stop))
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
ds.append(defer.maybeDeferred(self.peer_finder.stop))
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
ds.append(defer.maybeDeferred(self.wallet.stop))
ds.append(defer.maybeDeferred(self.blob_manager.stop))
if self.upnp_redirects_set is True:
if self.rate_limiter is not None:
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
if self.peer_finder is not None:
ds.append(defer.maybeDeferred(self.peer_finder.stop))
if self.hash_announcer is not None:
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
if self.wallet is not None:
ds.append(defer.maybeDeferred(self.wallet.stop))
if self.blob_manager is not None:
ds.append(defer.maybeDeferred(self.blob_manager.stop))
if self.use_upnp is True:
ds.append(defer.maybeDeferred(self._unset_upnp))
return defer.DeferredList(ds)
def _try_upnp(self):
logging.debug("In _try_upnp")
log.debug("In _try_upnp")
def threaded_try_upnp():
if self.use_upnp is False:
logging.debug("Not using upnp")
log.debug("Not using upnp")
return False
u = miniupnpc.UPnP()
num_devices_found = u.discover()
if num_devices_found > 0:
self.upnp_handler = u
u.selectigd()
external_ip = u.externalipaddress()
if external_ip != '0.0.0.0':
self.external_ip = external_ip
if self.peer_port is not None:
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
self.upnp_redirects.append((self.peer_port, 'TCP'))
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
else:
# see comment below
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
self.upnp_redirects.append((self.peer_port, 'TCP'))
if self.dht_node_port is not None:
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
self.upnp_redirects_set = True
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
log.info("Set UPnP redirect for UPD port %d", self.dht_node_port)
else:
# TODO: check that the existing redirect was put up by an old lbrynet session before grabbing it
# if such a disconnected redirect exists, then upnp won't work unless the redirect is appended
# or is torn down and set back up. a bad shutdown of lbrynet could leave such a redirect up
# and cause problems on the next start.
# this could be problematic if a previous lbrynet session didn't make the redirect, and it was
# made by another application
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
return True
return False
def upnp_failed(err):
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
return False
d = threads.deferToThread(threaded_try_upnp)
@@ -180,7 +205,7 @@ class LBRYSession(object):
from twisted.internet import reactor
logging.debug("Starting the dht")
log.debug("Starting the dht")
def match_port(h, p):
return h, p
@@ -204,8 +229,11 @@ class LBRYSession(object):
d.addCallback(match_port, port)
ds.append(d)
self.dht_node = node.Node(udpPort=self.dht_node_port, lbryid=self.lbryid,
externalIP=self.external_ip)
self.dht_node = self.dht_node_class(
udpPort=self.dht_node_port,
lbryid=self.lbryid,
externalIP=self.external_ip
)
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
if self.hash_announcer is None:
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
@@ -216,7 +244,7 @@ class LBRYSession(object):
return dl
def _setup_other_components(self):
logging.debug("Setting up the rest of the components")
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
@@ -227,19 +255,30 @@ class LBRYSession(object):
else:
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
self.rate_limiter.tick()
self.rate_limiter.start()
d1 = self.blob_manager.setup()
d2 = self.wallet.start()
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
dl.addErrback(lambda err: err.value.subFailure)
return dl
def _unset_upnp(self):
def threaded_unset_upnp():
u = self.upnp_handler
if self.peer_port is not None:
u.deleteportmapping(self.peer_port, 'TCP')
if self.dht_node_port is not None:
u.deleteportmapping(self.dht_node_port, 'UDP')
self.upnp_redirects_set = False
u = miniupnpc.UPnP()
num_devices_found = u.discover()
if num_devices_found > 0:
u.selectigd()
for port, protocol in self.upnp_redirects:
if u.getspecificportmapping(port, protocol) is None:
log.warning("UPnP redirect for %s %d was removed by something else.", protocol, port)
else:
u.deleteportmapping(port, protocol)
log.info("Removed UPnP redirect for %s %d.", protocol, port)
self.upnp_redirects = []
return threads.deferToThread(threaded_unset_upnp)
d = threads.deferToThread(threaded_unset_upnp)
d.addErrback(lambda err: str(err))
return d
+4 -1
View File
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
from zope.interface import implements
log = logging.getLogger(__name__)
class StreamCreator(object):
"""Classes which derive from this class create a 'stream', which can be any
collection of associated blobs and associated metadata. These classes
@@ -42,7 +45,7 @@ class StreamCreator(object):
def stop(self):
"""Stop creating the stream. Create the terminating zero-length blob."""
logging.debug("stop has been called for StreamCreator")
log.debug("stop has been called for StreamCreator")
self.stopped = True
if self.current_blob is not None:
current_blob = self.current_blob
+61 -15
View File
@@ -3,6 +3,10 @@ import json
import logging
from twisted.internet import threads
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
log = logging.getLogger(__name__)
class StreamDescriptorReader(object):
@@ -81,7 +85,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
def _write_stream_descriptor(self, raw_data):
def write_file():
logging.debug("Writing the sd file to disk")
log.debug("Writing the sd file to disk")
with open(self.sd_file_name, 'w') as sd_file:
sd_file.write(raw_data)
return self.sd_file_name
@@ -96,13 +100,26 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
self.blob_manager = blob_manager
def _write_stream_descriptor(self, raw_data):
logging.debug("Creating the new blob for the stream descriptor")
log.debug("Creating the new blob for the stream descriptor")
blob_creator = self.blob_manager.get_blob_creator()
blob_creator.write(raw_data)
logging.debug("Wrote the data to the new blob")
log.debug("Wrote the data to the new blob")
return blob_creator.close()
class StreamMetadata(object):
FROM_BLOB = 1
FROM_PLAIN = 2
def __init__(self, validator, options, factories):
self.validator = validator
self.options = options
self.factories = factories
self.metadata_source = None
self.source_blob_hash = None
self.source_file = None
class StreamDescriptorIdentifier(object):
"""Tries to determine the type of stream described by the stream descriptor using the
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
@@ -110,9 +127,10 @@ class StreamDescriptorIdentifier(object):
"""
def __init__(self):
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
self._stream_options = {} # {stream_type: IStreamOptions}
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
def add_stream_info_validator(self, stream_type, sd_info_validator):
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
"""
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
@@ -126,9 +144,13 @@ class StreamDescriptorIdentifier(object):
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
descriptor from being further processed.
@param stream_options: A class implementing the IStreamOptions interface. This class's constructor will be
passed the sd_info_validator object containing the raw metadata from the stream descriptor file.
@return: None
"""
self._sd_info_validators[stream_type] = sd_info_validator
self._stream_options[stream_type] = stream_options
def add_stream_downloader_factory(self, stream_type, factory):
"""
@@ -146,35 +168,59 @@ class StreamDescriptorIdentifier(object):
"""
self._stream_downloader_factories[stream_type].append(factory)
def get_info_and_factories_for_sd_file(self, sd_path):
def _return_metadata(self, options_validator_factories, source_type, source):
validator, options, factories = options_validator_factories
m = StreamMetadata(validator, options, factories)
m.metadata_source = source_type
if source_type == StreamMetadata.FROM_BLOB:
m.source_blob_hash = source
if source_type == StreamMetadata.FROM_PLAIN:
m.source_file = source
return m
def get_metadata_for_sd_file(self, sd_path):
sd_reader = PlainStreamDescriptorReader(sd_path)
d = sd_reader.get_info()
d.addCallback(self._return_info_and_factories)
d.addCallback(self._return_options_and_validator_and_factories)
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
return d
def get_info_and_factories_for_sd_blob(self, sd_blob):
def get_metadata_for_sd_blob(self, sd_blob):
sd_reader = BlobStreamDescriptorReader(sd_blob)
d = sd_reader.get_info()
d.addCallback(self._return_info_and_factories)
d.addCallback(self._return_options_and_validator_and_factories)
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
return d
def _get_factories(self, stream_type):
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
if not stream_type in self._stream_downloader_factories:
raise UnknownStreamTypeError(stream_type)
return self._stream_downloader_factories[stream_type]
def _get_validator(self, stream_type):
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
if not stream_type in self._sd_info_validators:
raise UnknownStreamTypeError(stream_type)
return self._sd_info_validators[stream_type]
def _return_info_and_factories(self, sd_info):
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
def _get_options(self, stream_type):
if not stream_type in self._stream_downloader_factories:
raise UnknownStreamTypeError(stream_type)
return self._stream_options[stream_type]
def _return_options_and_validator_and_factories(self, sd_info):
if not 'stream_type' in sd_info:
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
stream_type = sd_info['stream_type']
factories = self._get_factories(stream_type)
validator = self._get_validator(stream_type)(sd_info)
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
d = validator.validate()
d.addCallback(lambda _: (validator, factories))
def get_options():
options = self._get_options(stream_type)
return validator, options, factories
d.addCallback(lambda _: get_options())
return d
@@ -192,4 +238,4 @@ def download_sd_blob(session, blob_hash, payment_rate_manager):
"""
downloader = StandaloneBlobDownloader(blob_hash, session.blob_manager, session.peer_finder,
session.rate_limiter, payment_rate_manager, session.wallet)
return downloader.download()
return downloader.download()
+23 -11
View File
@@ -5,10 +5,14 @@ from twisted.python.failure import Failure
from zope.interface import implements
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
from lbrynet.interfaces import IRequestCreator
log = logging.getLogger(__name__)
class BlobRequester(object):
implements(IRequestCreator)
@@ -68,6 +72,7 @@ class BlobRequester(object):
sent_request = True
else:
d_r.cancel(InsufficientFundsError())
d_r.finished_deferred.addErrback(lambda _: True)
return defer.fail(InsufficientFundsError())
if sent_request is True:
if p_r is not None:
@@ -84,7 +89,7 @@ class BlobRequester(object):
######### internal calls #########
def _download_succeeded(self, arg, peer, blob):
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
log.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
self._update_local_score(peer, 5.0)
peer.update_stats('blobs_downloaded', 1)
peer.update_score(5.0)
@@ -105,8 +110,8 @@ class BlobRequester(object):
def _handle_download_error(self, err, peer, blob_to_download):
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
logging.warning("An error occurred while downloading %s from %s. Error: %s",
blob_to_download.blob_hash, str(peer), err.getTraceback())
log.warning("An error occurred while downloading %s from %s. Error: %s",
blob_to_download.blob_hash, str(peer), err.getTraceback())
if err.check(PriceDisagreementError):
# Don't kill the whole connection just because a price couldn't be agreed upon.
# Other information might be desired by other request creators at a better rate.
@@ -123,7 +128,7 @@ class BlobRequester(object):
else:
blob_hash = blobs_without_sources[0].blob_hash
r = blob_hash
logging.debug("Blob requester peer search response: %s", str(r))
log.debug("Blob requester peer search response: %s", str(r))
return defer.succeed(r)
def _find_peers_for_hash(self, h):
@@ -137,6 +142,12 @@ class BlobRequester(object):
return [p for p in peers if not p in bad_peers]
d.addCallback(choose_best_peers)
def lookup_failed(err):
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
return []
d.addErrback(lookup_failed)
return d
def _should_send_request_to(self, peer):
@@ -201,7 +212,7 @@ class BlobRequester(object):
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
cancel_func, blob_to_download)
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
log.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
return request
def _price_settled(self, protocol):
@@ -238,11 +249,11 @@ class BlobRequester(object):
def _handle_availability(self, response_dict, peer, request):
if not request.response_identifier in response_dict:
raise InvalidResponseError("response identifier not in response")
logging.debug("Received a response to the availability request")
log.debug("Received a response to the availability request")
blob_hashes = response_dict[request.response_identifier]
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
self._available_blobs[peer].append(blob_hash)
if blob_hash in self._unavailable_blobs[peer]:
self._unavailable_blobs[peer].remove(blob_hash)
@@ -296,12 +307,13 @@ class BlobRequester(object):
return
if reason.check(NoResponseError):
self._incompatible_peers.append(peer)
return
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
str(request_type), reason.getErrorMessage(), reason.type)
log.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
str(request_type), reason.getErrorMessage(), reason.type)
self._update_local_score(peer, -10.0)
if isinstance(reason, InvalidResponseError):
if isinstance(reason, InvalidResponseError) or isinstance(reason, NoResponseError):
peer.update_score(-10.0)
else:
peer.update_score(-2.0)
if reason.check(ConnectionClosedBeforeResponseError):
return
return reason
+38 -17
View File
@@ -1,6 +1,6 @@
import json
import logging
from twisted.internet import error, defer, reactor
from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.python import failure
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
@@ -11,6 +11,9 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
from zope.interface import implements
log = logging.getLogger(__name__)
class ClientProtocol(Protocol):
implements(IRequestSender, IRateLimited)
@@ -39,7 +42,7 @@ class ClientProtocol(Protocol):
else:
self._response_buff += data
if len(self._response_buff) > MAX_RESPONSE_SIZE:
logging.warning("Response is too large. Size %s", len(self._response_buff))
log.warning("Response is too large. Size %s", len(self._response_buff))
self.transport.loseConnection()
response, extra_data = self._get_valid_response(self._response_buff)
if response is not None:
@@ -55,7 +58,7 @@ class ClientProtocol(Protocol):
else:
err = reason
#if self._response_deferreds:
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
# log.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
for key, d in self._response_deferreds.items():
del self._response_deferreds[key]
d.errback(err)
@@ -70,7 +73,7 @@ class ClientProtocol(Protocol):
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
self._next_request.update(request.request_dict)
d = defer.Deferred()
logging.debug("Adding a request. Request: %s", str(request))
log.debug("Adding a request. Request: %s", str(request.request_dict))
self._response_deferreds[request.response_identifier] = d
return d
@@ -102,12 +105,14 @@ class ClientProtocol(Protocol):
######### Internal request handling #########
def _handle_request_error(self, err):
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
str(self.peer), err.getTraceback())
log.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
str(self.peer), err.getTraceback())
self.transport.loseConnection()
def _ask_for_request(self):
log.debug("In _ask_for_request")
if self.connection_closed is True or self.connection_closing is True:
return
@@ -117,7 +122,7 @@ class ClientProtocol(Protocol):
self._send_request_message(request_msg)
else:
# The connection manager has indicated that this connection should be terminated
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
log.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
self.transport.loseConnection()
d = self._connection_manager.get_next_request(self.peer, self)
@@ -153,14 +158,15 @@ class ClientProtocol(Protocol):
# If an error gets to this point, log it and kill the connection.
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
RequestCanceledError):
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
err.getErrorMessage())
if not err.check(RequestCanceledError):
self.transport.loseConnection()
log.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
err.getErrorMessage())
if not err.check(RequestCanceledError): # The connection manager is closing the connection, so
# there's no need to do it here.
return err
def _handle_response(self, response):
ds = []
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
log.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
for key, val in response.items():
if key in self._response_deferreds:
d = self._response_deferreds[key]
@@ -178,12 +184,27 @@ class ClientProtocol(Protocol):
d.addErrback(self._handle_response_error)
ds.append(d)
dl = defer.DeferredList(ds)
dl = defer.DeferredList(ds, consumeErrors=True)
dl.addCallback(lambda _: self._ask_for_request())
def get_next_request(results):
failed = False
for success, result in results:
if success is False:
failed = True
log.info("The connection is closing due to an error: %s", str(result.getTraceback()))
if failed is False:
log.debug("Asking for another request.")
from twisted.internet import reactor
reactor.callLater(0, self._ask_for_request)
#self._ask_for_request()
else:
log.debug("Not asking for another request.")
self.transport.loseConnection()
dl.addCallback(get_next_request)
def _downloading_finished(self, arg):
logging.debug("The blob has finished downloading")
log.debug("The blob has finished downloading")
self._blob_download_request = None
self._downloading_blob = False
return arg
@@ -194,8 +215,8 @@ class ClientProtocol(Protocol):
# TODO: always be this way. it's done this way now because the client has no other way
# TODO: of telling the server it wants the download to stop. It would be great if the
# TODO: protocol had such a mechanism.
logging.info("Closing the connection to %s because the download of blob %s was canceled",
str(self.peer), str(self._blob_download_request.blob))
log.info("Closing the connection to %s because the download of blob %s was canceled",
str(self.peer), str(self._blob_download_request.blob))
#self.transport.loseConnection()
#return True
return err
+69 -55
View File
@@ -7,49 +7,62 @@ from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
from lbrynet.core.Error import InsufficientFundsError
log = logging.getLogger(__name__)
class PeerConnectionHandler(object):
def __init__(self, request_creators, factory):
self.request_creators = request_creators
self.factory = factory
self.connection = None
class ConnectionManager(object):
implements(interfaces.IConnectionManager)
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
self.downloader = downloader
self.rate_limiter = rate_limiter
self.primary_request_creators = primary_request_creators
self.secondary_request_creators = secondary_request_creators
self.peer_connections = {} # {Peer: {'connection': connection,
# 'request_creators': [IRequestCreator if using this connection]}}
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
self.next_manage_call = None
self._primary_request_creators = primary_request_creators
self._secondary_request_creators = secondary_request_creators
self._peer_connections = {} # {Peer: PeerConnectionHandler}
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
self._next_manage_call = None
self.stopped = True
def start(self):
from twisted.internet import reactor
if self.next_manage_call is not None and self.next_manage_call.active() is True:
self.next_manage_call.cancel()
self.next_manage_call = reactor.callLater(0, self._manage)
self.stopped = False
if self._next_manage_call is not None and self._next_manage_call.active() is True:
self._next_manage_call.cancel()
self._next_manage_call = reactor.callLater(0, self._manage)
return defer.succeed(True)
def stop(self):
if self.next_manage_call is not None and self.next_manage_call.active() is True:
self.next_manage_call.cancel()
self.next_manage_call = None
self.stopped = True
if self._next_manage_call is not None and self._next_manage_call.active() is True:
self._next_manage_call.cancel()
self._next_manage_call = None
closing_deferreds = []
for peer in self.peer_connections.keys():
for peer in self._peer_connections.keys():
def close_connection(p):
logging.info("Abruptly closing a connection to %s due to downloading being paused",
str(p))
log.info("Abruptly closing a connection to %s due to downloading being paused",
str(p))
if self.peer_connections[p]['factory'].p is not None:
d = self.peer_connections[p]['factory'].p.cancel_requests()
if self._peer_connections[p].factory.p is not None:
d = self._peer_connections[p].factory.p.cancel_requests()
else:
d = defer.succeed(True)
def disconnect_peer():
self.peer_connections[p]['connection'].disconnect()
if p in self.peer_connections:
del self.peer_connections[p]
d = defer.Deferred()
self.connections_closing[p] = d
self._connections_closing[p] = d
self._peer_connections[p].connection.disconnect()
if p in self._peer_connections:
del self._peer_connections[p]
return d
d.addBoth(lambda _: disconnect_peer())
@@ -60,26 +73,27 @@ class ConnectionManager(object):
def get_next_request(self, peer, protocol):
logging.debug("Trying to get the next request for peer %s", str(peer))
log.debug("Trying to get the next request for peer %s", str(peer))
if not peer in self.peer_connections:
logging.debug("The peer has already been told to shut down.")
if not peer in self._peer_connections or self.stopped is True:
log.debug("The peer has already been told to shut down.")
return defer.succeed(False)
def handle_error(err):
if err.check(InsufficientFundsError):
self.downloader.insufficient_funds()
return False
else:
return err
err.trap(InsufficientFundsError)
self.downloader.insufficient_funds(err)
return False
def check_if_request_sent(request_sent, request_creator):
if peer not in self._peer_connections:
# This can happen if the connection is told to close
return False
if request_sent is False:
if request_creator in self.peer_connections[peer]['request_creators']:
self.peer_connections[peer]['request_creators'].remove(request_creator)
if request_creator in self._peer_connections[peer].request_creators:
self._peer_connections[peer].request_creators.remove(request_creator)
else:
if not request_creator in self.peer_connections[peer]['request_creators']:
self.peer_connections[peer]['request_creators'].append(request_creator)
if not request_creator in self._peer_connections[peer].request_creators:
self._peer_connections[peer].request_creators.append(request_creator)
return request_sent
def check_requests(requests):
@@ -89,7 +103,7 @@ class ConnectionManager(object):
def get_secondary_requests_if_necessary(have_request):
if have_request is True:
ds = []
for s_r_c in self.secondary_request_creators:
for s_r_c in self._secondary_request_creators:
d = s_r_c.send_next_request(peer, protocol)
ds.append(d)
dl = defer.DeferredList(ds)
@@ -100,7 +114,7 @@ class ConnectionManager(object):
ds = []
for p_r_c in self.primary_request_creators:
for p_r_c in self._primary_request_creators:
d = p_r_c.send_next_request(peer, protocol)
d.addErrback(handle_error)
d.addCallback(check_if_request_sent, p_r_c)
@@ -112,11 +126,11 @@ class ConnectionManager(object):
return dl
def protocol_disconnected(self, peer, protocol):
if peer in self.peer_connections:
del self.peer_connections[peer]
if peer in self.connections_closing:
d = self.connections_closing[peer]
del self.connections_closing[peer]
if peer in self._peer_connections:
del self._peer_connections[peer]
if peer in self._connections_closing:
d = self._connections_closing[peer]
del self._connections_closing[peer]
d.callback(True)
def _rank_request_creator_connections(self):
@@ -125,30 +139,30 @@ class ConnectionManager(object):
connections open that it likes
"""
def count_peers(request_creator):
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
return len([p for p in self._peer_connections.itervalues() if request_creator in p.request_creators])
return sorted(self.primary_request_creators, key=count_peers)
return sorted(self._primary_request_creators, key=count_peers)
def _connect_to_peer(self, peer):
from twisted.internet import reactor
if peer is not None:
logging.debug("Trying to connect to %s", str(peer))
if peer is not None and self.stopped is False:
log.debug("Trying to connect to %s", str(peer))
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
factory)
connection = reactor.connectTCP(peer.host, peer.port, factory)
self.peer_connections[peer] = {'connection': connection,
'request_creators': self.primary_request_creators[:],
'factory': factory}
self._peer_connections[peer].connection = connection
def _manage(self):
from twisted.internet import reactor
def get_new_peers(request_creators):
logging.debug("Trying to get a new peer to connect to")
log.debug("Trying to get a new peer to connect to")
if len(request_creators) > 0:
logging.debug("Got a creator to check: %s", str(request_creators[0]))
log.debug("Got a creator to check: %s", str(request_creators[0]))
d = request_creators[0].get_new_peers()
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
return d
@@ -158,20 +172,20 @@ class ConnectionManager(object):
def pick_best_peer(peers):
# TODO: Eventually rank them based on past performance/reputation. For now
# TODO: just pick the first to which we don't have an open connection
logging.debug("Got a list of peers to choose from: %s", str(peers))
log.debug("Got a list of peers to choose from: %s", str(["%s:%i" % (p.host, p.port) for p in peers]))
if peers is None:
return None
for peer in peers:
if not peer in self.peer_connections:
logging.debug("Got a good peer. Returning peer %s", str(peer))
if not peer in self._peer_connections:
log.debug("Got a good peer. Returning peer %s", str(peer))
return peer
logging.debug("Couldn't find a good peer to connect to")
log.debug("Couldn't find a good peer to connect to")
return None
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
if len(self._peer_connections) < MAX_CONNECTIONS_PER_STREAM:
ordered_request_creators = self._rank_request_creator_connections()
d = get_new_peers(ordered_request_creators)
d.addCallback(pick_best_peer)
d.addCallback(self._connect_to_peer)
self.next_manage_call = reactor.callLater(1, self._manage)
self._next_manage_call = reactor.callLater(1, self._manage)
+2 -1
View File
@@ -32,8 +32,9 @@ class DHTPeerFinder(object):
bin_hash = binascii.unhexlify(blob_hash)
def filter_peers(peer_list):
peers = set(peer_list)
good_peers = []
for host, port in peer_list:
for host, port in peers:
peer = self.peer_manager.get_peer(host, port)
if peer.is_available() is True:
good_peers.append(peer)
+11 -8
View File
@@ -5,6 +5,9 @@ from zope.interface import implements
from lbrynet import interfaces
log = logging.getLogger(__name__)
class DownloadManager(object):
implements(interfaces.IDownloadManager)
@@ -24,7 +27,7 @@ class DownloadManager(object):
def start_downloading(self):
d = self.blob_info_finder.get_initial_blobs()
logging.debug("Requested the initial blobs from the info finder")
log.debug("Requested the initial blobs from the info finder")
d.addCallback(self.add_blobs_to_download)
d.addCallback(lambda _: self.resume_downloading())
return d
@@ -33,7 +36,7 @@ class DownloadManager(object):
def check_start(result, manager):
if isinstance(result, failure.Failure):
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
log.error("Failed to start the %s: %s", manager, result.getErrorMessage())
return False
return True
@@ -49,7 +52,7 @@ class DownloadManager(object):
def check_stop(result, manager):
if isinstance(result, failure.Failure):
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
log.error("Failed to stop the %s: %s", manager, result.getErrorMessage())
return False
return True
@@ -63,21 +66,21 @@ class DownloadManager(object):
def add_blobs_to_download(self, blob_infos):
logging.debug("Adding %s to blobs", str(blob_infos))
log.debug("Adding %s blobs to blobs", len(blob_infos))
def add_blob_to_list(blob, blob_num):
self.blobs[blob_num] = blob
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
log.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
def error_during_add(err):
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
log.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
return err
ds = []
for blob_info in blob_infos:
if not blob_info.blob_num in self.blobs:
self.blob_infos[blob_info.blob_num] = blob_info
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
log.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
d.addCallback(add_blob_to_list, blob_info.blob_num)
d.addErrback(error_during_add)
@@ -112,4 +115,4 @@ class DownloadManager(object):
if not self.blobs:
return self.calculate_total_bytes()
else:
return sum([b.length for b in self.needed_blobs() if b.length is not None])
return sum([b.length for b in self.needed_blobs() if b.length is not None])
@@ -5,9 +5,15 @@ from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.Error import InvalidBlobHashError
from lbrynet.core.utils import is_valid_blobhash
from twisted.python.failure import Failure
from twisted.internet import defer
log = logging.getLogger(__name__)
class SingleBlobMetadataHandler(object):
implements(interfaces.IMetadataHandler)
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
######## IMetadataHandler #########
def get_initial_blobs(self):
logging.debug("Returning the blob info")
log.debug("Returning the blob info")
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
def final_blob_num(self):
@@ -72,7 +78,7 @@ class SingleProgressManager(object):
from twisted.internet import reactor
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
log.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
if self.finished is False:
self.finished = True
reactor.callLater(0, self.finished_callback, blob)
@@ -99,6 +105,9 @@ class StandaloneBlobDownloader(object):
self.finished_deferred = None
def download(self):
if not is_valid_blobhash(self.blob_hash):
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
def cancel_download(d):
self.stop()
@@ -127,7 +136,10 @@ class StandaloneBlobDownloader(object):
def _blob_downloaded(self, blob):
self.stop()
self.finished_deferred.callback(blob)
if not self.finished_deferred.called:
self.finished_deferred.callback(blob)
def insufficient_funds(self):
return self.stop()
def insufficient_funds(self, err):
self.stop()
if not self.finished_deferred.called:
self.finished_deferred.errback(err)
+17 -8
View File
@@ -4,6 +4,9 @@ from twisted.internet import defer
from zope.interface import implements
log = logging.getLogger(__name__)
class StreamProgressManager(object):
implements(IProgressManager)
@@ -61,18 +64,18 @@ class StreamProgressManager(object):
return defer.succeed(None)
def _finished_with_blob(self, blob_num):
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
if self.delete_blob_after_finished is True:
logging.debug("delete_blob_after_finished is True")
log.debug("delete_blob_after_finished is True")
blobs = self.download_manager.blobs
if blob_num in blobs:
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
blobs[blob_num].blob_hash)
log.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
blobs[blob_num].blob_hash)
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
else:
logging.debug("Blob number %s was not in blobs", str(blob_num))
log.debug("Blob number %s was not in blobs", str(blob_num))
else:
logging.debug("delete_blob_after_finished is False")
log.debug("delete_blob_after_finished is False")
class FullStreamProgressManager(StreamProgressManager):
@@ -115,6 +118,8 @@ class FullStreamProgressManager(StreamProgressManager):
def finished_outputting_blob():
self.last_blob_outputted += 1
def check_if_finished():
final_blob_num = self.download_manager.final_blob_num()
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
self._finished_outputting()
@@ -126,14 +131,18 @@ class FullStreamProgressManager(StreamProgressManager):
current_blob_num = self.last_blob_outputted + 1
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
log.info("Outputting blob %s", str(self.last_blob_outputted + 1))
self.provided_blob_nums.append(self.last_blob_outputted + 1)
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
d.addCallback(lambda _: finished_outputting_blob())
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
d.addCallback(lambda _: check_if_finished())
def log_error(err):
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
if self.outputting_d is not None and not self.outputting_d.called:
self.outputting_d.callback(True)
self.outputting_d = None
d.addErrback(log_error)
else:
+127
View File
@@ -0,0 +1,127 @@
import json
import logging
import logging.handlers
import sys
import traceback
from requests_futures.sessions import FuturesSession
import lbrynet
from lbrynet import conf
from lbrynet.core import utils
session = FuturesSession()
def bg_cb(sess, resp):
""" Don't do anything with the response """
pass
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None):
logging.Handler.__init__(self)
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
def emit(self, record):
try:
payload = self.format(record)
session.post(self.url, data=payload, background_callback=bg_cb)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
if not isinstance(level, int):
# despite the name, getLevelName returns
# the numeric level when passed a text level
level = logging.getLevelName(level)
handler = fn(*args, **kwargs)
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
if log.level > level:
log.setLevel(level)
return helper
def disable_third_party_loggers():
logging.getLogger('requests').setLevel(logging.WARNING)
def disable_noisy_loggers():
logging.getLogger('lbrynet.dht').setLevel(logging.INFO)
logging.getLogger('lbrynet.core.client.ConnectionManager').setLevel(logging.INFO)
logging.getLogger('lbrynet.core.client.BlobRequester').setLevel(logging.INFO)
logging.getLogger('lbrynet.core.client.ClientProtocol').setLevel(logging.INFO)
logging.getLogger('lbrynet.analytics.api').setLevel(logging.INFO)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a logger that outputs to stdout"""
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'console'
return handler
@_log_decorator
def configure_file_handler(file_name, **kwargs):
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'file'
return handler
def get_loggly_url(token=None, version=None):
token = token or utils.deobfuscate(conf.LOGGLY_TOKEN)
version = version or lbrynet.__version__
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
@_log_decorator
def configure_loggly_handler(url=None, **kwargs):
url = url or get_loggly_url()
json_format = {
"loggerName": "%(name)s",
"asciTime": "%(asctime)s",
"fileName": "%(filename)s",
"functionName": "%(funcName)s",
"levelNo": "%(levelno)s",
"lineNo": "%(lineno)d",
"levelName": "%(levelname)s",
"message": "%(message)s",
}
json_format.update(kwargs)
formatter = logging.Formatter(json.dumps(json_format))
handler = HTTPSHandler(url)
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler
@@ -4,6 +4,9 @@ from zope.interface import implements
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
log = logging.getLogger(__name__)
class BlobAvailabilityHandlerFactory(object):
implements(IQueryHandlerFactory)
@@ -37,10 +40,11 @@ class BlobAvailabilityHandler(object):
def handle_queries(self, queries):
if self.query_identifiers[0] in queries:
logging.debug("Received the client's list of requested blobs")
log.debug("Received the client's list of requested blobs")
d = self._get_available_blobs(queries[self.query_identifiers[0]])
def set_field(available_blobs):
log.debug("available blobs: %s", str(available_blobs))
return {'available_blobs': available_blobs}
d.addCallback(set_field)
+11 -7
View File
@@ -6,6 +6,9 @@ from zope.interface import implements
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
log = logging.getLogger(__name__)
class BlobRequestHandlerFactory(object):
implements(IQueryHandlerFactory)
@@ -58,7 +61,7 @@ class BlobRequestHandler(object):
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
if self.query_identifiers[1] in queries:
logging.debug("Received the client's request to send a blob")
log.debug("Received the client's request to send a blob")
response_fields = {}
response['incoming_blob'] = response_fields
@@ -75,11 +78,11 @@ class BlobRequestHandler(object):
if read_handle is not None:
self.currently_uploading = blob
self.read_handle = read_handle
logging.debug("Sending %s to client", str(blob))
log.debug("Sending %s to client", str(blob))
response_fields['blob_hash'] = blob.blob_hash
response_fields['length'] = blob.length
return response
logging.debug("We can not send %s", str(blob))
log.debug("We can not send %s", str(blob))
response_fields['error'] = "BLOB_UNAVAILABLE"
return response
@@ -129,14 +132,15 @@ class BlobRequestHandler(object):
def start_transfer():
self.file_sender = FileSender()
logging.info("Starting the file upload")
log.info("Starting the file upload")
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
return d
def set_expected_payment():
logging.info("Setting expected payment")
log.info("Setting expected payment")
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
# TODO: explain why 2**20
self.wallet.add_expected_payment(self.peer,
self.currently_uploading.length * 1.0 *
self.blob_data_payment_rate / 2**20)
@@ -151,6 +155,6 @@ class BlobRequestHandler(object):
self.currently_uploading = None
self.file_sender = None
if reason is not None and isinstance(reason, Failure):
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
log.info("Upload has failed. Reason: %s", reason.getErrorMessage())
return _send_file()
return _send_file()
+2 -2
View File
@@ -1,5 +1,5 @@
import binascii
from twisted.internet import defer, task, reactor
from twisted.internet import defer, reactor
import collections
@@ -78,4 +78,4 @@ class DHTHashSupplier(object):
self.hash_reannounce_time = 60 * 60 # 1 hour
def hashes_to_announce(self):
pass
pass
+9 -6
View File
@@ -6,6 +6,9 @@ from zope.interface import implements
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
log = logging.getLogger(__name__)
class ServerProtocol(Protocol):
"""ServerProtocol needs to:
@@ -26,7 +29,7 @@ class ServerProtocol(Protocol):
#Protocol stuff
def connectionMade(self):
logging.debug("Got a connection")
log.debug("Got a connection")
peer_info = self.transport.getPeer()
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
self.request_handler = ServerRequestHandler(self)
@@ -34,7 +37,7 @@ class ServerProtocol(Protocol):
if enabled is True:
query_handler = query_handler_factory.build_query_handler()
query_handler.register_with_request_handler(self.request_handler, self.peer)
logging.debug("Setting the request handler")
log.debug("Setting the request handler")
self.factory.rate_limiter.register_protocol(self)
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
@@ -42,10 +45,10 @@ class ServerProtocol(Protocol):
self.request_handler.stopProducing()
self.factory.rate_limiter.unregister_protocol(self)
if not reason.check(error.ConnectionDone):
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
def dataReceived(self, data):
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
self.factory.rate_limiter.report_dl_bytes(len(data))
if self.request_handler is not None:
self.request_handler.data_received(data)
@@ -53,7 +56,7 @@ class ServerProtocol(Protocol):
#IConsumer stuff
def registerProducer(self, producer, streaming):
logging.debug("Registering the producer")
log.debug("Registering the producer")
assert streaming is True
def unregisterProducer(self):
@@ -61,7 +64,7 @@ class ServerProtocol(Protocol):
self.transport.loseConnection()
def write(self, data):
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
log.debug("Writing %s bytes of data to the transport", str(len(data)))
self.transport.write(data)
self.factory.rate_limiter.report_ul_bytes(len(data))
+18 -15
View File
@@ -5,6 +5,9 @@ from zope.interface import implements
from lbrynet.interfaces import IRequestHandler
log = logging.getLogger(__name__)
class ServerRequestHandler(object):
"""This class handles requests from clients. It can upload blobs and return request for information about
more blobs that are associated with streams"""
@@ -52,7 +55,7 @@ class ServerRequestHandler(object):
chunk = self.response_buff[:self.CHUNK_SIZE]
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
if chunk != '':
logging.debug("writing %s bytes to the client", str(len(chunk)))
log.debug("writing %s bytes to the client", str(len(chunk)))
self.consumer.write(chunk)
reactor.callLater(0, self._produce_more)
@@ -76,7 +79,7 @@ class ServerRequestHandler(object):
def get_more_data():
if self.producer is not None:
logging.debug("Requesting more data from the producer")
log.debug("Requesting more data from the producer")
self.producer.resumeProducing()
reactor.callLater(0, get_more_data)
@@ -84,8 +87,8 @@ class ServerRequestHandler(object):
#From Protocol
def data_received(self, data):
logging.debug("Received data")
logging.debug("%s", str(data))
log.debug("Received data")
log.debug("%s", str(data))
if self.request_received is False:
self.request_buff = self.request_buff + data
msg = self.try_to_parse_request(self.request_buff)
@@ -96,10 +99,10 @@ class ServerRequestHandler(object):
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
else:
logging.info("Request buff not a valid json message")
logging.info("Request buff: %s", str(self.request_buff))
log.debug("Request buff not a valid json message")
log.debug("Request buff: %s", str(self.request_buff))
else:
logging.warning("The client sent data when we were uploading a file. This should not happen")
log.warning("The client sent data when we were uploading a file. This should not happen")
######### IRequestHandler #########
@@ -112,7 +115,7 @@ class ServerRequestHandler(object):
#response handling
def request_failure_handler(self, err):
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
self.stopProducing()
return err
@@ -122,15 +125,15 @@ class ServerRequestHandler(object):
def send_response(self, msg):
m = json.dumps(msg)
logging.info("Sending a response of length %s", str(len(m)))
logging.debug("Response: %s", str(m))
log.debug("Sending a response of length %s", str(len(m)))
log.debug("Response: %s", str(m))
self.response_buff = self.response_buff + m
self._produce_more()
return True
def handle_request(self, msg):
logging.debug("Handling a request")
logging.debug(str(msg))
log.debug("Handling a request")
log.debug(str(msg))
def create_response_message(results):
response = {}
@@ -140,11 +143,11 @@ class ServerRequestHandler(object):
else:
# result is a Failure
return result
logging.debug("Finished making the response message. Response: %s", str(response))
log.debug("Finished making the response message. Response: %s", str(response))
return response
def log_errors(err):
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
log.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
return err
def send_response(response):
@@ -168,4 +171,4 @@ class ServerRequestHandler(object):
msg = json.loads(request_buff)
return msg
except ValueError:
return None
return None
+23
View File
@@ -0,0 +1,23 @@
import sqlite3
from twisted.internet import task, reactor
import logging
log = logging.getLogger(__name__)
def rerun_if_locked(f):
def rerun(err, *args, **kwargs):
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
str(f), str(args), str(kwargs))
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
return err
def wrapper(*args, **kwargs):
d = f(*args, **kwargs)
d.addErrback(rerun, *args, **kwargs)
return d
return wrapper
+22 -2
View File
@@ -1,6 +1,10 @@
from lbrynet.core.cryptoutils import get_lbry_hash_obj
import base64
import distutils.version
import random
from lbrynet.core.cryptoutils import get_lbry_hash_obj
blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded
@@ -25,4 +29,20 @@ def is_valid_blobhash(blobhash):
for l in blobhash:
if l not in "0123456789abcdef":
return False
return True
return True
def version_is_greater_than(a, b):
"""Returns True if version a is more recent than version b"""
try:
return distutils.version.StrictVersion(a) > distutils.version.StrictVersion(b)
except ValueError:
return distutils.version.LooseVersion(a) > distutils.version.LooseVersion(b)
def deobfuscate(obfustacated):
return base64.b64decode(obfustacated.decode('rot13'))
def obfuscate(plain):
return base64.b64encode(plain).encode('rot13')
+5 -2
View File
@@ -5,6 +5,9 @@ from lbrynet.conf import BLOB_SIZE
from lbrynet.core.BlobInfo import BlobInfo
log = logging.getLogger(__name__)
class CryptBlobInfo(BlobInfo):
def __init__(self, blob_hash, blob_num, length, iv):
BlobInfo.__init__(self, blob_hash, blob_num, length)
@@ -78,12 +81,12 @@ class CryptStreamBlobMaker(object):
return done, num_bytes_to_write
def close(self):
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
if self.length != 0:
self._close_buffer()
d = self.blob.close()
d.addCallback(self._return_info)
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
return d
def _write_buffer(self):
+8 -5
View File
@@ -12,6 +12,9 @@ from lbrynet.core.StreamCreator import StreamCreator
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
log = logging.getLogger(__name__)
class CryptStreamCreator(StreamCreator):
"""Create a new stream with blobs encrypted by a symmetric cipher.
@@ -57,18 +60,18 @@ class CryptStreamCreator(StreamCreator):
return defer.succeed(True)
def _finalize(self):
logging.debug("_finalize has been called")
log.debug("_finalize has been called")
self.blob_count += 1
iv = self.iv_generator.next()
final_blob_creator = self.blob_manager.get_blob_creator()
logging.debug("Created the finished_deferred")
log.debug("Created the finished_deferred")
final_blob = self._get_blob_maker(iv, final_blob_creator)
logging.debug("Created the final blob")
logging.debug("Calling close on final blob")
log.debug("Created the final blob")
log.debug("Calling close on final blob")
d = final_blob.close()
d.addCallback(self._blob_finished)
self.finished_deferreds.append(d)
logging.debug("called close on final blob, returning from make_final_blob")
log.debug("called close on final blob, returning from make_final_blob")
return d
def _write(self, data):
@@ -1,3 +1,4 @@
import logging
from zope.interface import implements
from lbrynet.interfaces import IStreamDownloader
from lbrynet.core.client.BlobRequester import BlobRequester
@@ -9,6 +10,9 @@ from twisted.internet import defer
from twisted.python.failure import Failure
log = logging.getLogger(__name__)
class StartFailedError(Exception):
pass
@@ -71,6 +75,9 @@ class CryptStreamDownloader(object):
self.points_paid = 0.0
def __str__(self):
return str(self.stream_name)
def toggle_running(self):
if self.stopped is True:
return self.start()
@@ -79,10 +86,6 @@ class CryptStreamDownloader(object):
def start(self):
def set_finished_deferred():
self.finished_deferred = defer.Deferred()
return self.finished_deferred
if self.starting is True:
raise CurrentlyStartingError()
if self.stopping is True:
@@ -92,11 +95,13 @@ class CryptStreamDownloader(object):
assert self.download_manager is None
self.starting = True
self.completed = False
self.finished_deferred = defer.Deferred()
fd = self.finished_deferred
d = self._start()
d.addCallback(lambda _: set_finished_deferred())
d.addCallback(lambda _: fd)
return d
def stop(self):
def stop(self, err=None):
def check_if_stop_succeeded(success):
self.stopping = False
@@ -112,8 +117,8 @@ class CryptStreamDownloader(object):
assert self.download_manager is not None
self.stopping = True
d = self.download_manager.stop_downloading()
self._fire_completed_deferred()
d.addCallback(check_if_stop_succeeded)
d.addCallback(lambda _: self._fire_completed_deferred(err))
return d
def _start_failed(self):
@@ -196,10 +201,15 @@ class CryptStreamDownloader(object):
self._get_primary_request_creators(download_manager),
self._get_secondary_request_creators(download_manager))
def _fire_completed_deferred(self):
def _fire_completed_deferred(self, err=None):
self.finished_deferred, d = None, self.finished_deferred
if d is not None:
d.callback(self._get_finished_deferred_callback_value())
if err is not None:
d.errback(err)
else:
d.callback(self._get_finished_deferred_callback_value())
else:
log.debug("Not firing the completed deferred because d is None")
def _get_finished_deferred_callback_value(self):
return None
@@ -209,5 +219,5 @@ class CryptStreamDownloader(object):
self.completed = True
return self.stop()
def insufficient_funds(self):
return self.stop()
def insufficient_funds(self, err):
return self.stop(err=err)
View File
Whitespace-only changes.
+25
View File
@@ -0,0 +1,25 @@
import logging
def migrate_db(db_dir, start, end):
current = start
old_dirs = []
while current < end:
if current == 0:
from lbrynet.db_migrator.migrate0to1 import do_migration
old_dirs.append(do_migration(db_dir))
current += 1
return old_dirs
def run_migration_script():
import sys
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
sys.stdout = open("migrator.out.log", 'w')
sys.stderr = open("migrator.err.log", 'w')
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
if __name__ == "__main__":
run_migration_script()
+307
View File
@@ -0,0 +1,307 @@
import sqlite3
import unqlite
import leveldb
import shutil
import os
import logging
import json
log = logging.getLogger(__name__)
known_dbs = ['lbryfile_desc.db', 'lbryfiles.db', 'valuable_blobs.db', 'blobs.db',
'lbryfile_blob.db', 'lbryfile_info.db', 'settings.db', 'blind_settings.db',
'blind_peers.db', 'blind_info.db', 'lbryfile_info.db', 'lbryfile_manager.db',
'live_stream.db', 'stream_info.db', 'stream_blob.db', 'stream_desc.db']
def do_move(from_dir, to_dir):
for known_db in known_dbs:
known_db_path = os.path.join(from_dir, known_db)
if os.path.exists(known_db_path):
log.debug("Moving %s to %s",
os.path.abspath(known_db_path),
os.path.abspath(os.path.join(to_dir, known_db)))
shutil.move(known_db_path, os.path.join(to_dir, known_db))
else:
log.debug("Did not find %s", os.path.abspath(known_db_path))
def do_migration(db_dir):
old_dir = os.path.join(db_dir, "_0_to_1_old")
new_dir = os.path.join(db_dir, "_0_to_1_new")
try:
log.info("Moving dbs from the real directory to %s", os.path.abspath(old_dir))
os.makedirs(old_dir)
do_move(db_dir, old_dir)
except:
log.error("An error occurred moving the old db files.")
raise
try:
log.info("Creating the new directory in %s", os.path.abspath(new_dir))
os.makedirs(new_dir)
except:
log.error("An error occurred creating the new directory.")
raise
try:
log.info("Doing the migration")
migrate_blob_db(old_dir, new_dir)
migrate_lbryfile_db(old_dir, new_dir)
migrate_livestream_db(old_dir, new_dir)
migrate_ptc_db(old_dir, new_dir)
migrate_lbryfile_manager_db(old_dir, new_dir)
migrate_settings_db(old_dir, new_dir)
migrate_repeater_db(old_dir, new_dir)
log.info("Migration succeeded")
except:
log.error("An error occurred during the migration. Restoring.")
do_move(old_dir, db_dir)
raise
try:
log.info("Moving dbs in the new directory to the real directory")
do_move(new_dir, db_dir)
db_revision = open(os.path.join(db_dir, 'db_revision'), mode='w+')
db_revision.write("1")
db_revision.close()
os.rmdir(new_dir)
except:
log.error("An error occurred moving the new db files.")
raise
return old_dir
def migrate_blob_db(old_db_dir, new_db_dir):
old_blob_db_path = os.path.join(old_db_dir, "blobs.db")
if not os.path.exists(old_blob_db_path):
return True
old_db = leveldb.LevelDB(old_blob_db_path)
new_db_conn = sqlite3.connect(os.path.join(new_db_dir, "blobs.db"))
c = new_db_conn.cursor()
c.execute("create table if not exists blobs (" +
" blob_hash text primary key, " +
" blob_length integer, " +
" last_verified_time real, " +
" next_announce_time real"
")")
new_db_conn.commit()
c = new_db_conn.cursor()
for blob_hash, blob_info in old_db.RangeIter():
blob_length, verified_time, announce_time = json.loads(blob_info)
c.execute("insert into blobs values (?, ?, ?, ?)",
(blob_hash, blob_length, verified_time, announce_time))
new_db_conn.commit()
new_db_conn.close()
def migrate_lbryfile_db(old_db_dir, new_db_dir):
old_lbryfile_db_path = os.path.join(old_db_dir, "lbryfiles.db")
if not os.path.exists(old_lbryfile_db_path):
return True
stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_info.db"))
stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_blob.db"))
stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_desc.db"))
db_conn = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
c = db_conn.cursor()
c.execute("create table if not exists lbry_files (" +
" stream_hash text primary key, " +
" key text, " +
" stream_name text, " +
" suggested_file_name text" +
")")
c.execute("create table if not exists lbry_file_blobs (" +
" blob_hash text, " +
" stream_hash text, " +
" position integer, " +
" iv text, " +
" length integer, " +
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
c.execute("create table if not exists lbry_file_descriptors (" +
" sd_blob_hash TEXT PRIMARY KEY, " +
" stream_hash TEXT, " +
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
db_conn.commit()
c = db_conn.cursor()
for stream_hash, stream_info in stream_info_db.RangeIter():
key, name, suggested_file_name = json.loads(stream_info)
c.execute("insert into lbry_files values (?, ?, ?, ?)",
(stream_hash, key, name, suggested_file_name))
db_conn.commit()
c = db_conn.cursor()
for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
position, iv, length = json.loads(blob_info)
c.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
(b_h, s_h, position, iv, length))
db_conn.commit()
c = db_conn.cursor()
for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
c.execute("insert into lbry_file_descriptors values (?, ?)",
(sd_blob_hash, stream_hash))
db_conn.commit()
db_conn.close()
def migrate_livestream_db(old_db_dir, new_db_dir):
old_db_path = os.path.join(old_db_dir, "stream_info.db")
if not os.path.exists(old_db_path):
return True
stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_info.db"))
stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_blob.db"))
stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_desc.db"))
db_conn = sqlite3.connect(os.path.join(new_db_dir, "live_stream.db"))
c = db_conn.cursor()
c.execute("create table if not exists live_streams (" +
" stream_hash text primary key, " +
" public_key text, " +
" key text, " +
" stream_name text, " +
" next_announce_time real" +
")")
c.execute("create table if not exists live_stream_blobs (" +
" blob_hash text, " +
" stream_hash text, " +
" position integer, " +
" revision integer, " +
" iv text, " +
" length integer, " +
" signature text, " +
" foreign key(stream_hash) references live_streams(stream_hash)" +
")")
c.execute("create table if not exists live_stream_descriptors (" +
" sd_blob_hash TEXT PRIMARY KEY, " +
" stream_hash TEXT, " +
" foreign key(stream_hash) references live_streams(stream_hash)" +
")")
db_conn.commit()
c = db_conn.cursor()
for stream_hash, stream_info in stream_info_db.RangeIter():
public_key, key, name, next_announce_time = json.loads(stream_info)
c.execute("insert into live_streams values (?, ?, ?, ?, ?)",
(stream_hash, public_key, key, name, next_announce_time))
db_conn.commit()
c = db_conn.cursor()
for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
position, revision, iv, length, signature = json.loads(blob_info)
c.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
(b_h, s_h, position, revision, iv, length, signature))
db_conn.commit()
c = db_conn.cursor()
for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
c.execute("insert into live_stream_descriptors values (?, ?)",
(sd_blob_hash, stream_hash))
db_conn.commit()
db_conn.close()
def migrate_ptc_db(old_db_dir, new_db_dir):
old_db_path = os.path.join(old_db_dir, "ptcwallet.db")
if not os.path.exists(old_db_path):
return True
old_db = leveldb.LevelDB(old_db_path)
try:
p_key = old_db.Get("private_key")
new_db = unqlite.UnQLite(os.path.join(new_db_dir, "ptcwallet.db"))
new_db['private_key'] = p_key
except KeyError:
pass
def migrate_lbryfile_manager_db(old_db_dir, new_db_dir):
old_db_path = os.path.join(old_db_dir, "lbryfiles.db")
if not os.path.exists(old_db_path):
return True
old_db = leveldb.LevelDB(old_db_path)
new_db = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
c = new_db.cursor()
c.execute("create table if not exists lbry_file_options (" +
" blob_data_rate real, " +
" status text," +
" stream_hash text,"
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
new_db.commit()
LBRYFILE_STATUS = "t"
LBRYFILE_OPTIONS = "o"
c = new_db.cursor()
for k, v in old_db.RangeIter():
key_type, stream_hash = json.loads(k)
if key_type == LBRYFILE_STATUS:
try:
rate = json.loads(old_db.Get(json.dumps((LBRYFILE_OPTIONS, stream_hash))))[0]
except KeyError:
rate = None
c.execute("insert into lbry_file_options values (?, ?, ?)",
(rate, v, stream_hash))
new_db.commit()
new_db.close()
def migrate_settings_db(old_db_dir, new_db_dir):
old_settings_db_path = os.path.join(old_db_dir, "settings.db")
if not os.path.exists(old_settings_db_path):
return True
old_db = leveldb.LevelDB(old_settings_db_path)
new_db = unqlite.UnQLite(os.path.join(new_db_dir, "settings.db"))
for k, v in old_db.RangeIter():
new_db[k] = v
def migrate_repeater_db(old_db_dir, new_db_dir):
old_repeater_db_path = os.path.join(old_db_dir, "valuable_blobs.db")
if not os.path.exists(old_repeater_db_path):
return True
old_db = leveldb.LevelDB(old_repeater_db_path)
info_db = sqlite3.connect(os.path.join(new_db_dir, "blind_info.db"))
peer_db = sqlite3.connect(os.path.join(new_db_dir, "blind_peers.db"))
unql_db = unqlite.UnQLite(os.path.join(new_db_dir, "blind_settings.db"))
BLOB_INFO_TYPE = 'b'
SETTING_TYPE = 's'
PEER_TYPE = 'p'
info_c = info_db.cursor()
info_c.execute("create table if not exists valuable_blobs (" +
" blob_hash text primary key, " +
" blob_length integer, " +
" reference text, " +
" peer_host text, " +
" peer_port integer, " +
" peer_score text" +
")")
info_db.commit()
peer_c = peer_db.cursor()
peer_c.execute("create table if not exists approved_peers (" +
" ip_address text, " +
" port integer" +
")")
peer_db.commit()
info_c = info_db.cursor()
peer_c = peer_db.cursor()
for k, v in old_db.RangeIter():
key_type, key_rest = json.loads(k)
if key_type == PEER_TYPE:
host, port = key_rest
peer_c.execute("insert into approved_peers values (?, ?)",
(host, port))
elif key_type == SETTING_TYPE:
unql_db[key_rest] = v
elif key_type == BLOB_INFO_TYPE:
blob_hash = key_rest
length, reference, peer_host, peer_port, peer_score = json.loads(v)
info_c.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
(blob_hash, length, reference, peer_host, peer_port, peer_score))
info_db.commit()
peer_db.commit()
info_db.close()
peer_db.close()
-3
View File
@@ -8,10 +8,7 @@
# may be created by processing this file with epydoc: http://epydoc.sf.net
import UserDict
#import sqlite3
import cPickle as pickle
import time
import os
import constants
+4 -1
View File
@@ -99,7 +99,10 @@ class Bencode(Encoding):
"""
if len(data) == 0:
raise DecodeError, 'Cannot decode empty string'
return self._decodeRecursive(data)[0]
try:
return self._decodeRecursive(data)[0]
except ValueError as e:
raise DecodeError, e.message
@staticmethod
def _decodeRecursive(data, startIndex=0):
+20 -11
View File
@@ -7,9 +7,9 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
import hashlib, random, struct, time, math, binascii
import hashlib, random, struct, time, binascii
import argparse
from twisted.internet import defer
from twisted.internet import defer, error
import constants
import routingtable
import datastore
@@ -22,6 +22,9 @@ from hashwatcher import HashWatcher
import logging
log = logging.getLogger(__name__)
def rpcmethod(func):
""" Decorator to expose Node methods as remote procedure calls
@@ -139,7 +142,13 @@ class Node(object):
"""
# Prepare the underlying Kademlia protocol
if self.port is not None:
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol) #IGNORE:E1101
try:
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
raise ValueError("%s lbrynet may already be running." % str(e))
#IGNORE:E1101
# Create temporary contact information for the list of addresses of known nodes
if knownNodeAddresses != None:
bootstrapContacts = []
@@ -231,12 +240,12 @@ class Node(object):
known_nodes = {}
def log_error(err, n):
logging.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
logging.error(binascii.hexlify(err.getErrorMessage()))
logging.error(err.getTraceback())
log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
log.debug(err.getErrorMessage())
log.debug(err.getTraceback())
def log_success(res):
logging.debug("Response to store request: %s", str(res))
log.debug("Response to store request: %s", str(res))
return res
def announce_to_peer(responseTuple):
@@ -332,7 +341,7 @@ class Node(object):
# originalPublisherID = self.id
#
# def log_error(err):
# logging.error(err.getErrorMessage())
# log.error(err.getErrorMessage())
#
# # Prepare a callback for doing "STORE" RPC calls
# def executeStoreRPCs(nodes):
@@ -507,7 +516,7 @@ class Node(object):
else:
raise TypeError, 'No NodeID given. Therefore we can\'t store this node'
if self_store is True and self.externalIP is not None:
if self_store is True and self.externalIP:
contact = Contact(self.id, self.externalIP, self.port, None, None)
compact_ip = contact.compact_ip()
elif '_rpcNodeContact' in kwargs:
@@ -756,7 +765,7 @@ class Node(object):
searchIteration()
def log_error(err):
logging.error(err.getErrorMessage())
log.error(err.getErrorMessage())
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
def searchIteration():
@@ -1008,4 +1017,4 @@ def main():
twisted.internet.reactor.run()
if __name__ == '__main__':
main()
main()
+8
View File
@@ -7,6 +7,7 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
import binascii
import time
from twisted.internet import protocol, defer
@@ -23,6 +24,13 @@ reactor = twisted.internet.reactor
class TimeoutError(Exception):
""" Raised when a RPC times out """
def __init__(self, remote_contact_id):
# remote_contact_id is a binary blob so we need to convert it
# into something more readable
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
Exception.__init__(self, msg)
self.remote_contact_id = remote_contact_id
class KademliaProtocol(protocol.DatagramProtocol):
""" Implements all low-level network-related functions of a Kademlia node """
+12 -9
View File
@@ -6,6 +6,9 @@ import sys
from lbrynet.core.utils import generate_id
log = logging.getLogger(__name__)
def print_usage():
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
@@ -13,15 +16,15 @@ def print_usage():
def join_network(udp_port, known_nodes):
lbryid = generate_id()
logging.info('Creating Node...')
log.info('Creating Node...')
node = Node(udpPort=udp_port, lbryid=lbryid)
logging.info('Joining network...')
log.info('Joining network...')
d = node.joinNetwork(known_nodes)
def log_network_size():
logging.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
logging.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
log.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
log.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
d.addCallback(lambda _: log_network_size())
@@ -36,7 +39,7 @@ def get_hosts(node, h):
print "Hosts returned from the DHT: "
print hosts
logging.info("Looking up %s", h)
log.info("Looking up %s", h)
d = node.getPeersForBlob(h)
d.addCallback(print_hosts)
return d
@@ -48,9 +51,9 @@ def announce_hash(node, h):
def log_results(results):
for success, result in results:
if success:
logging.info("Succeeded: %s", str(result))
log.info("Succeeded: %s", str(result))
else:
logging.info("Failed: %s", str(result.getErrorMessage()))
log.info("Failed: %s", str(result.getErrorMessage()))
d.addCallback(log_results)
return d
@@ -80,11 +83,11 @@ def run_dht_script(dht_func):
d.addCallback(run_dht_func)
def log_err(err):
logging.error("An error occurred: %s", err.getTraceback())
log.error("An error occurred: %s", err.getTraceback())
return err
def shut_down():
logging.info("Shutting down")
log.info("Shutting down")
reactor.stop()
d.addErrback(log_err)
+2 -2
View File
@@ -20,7 +20,7 @@
import os, sys, time, signal, hashlib, random
import sys, hashlib, random
import twisted.internet.reactor
from lbrynet.dht.node import Node
#from entangled.kademlia.datastore import SQLiteDataStore
@@ -106,7 +106,7 @@ def stop():
if __name__ == '__main__':
import sys, os
import sys
if len(sys.argv) < 2:
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]
+26 -9
View File
@@ -445,10 +445,7 @@ class IQueryHandlerFactory(Interface):
"""
class IStreamDownloaderFactory(Interface):
"""
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
"""
class IStreamDownloaderOptions(Interface):
def get_downloader_options(self, sd_validator, payment_rate_manager):
"""
Return the list of options that can be used to modify IStreamDownloader behavior
@@ -459,8 +456,28 @@ class IStreamDownloaderFactory(Interface):
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
@type payment_rate_manager: PaymentRateManager
@return: [(option_description, default)]
@rtype: [(string, string)]
@return: [DownloadOption]
@rtype: [DownloadOption]
"""
class IStreamDownloaderFactory(Interface):
"""
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
"""
def can_download(self, sd_validator, payment_rate_manager):
"""
Decide whether the downloaders created by this factory can download the stream described by sd_validator
@param sd_validator: object containing stream metadata
@type sd_validator: object which implements IStreamDescriptorValidator interface
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
@type payment_rate_manager: PaymentRateManager
@return: True if the downloaders can download the stream, False otherwise
@rtype: bool
"""
def make_downloader(self, sd_validator, options, payment_rate_manager):
@@ -470,10 +487,10 @@ class IStreamDownloaderFactory(Interface):
@param sd_validator: object containing stream metadata which will be given to the IStreamDownloader
@type sd_validator: object which implements IStreamDescriptorValidator interface
@param options: a list of strings that will be used by the IStreamDownloaderFactory to
@param options: a list of values that will be used by the IStreamDownloaderFactory to
construct the IStreamDownloader. the options are in the same order as they were given
by get_downloader_options.
@type options: [string]
@type options: [Object]
@param payment_rate_manager: the PaymentRateManager which the IStreamDownloader should use.
@type payment_rate_manager: PaymentRateManager
@@ -504,7 +521,7 @@ class IStreamDownloader(Interface):
@rtype: Deferred which fires with anything
"""
def insufficient_funds(self):
def insufficient_funds(self, err):
"""
this function informs the stream downloader that funds are too low to finish downloading.
+164 -96
View File
@@ -1,9 +1,14 @@
import logging
import leveldb
import json
import sqlite3
import os
from twisted.internet import threads, defer
from lbrynet.core.Error import DuplicateStreamHashError
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.enterprise import adbapi
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
from lbrynet.core.sqlite_helpers import rerun_if_locked
log = logging.getLogger(__name__)
class DBLBRYFileMetadataManager(object):
@@ -16,163 +21,226 @@ class DBLBRYFileMetadataManager(object):
self.stream_desc_db = None
def setup(self):
return threads.deferToThread(self._open_db)
return self._open_db()
def stop(self):
self.stream_info_db = None
self.stream_blob_db = None
self.stream_desc_db = None
self.db_conn = None
return defer.succeed(True)
def get_all_streams(self):
return threads.deferToThread(self._get_all_streams)
return self._get_all_streams()
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
d = threads.deferToThread(self._store_stream, stream_hash, file_name, key, suggested_file_name)
d = self._store_stream(stream_hash, file_name, key, suggested_file_name)
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
return d
def get_stream_info(self, stream_hash):
return threads.deferToThread(self._get_stream_info, stream_hash)
return self._get_stream_info(stream_hash)
def check_if_stream_exists(self, stream_hash):
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
return self._check_if_stream_exists(stream_hash)
def delete_stream(self, stream_hash):
return threads.deferToThread(self._delete_stream, stream_hash)
return self._delete_stream(stream_hash)
def add_blobs_to_stream(self, stream_hash, blobs):
def add_blobs():
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
return threads.deferToThread(add_blobs)
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
logging.info("Getting blobs for a stream. Count is %s", str(count))
log.debug("Getting blobs for a stream. Count is %s", str(count))
def get_positions_of_start_and_end():
if start_blob is not None:
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
else:
start_num = None
d1 = defer.succeed(None)
if end_blob is not None:
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
else:
d2 = defer.succeed(None)
dl = defer.DeferredList([d1, d2])
def get_positions(results):
start_num = None
end_num = None
return start_num, end_num
if results[0][0] is True:
start_num = results[0][1]
if results[1][0] is True:
end_num = results[1][1]
return start_num, end_num
dl.addCallback(get_positions)
return dl
def get_blob_infos(nums):
start_num, end_num = nums
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
count, reverse)
return self._get_further_blob_infos(stream_hash, start_num, end_num,
count, reverse)
d = threads.deferToThread(get_positions_of_start_and_end)
d = get_positions_of_start_and_end()
d.addCallback(get_blob_infos)
return d
def get_stream_of_blob(self, blob_hash):
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
return self._get_stream_of_blobhash(blob_hash)
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
def get_sd_blob_hashes_for_stream(self, stream_hash):
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
return self._get_sd_blob_hashes_for_stream(stream_hash)
def _open_db(self):
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_info.db"))
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_blob.db"))
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_desc.db"))
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.db_conn = adbapi.ConnectionPool("sqlite3", (os.path.join(self.db_dir, "lbryfile_info.db")),
check_same_thread=False)
def create_tables(transaction):
transaction.execute("create table if not exists lbry_files (" +
" stream_hash text primary key, " +
" key text, " +
" stream_name text, " +
" suggested_file_name text" +
")")
transaction.execute("create table if not exists lbry_file_blobs (" +
" blob_hash text, " +
" stream_hash text, " +
" position integer, " +
" iv text, " +
" length integer, " +
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
transaction.execute("create table if not exists lbry_file_descriptors (" +
" sd_blob_hash TEXT PRIMARY KEY, " +
" stream_hash TEXT, " +
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
return self.db_conn.runInteraction(create_tables)
@rerun_if_locked
def _delete_stream(self, stream_hash):
desc_batch = leveldb.WriteBatch()
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
if stream_hash == s_h:
desc_batch.Delete(sd_blob_hash)
self.stream_desc_db.Write(desc_batch, sync=True)
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
blob_batch = leveldb.WriteBatch()
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if stream_hash == s_h:
blob_batch.Delete(blob_hash_stream_hash)
self.stream_blob_db.Write(blob_batch, sync=True)
def do_delete(transaction, s_h):
transaction.execute("delete from lbry_files where stream_hash = ?", (s_h,))
transaction.execute("delete from lbry_file_blobs where stream_hash = ?", (s_h,))
transaction.execute("delete from lbry_file_descriptors where stream_hash = ?", (s_h,))
stream_batch = leveldb.WriteBatch()
for s_h, stream_info in self.stream_info_db.RangeIter():
if stream_hash == s_h:
stream_batch.Delete(s_h)
self.stream_info_db.Write(stream_batch, sync=True)
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
return d
@rerun_if_locked
def _store_stream(self, stream_hash, name, key, suggested_file_name):
try:
self.stream_info_db.Get(stream_hash)
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
except KeyError:
pass
self.stream_info_db.Put(stream_hash, json.dumps((key, name, suggested_file_name)), sync=True)
d = self.db_conn.runQuery("insert into lbry_files values (?, ?, ?, ?)",
(stream_hash, key, name, suggested_file_name))
def check_duplicate(err):
if err.check(sqlite3.IntegrityError):
raise DuplicateStreamHashError(stream_hash)
return err
d.addErrback(check_duplicate)
return d
@rerun_if_locked
def _get_all_streams(self):
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
d = self.db_conn.runQuery("select stream_hash from lbry_files")
d.addCallback(lambda results: [r[0] for r in results])
return d
@rerun_if_locked
def _get_stream_info(self, stream_hash):
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
d = self.db_conn.runQuery("select key, stream_name, suggested_file_name from lbry_files where stream_hash = ?",
(stream_hash,))
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
return d
@rerun_if_locked
def _check_if_stream_exists(self, stream_hash):
try:
self.stream_info_db.Get(stream_hash)
return True
except KeyError:
return False
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
d.addCallback(lambda r: True if len(r) else False)
return d
@rerun_if_locked
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
d = self.db_conn.runQuery("select position from lbry_file_blobs where stream_hash = ? and blob_hash = ?",
(stream_hash, blob_hash))
d.addCallback(lambda r: r[0][0] if len(r) else None)
return d
@rerun_if_locked
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
blob_infos = []
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if stream_hash == s_h:
position, iv, length = json.loads(blob_info)
if (start_num is None) or (position > start_num):
if (end_num is None) or (position < end_num):
blob_infos.append((b_h, position, iv, length))
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
params = []
q_string = "select * from ("
q_string += " select blob_hash, position, iv, length from lbry_file_blobs "
q_string += " where stream_hash = ? "
params.append(stream_hash)
if start_num is not None:
q_string += " and position > ? "
params.append(start_num)
if end_num is not None:
q_string += " and position < ? "
params.append(end_num)
q_string += " order by position "
if reverse is True:
q_string += " DESC "
if count is not None:
blob_infos = blob_infos[:count]
return blob_infos
q_string += " limit ? "
params.append(count)
q_string += ") order by position"
# Order by position is done twice so that it always returns them from lowest position to
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
return self.db_conn.runQuery(q_string, tuple(params))
@rerun_if_locked
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
batch = leveldb.WriteBatch()
for blob_info in blob_infos:
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
try:
self.stream_blob_db.Get(blob_hash_stream_hash)
if ignore_duplicate_error is False:
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
continue
except KeyError:
pass
batch.Put(blob_hash_stream_hash,
json.dumps((blob_info.blob_num,
blob_info.iv,
blob_info.length)))
self.stream_blob_db.Write(batch, sync=True)
def add_blobs(transaction):
for blob_info in blob_infos:
try:
transaction.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
blob_info.iv, blob_info.length))
except sqlite3.IntegrityError:
if ignore_duplicate_error is False:
raise
return self.db_conn.runInteraction(add_blobs)
@rerun_if_locked
def _get_stream_of_blobhash(self, blob_hash):
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if blob_hash == b_h:
return s_h
return None
d = self.db_conn.runQuery("select stream_hash from lbry_file_blobs where blob_hash = ?",
(blob_hash,))
d.addCallback(lambda r: r[0][0] if len(r) else None)
return d
@rerun_if_locked
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
log.info("Saving sd blob hash %s to stream hash %s", str(sd_blob_hash), str(stream_hash))
d = self.db_conn.runQuery("insert into lbry_file_descriptors values (?, ?)",
(sd_blob_hash, stream_hash))
def ignore_duplicate(err):
err.trap(sqlite3.IntegrityError)
log.info("sd blob hash already known")
d.addErrback(ignore_duplicate)
return d
@rerun_if_locked
def _get_sd_blob_hashes_for_stream(self, stream_hash):
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
log.info("Looking up sd blob hashes for stream hash %s", str(stream_hash))
d = self.db_conn.runQuery("select sd_blob_hash from lbry_file_descriptors where stream_hash = ?",
(stream_hash,))
d.addCallback(lambda results: [r[0] for r in results])
return d
class TempLBRYFileMetadataManager(object):
+64 -12
View File
@@ -2,15 +2,20 @@ import binascii
import logging
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
from twisted.internet import defer
from lbrynet.core.Error import DuplicateStreamHashError
from twisted.internet import defer, threads
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
import os
log = logging.getLogger(__name__)
LBRYFileStreamType = "lbryfile"
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
logging.debug("Saving info for %s", str(sd_info['stream_name']))
log.debug("Saving info for %s", str(sd_info['stream_name']))
hex_stream_name = sd_info['stream_name']
key = sd_info['key']
stream_hash = sd_info['stream_hash']
@@ -26,7 +31,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
blob_num = blob['blob_num']
iv = blob['iv']
crypt_blobs.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
log.debug("Trying to save stream info for %s", str(hex_stream_name))
d = stream_info_manager.save_stream(stream_hash, hex_stream_name, key,
suggested_file_name, crypt_blobs)
@@ -75,12 +80,53 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
return d
def publish_sd_blob(stream_info_manager, blob_manager, stream_hash):
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
d = get_sd_info(stream_info_manager, stream_hash, True)
d.addCallback(descriptor_writer.create_descriptor)
def add_sd_blob_to_stream(sd_blob_hash):
d = stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
d.addCallback(lambda _: sd_blob_hash)
return d
d.addCallback(add_sd_blob_to_stream)
return d
def create_plain_sd(stream_info_manager, stream_hash, file_name, overwrite_existing=False):
def _get_file_name():
actual_file_name = file_name
if os.path.exists(actual_file_name):
ext_num = 1
while os.path.exists(actual_file_name + "_" + str(ext_num)):
ext_num += 1
actual_file_name = actual_file_name + "_" + str(ext_num)
return actual_file_name
if overwrite_existing is False:
d = threads.deferToThread(_get_file_name())
else:
d = defer.succeed(file_name)
def do_create(file_name):
descriptor_writer = PlainStreamDescriptorWriter(file_name)
d = get_sd_info(stream_info_manager, stream_hash, True)
d.addCallback(descriptor_writer.create_descriptor)
return d
d.addCallback(do_create)
return d
class LBRYFileStreamDescriptorValidator(object):
def __init__(self, raw_info):
self.raw_info = raw_info
def validate(self):
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
try:
hex_stream_name = self.raw_info['stream_name']
key = self.raw_info['key']
@@ -88,11 +134,10 @@ class LBRYFileStreamDescriptorValidator(object):
stream_hash = self.raw_info['stream_hash']
blobs = self.raw_info['blobs']
except KeyError as e:
raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
for c in hex_suggested_file_name:
if c not in '0123456789abcdef':
raise ValueError("Invalid stream descriptor: "
"suggested file name is not a hex-encoded string")
raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string")
h = get_lbry_hash_obj()
h.update(hex_stream_name)
h.update(key)
@@ -118,10 +163,11 @@ class LBRYFileStreamDescriptorValidator(object):
for blob in blobs:
blobs_hashsum.update(get_blob_hashsum(blob))
if blobs[-1]['length'] != 0:
raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
h.update(blobs_hashsum.digest())
if h.hexdigest() != stream_hash:
raise ValueError("Stream hash does not match stream metadata")
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
log.debug("It is validated")
return defer.succeed(True)
def info_to_show(self):
@@ -130,9 +176,15 @@ class LBRYFileStreamDescriptorValidator(object):
size_so_far = 0
for blob_info in self.raw_info.get("blobs", []):
size_so_far += int(blob_info['length'])
info.append(("stream_size", str(size_so_far)))
info.append(("stream_size", str(self.get_length_of_stream())))
suggested_file_name = self.raw_info.get("suggested_file_name", None)
if suggested_file_name is not None:
suggested_file_name = binascii.unhexlify(suggested_file_name)
info.append(("suggested_file_name", suggested_file_name))
return info
return info
def get_length_of_stream(self):
size_so_far = 0
for blob_info in self.raw_info.get("blobs", []):
size_so_far += int(blob_info['length'])
return size_so_far
+2
View File
@@ -0,0 +1,2 @@
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
+107 -37
View File
@@ -3,14 +3,20 @@ import binascii
from zope.interface import implements
from lbrynet.core.DownloadOption import DownloadOption
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.core.StreamDescriptor import StreamMetadata
from lbrynet.interfaces import IStreamDownloaderFactory
from lbrynet.lbryfile.client.LBRYFileMetadataHandler import LBRYFileMetadataHandler
import os
from twisted.internet import defer, threads, reactor
from twisted.python.procutils import which
import logging
import traceback
log = logging.getLogger(__name__)
class LBRYFileDownloader(CryptStreamDownloader):
@@ -40,9 +46,34 @@ class LBRYFileDownloader(CryptStreamDownloader):
else:
return defer.succeed(True)
def stop(self):
def delete_data(self):
d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
def get_blob_hashes(blob_infos):
return [b[0] for b in blob_infos if b[0] is not None]
d1.addCallback(get_blob_hashes)
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash)
def combine_blob_hashes(results):
blob_hashes = []
for success, result in results:
if success is True:
blob_hashes.extend(result)
return blob_hashes
def delete_blobs(blob_hashes):
self.blob_manager.delete_blobs(blob_hashes)
return True
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
dl.addCallback(combine_blob_hashes)
dl.addCallback(delete_blobs)
return dl
def stop(self, err=None):
d = self._close_output()
d.addCallback(lambda _: CryptStreamDownloader.stop(self))
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
return d
def _get_progress_manager(self, download_manager):
@@ -60,6 +91,15 @@ class LBRYFileDownloader(CryptStreamDownloader):
pass
def get_total_bytes(self):
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
def calculate_size(blobs):
return sum([b[3] for b in blobs])
d.addCallback(calculate_size)
return d
def get_total_bytes_cached(self):
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
if self.download_manager is None:
return 0
@@ -94,37 +134,30 @@ class LBRYFileDownloaderFactory(object):
self.stream_info_manager = stream_info_manager
self.wallet = wallet
def get_downloader_options(self, sd_validator, payment_rate_manager):
options = [
DownloadOption(
[float, None],
"rate which will be paid for data (None means use application default)",
"data payment rate",
None
),
DownloadOption(
[bool],
"allow reuploading data downloaded for this file",
"allow upload",
True
),
]
return options
def can_download(self, sd_validator):
return True
def make_downloader(self, sd_validator, options, payment_rate_manager, **kwargs):
if options[0] is not None:
payment_rate_manager.float(options[0])
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
payment_rate_manager.min_blob_data_payment_rate = options[0]
upload_allowed = options[1]
def save_source_if_blob(stream_hash):
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
else:
d = defer.succeed(True)
d.addCallback(lambda _: stream_hash)
return d
def create_downloader(stream_hash):
downloader = self._make_downloader(stream_hash, payment_rate_manager, sd_validator.raw_info,
upload_allowed)
downloader = self._make_downloader(stream_hash, payment_rate_manager,
metadata.validator.raw_info, upload_allowed)
d = downloader.set_stream_info()
d.addCallback(lambda _: downloader)
return d
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
d.addCallback(save_source_if_blob)
d.addCallback(create_downloader)
return d
@@ -139,8 +172,15 @@ class LBRYFileSaver(LBRYFileDownloader):
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
self.download_directory = download_directory
self.file_name = file_name
self.file_written_to = None
self.file_handle = None
def __str__(self):
if self.file_written_to is not None:
return str(self.file_written_to)
else:
return str(self.file_name)
def set_stream_info(self):
d = LBRYFileDownloader.set_stream_info(self)
@@ -154,14 +194,14 @@ class LBRYFileSaver(LBRYFileDownloader):
d.addCallback(lambda _: set_file_name())
return d
def stop(self):
d = LBRYFileDownloader.stop(self)
def stop(self, err=None):
d = LBRYFileDownloader.stop(self, err=err)
d.addCallback(lambda _: self._delete_from_info_manager())
return d
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
delete_blob_after_finished=True)
delete_blob_after_finished=not self.upload_allowed)
def _setup_output(self):
def open_file():
@@ -171,11 +211,28 @@ class LBRYFileSaver(LBRYFileDownloader):
file_name = "_"
if os.path.exists(os.path.join(self.download_directory, file_name)):
ext_num = 1
def _get_file_name(ext):
if len(file_name.split(".")):
fn = ''.join(file_name.split(".")[:-1])
file_ext = ''.join(file_name.split(".")[-1])
return fn + "-" + str(ext) + "." + file_ext
else:
return file_name + "_" + str(ext)
while os.path.exists(os.path.join(self.download_directory,
file_name + "_" + str(ext_num))):
_get_file_name(ext_num))):
ext_num += 1
file_name = file_name + "_" + str(ext_num)
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
file_name = _get_file_name(ext_num)
try:
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
self.file_written_to = os.path.join(self.download_directory, file_name)
except IOError:
log.error(traceback.format_exc())
raise ValueError("Failed to open %s. Make sure you have permission to save files to that"
" location." % str(os.path.join(self.download_directory,
file_name)))
return threads.deferToThread(open_file)
def _close_output(self):
@@ -212,7 +269,8 @@ class LBRYFileSaverFactory(LBRYFileDownloaderFactory):
self.stream_info_manager, payment_rate_manager, self.wallet,
self.download_directory, upload_allowed)
def get_description(self):
@staticmethod
def get_description():
return "Save"
@@ -224,14 +282,14 @@ class LBRYFileOpener(LBRYFileDownloader):
self.process = None
self.process_log = None
def stop(self):
d = LBRYFileDownloader.stop(self)
def stop(self, err=None):
d = LBRYFileDownloader.stop(self, err=err)
d.addCallback(lambda _: self._delete_from_info_manager())
return d
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
delete_blob_after_finished=True)
delete_blob_after_finished=not self.upload_allowed)
def _setup_output(self):
def start_process():
@@ -276,9 +334,21 @@ class LBRYFileOpener(LBRYFileDownloader):
class LBRYFileOpenerFactory(LBRYFileDownloaderFactory):
def can_download(self, sd_validator):
if which('vlc'):
return True
elif os.name == "nt":
paths = [r'C:\Program Files\VideoLAN\VLC\vlc.exe',
r'C:\Program Files (x86)\VideoLAN\VLC\vlc.exe']
for p in paths:
if os.path.exists(p):
return True
return False
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
return LBRYFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
self.stream_info_manager, payment_rate_manager, self.wallet, upload_allowed)
def get_description(self):
@staticmethod
def get_description():
return "Stream"
@@ -4,6 +4,9 @@ from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
from lbrynet.interfaces import IMetadataHandler
log = logging.getLogger(__name__)
class LBRYFileMetadataHandler(object):
implements(IMetadataHandler)
@@ -31,6 +34,6 @@ class LBRYFileMetadataHandler(object):
if blob_hash is not None:
infos.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
else:
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
self._final_blob_num = blob_num - 1
return infos
@@ -0,0 +1,55 @@
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType, LBRYFileStreamDescriptorValidator
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
def add_lbry_file_to_sd_identifier(sd_identifier):
sd_identifier.add_stream_type(LBRYFileStreamType, LBRYFileStreamDescriptorValidator, LBRYFileOptions())
class LBRYFileOptions(object):
def __init__(self):
pass
def get_downloader_options(self, sd_validator, payment_rate_manager):
prm = payment_rate_manager
def get_default_data_rate_description():
if prm.min_blob_data_payment_rate is None:
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
else:
return "%f LBC/MB" % prm.min_blob_data_payment_rate
rate_choices = []
rate_choices.append(DownloadOptionChoice(prm.min_blob_data_payment_rate,
"No change - %s" % get_default_data_rate_description(),
"No change - %s" % get_default_data_rate_description()))
if prm.min_blob_data_payment_rate is not None:
rate_choices.append(DownloadOptionChoice(None,
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
rate_choices.append(DownloadOptionChoice(float,
"Enter rate in LBC/MB",
"Enter rate in LBC/MB"))
options = [
DownloadOption(
rate_choices,
"Rate which will be paid for data",
"data payment rate",
prm.min_blob_data_payment_rate,
get_default_data_rate_description()
),
DownloadOption(
[
DownloadOptionChoice(bool,
None,
None,
bool_options_description=("Allow", "Disallow")),
],
"Allow reuploading data downloaded for this file",
"allow upload",
True,
"Allow"
),
]
return options
+11 -13
View File
@@ -11,7 +11,9 @@ from lbrynet import conf
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from twisted.protocols.basic import FileSender
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
log = logging.getLogger(__name__)
class LBRYFileStreamCreator(CryptStreamCreator):
@@ -30,10 +32,10 @@ class LBRYFileStreamCreator(CryptStreamCreator):
self.blob_infos = []
def _blob_finished(self, blob_info):
logging.debug("length: %s", str(blob_info.length))
log.debug("length: %s", str(blob_info.length))
self.blob_infos.append(blob_info)
def _save_lbry_file_info(self):
def _save_stream_info(self):
stream_info_manager = self.lbry_file_manager.stream_info_manager
d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
binascii.hexlify(self.key),
@@ -43,8 +45,6 @@ class LBRYFileStreamCreator(CryptStreamCreator):
def setup(self):
d = CryptStreamCreator.setup(self)
d.addCallback(lambda _: self.stream_hash)
return d
def _get_blobs_hashsum(self):
@@ -76,10 +76,7 @@ class LBRYFileStreamCreator(CryptStreamCreator):
def _finished(self):
self._make_stream_hash()
d = self._save_lbry_file_info()
d.addCallback(lambda _: self.lbry_file_manager.change_lbry_file_status(
self.stream_hash, ManagedLBRYFileDownloader.STATUS_FINISHED
))
d = self._save_stream_info()
return d
@@ -128,12 +125,13 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
"""
def stop_file(creator):
logging.debug("the file sender has triggered its deferred. stopping the stream writer")
log.debug("the file sender has triggered its deferred. stopping the stream writer")
return creator.stop()
def make_stream_desc_file(stream_hash):
logging.debug("creating the stream descriptor file")
descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
log.debug("creating the stream descriptor file")
descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION)
descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
@@ -156,4 +154,4 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
d = lbry_file_creator.setup()
d.addCallback(lambda _: start_stream())
return d
return d
+73 -29
View File
@@ -2,9 +2,9 @@
Download LBRY Files from LBRYnet and save them to disk.
"""
from lbrynet.core.DownloadOption import DownloadOption
from zope.interface import implements
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.core.StreamDescriptor import StreamMetadata
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaver, LBRYFileDownloader
from lbrynet.lbryfilemanager.LBRYFileStatusReport import LBRYFileStatusReport
from lbrynet.interfaces import IStreamDownloaderFactory
@@ -18,19 +18,46 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
lbry_file_manager, payment_rate_manager, wallet, download_directory, upload_allowed,
file_name=None):
LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
stream_info_manager, payment_rate_manager, wallet, download_directory,
upload_allowed)
upload_allowed, file_name)
self.sd_hash = None
self.txid = None
self.uri = None
self.claim_id = None
self.rowid = rowid
self.lbry_file_manager = lbry_file_manager
self.file_name = file_name
self.file_handle = None
self.saving_status = False
def restore(self):
d = self.lbry_file_manager.get_lbry_file_status(self.stream_hash)
d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
def _save_sd_hash(sd_hash):
if len(sd_hash):
self.sd_hash = sd_hash[0]
d = self.wallet._get_claim_metadata_for_sd_hash(self.sd_hash)
else:
d = defer.succeed(None)
return d
def _save_claim_id(claim_id):
self.claim_id = claim_id
return defer.succeed(None)
def _save_claim(name, txid):
self.uri = name
self.txid = txid
d = self.wallet.get_claimid(name, txid)
d.addCallback(_save_claim_id)
return d
d.addCallback(_save_sd_hash)
d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None)
d.addCallback(lambda _: self.lbry_file_manager.get_lbry_file_status(self))
def restore_status(status):
if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
@@ -44,12 +71,12 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
d.addCallback(restore_status)
return d
def stop(self, change_status=True):
def stop(self, err=None, change_status=True):
def set_saving_status_done():
self.saving_status = False
d = LBRYFileDownloader.stop(self) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
d = LBRYFileDownloader.stop(self, err=err) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
if change_status is True:
self.saving_status = True
d.addCallback(lambda _: self._save_status())
@@ -88,6 +115,24 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
d = LBRYFileSaver._start(self)
d.addCallback(lambda _: self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash))
def _save_sd_hash(sd_hash):
if len(sd_hash):
self.sd_hash = sd_hash[0]
d = self.wallet._get_claim_metadata_for_sd_hash(self.sd_hash)
else:
d = defer.succeed(None)
return d
def _save_claim(name, txid):
self.uri = name
self.txid = txid
return defer.succeed(None)
d.addCallback(_save_sd_hash)
d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None)
d.addCallback(lambda _: self._save_status())
return d
@@ -105,7 +150,7 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
s = ManagedLBRYFileDownloader.STATUS_STOPPED
else:
s = ManagedLBRYFileDownloader.STATUS_RUNNING
return self.lbry_file_manager.change_lbry_file_status(self.stream_hash, s)
return self.lbry_file_manager.change_lbry_file_status(self, s)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
@@ -117,33 +162,32 @@ class ManagedLBRYFileDownloaderFactory(object):
def __init__(self, lbry_file_manager):
self.lbry_file_manager = lbry_file_manager
def get_downloader_options(self, sd_validator, payment_rate_manager):
options = [
DownloadOption(
[float, None],
"rate which will be paid for data (None means use application default)",
"data payment rate",
None
),
DownloadOption(
[bool],
"allow reuploading data downloaded for this file",
"allow upload",
True
),
]
return options
def can_download(self, sd_validator):
return True
def make_downloader(self, sd_validator, options, payment_rate_manager):
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None):
data_rate = options[0]
upload_allowed = options[1]
d = save_sd_info(self.lbry_file_manager.stream_info_manager, sd_validator.raw_info)
def save_source_if_blob(stream_hash):
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
d = self.lbry_file_manager.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash,
metadata.source_blob_hash)
else:
d = defer.succeed(True)
d.addCallback(lambda _: stream_hash)
return d
d = save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info)
d.addCallback(save_source_if_blob)
d.addCallback(lambda stream_hash: self.lbry_file_manager.add_lbry_file(stream_hash,
payment_rate_manager,
data_rate,
upload_allowed))
upload_allowed,
download_directory=download_directory,
file_name=file_name))
return d
def get_description(self):
@staticmethod
def get_description():
return "Save the file to disk"
+113 -123
View File
@@ -3,84 +3,55 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
"""
import logging
import json
import leveldb
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamDescriptorValidator
import os
from twisted.enterprise import adbapi
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.core.PaymentRateManager import PaymentRateManager
from twisted.internet import threads, defer, task, reactor
from twisted.python.failure import Failure
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
from lbrynet.core.sqlite_helpers import rerun_if_locked
log = logging.getLogger(__name__)
class LBRYFileManager(object):
"""
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
"""
SETTING = "s"
LBRYFILE_STATUS = "t"
LBRYFILE_OPTIONS = "o"
def __init__(self, session, stream_info_manager, sd_identifier):
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
self.session = session
self.stream_info_manager = stream_info_manager
self.sd_identifier = sd_identifier
self.lbry_files = []
self.db = None
self.download_directory = os.getcwd()
self.sql_db = None
if download_directory:
self.download_directory = download_directory
else:
self.download_directory = os.getcwd()
log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
def setup(self):
d = threads.deferToThread(self._open_db)
d = self._open_db()
d.addCallback(lambda _: self._add_to_sd_identifier())
d.addCallback(lambda _: self._start_lbry_files())
return d
def get_all_lbry_file_stream_hashes_and_options(self):
d = threads.deferToThread(self._get_all_lbry_file_stream_hashes)
def get_lbry_file_status(self, lbry_file):
return self._get_lbry_file_status(lbry_file.rowid)
def get_options(stream_hashes):
ds = []
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)
def get_options_for_stream_hash(stream_hash):
d = self.get_lbry_file_options(stream_hash)
d.addCallback(lambda options: (stream_hash, options))
return d
for stream_hash in stream_hashes:
ds.append(get_options_for_stream_hash(stream_hash))
dl = defer.DeferredList(ds)
dl.addCallback(lambda results: [r[1] for r in results if r[0]])
return dl
d.addCallback(get_options)
return d
def get_lbry_file_status(self, stream_hash):
return threads.deferToThread(self._get_lbry_file_status, stream_hash)
def save_lbry_file_options(self, stream_hash, blob_data_rate):
return threads.deferToThread(self._save_lbry_file_options, stream_hash, blob_data_rate)
def get_lbry_file_options(self, stream_hash):
return threads.deferToThread(self._get_lbry_file_options, stream_hash)
def delete_lbry_file_options(self, stream_hash):
return threads.deferToThread(self._delete_lbry_file_options, stream_hash)
def set_lbry_file_data_payment_rate(self, stream_hash, new_rate):
return threads.deferToThread(self._set_lbry_file_payment_rate, stream_hash, new_rate)
def change_lbry_file_status(self, stream_hash, status):
logging.debug("Changing status of %s to %s", stream_hash, status)
return threads.deferToThread(self._change_file_status, stream_hash, status)
def delete_lbry_file_status(self, stream_hash):
return threads.deferToThread(self._delete_lbry_file_status, stream_hash)
def change_lbry_file_status(self, lbry_file, status):
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
return self._change_file_status(lbry_file.rowid, status)
def get_lbry_file_status_reports(self):
ds = []
@@ -98,52 +69,66 @@ class LBRYFileManager(object):
def _add_to_sd_identifier(self):
downloader_factory = ManagedLBRYFileDownloaderFactory(self)
self.sd_identifier.add_stream_info_validator(LBRYFileStreamType, LBRYFileStreamDescriptorValidator)
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
def _start_lbry_files(self):
def set_options_and_restore(stream_hash, options):
def set_options_and_restore(rowid, stream_hash, options):
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
d = self.add_lbry_file(stream_hash, payment_rate_manager, blob_data_rate=options[0])
d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
blob_data_rate=options)
d.addCallback(lambda downloader: downloader.restore())
return d
def log_error(err):
logging.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
def log_error(err, rowid, stream_hash, options):
log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
log.error(rowid)
log.error(stream_hash)
log.error(options)
def start_lbry_files(stream_hashes_and_options):
for stream_hash, options in stream_hashes_and_options:
d = set_options_and_restore(stream_hash, options)
d.addErrback(log_error)
def start_lbry_files(lbry_files_and_options):
for rowid, stream_hash, options in lbry_files_and_options:
d = set_options_and_restore(rowid, stream_hash, options)
d.addErrback(lambda err: log_error(err, rowid, stream_hash, options))
return True
d = self.get_all_lbry_file_stream_hashes_and_options()
d = self._get_all_lbry_files()
d.addCallback(start_lbry_files)
return d
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True):
def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
download_directory=None, file_name=None):
if not download_directory:
download_directory = self.download_directory
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
lbry_file_downloader = ManagedLBRYFileDownloader(stream_hash, self.session.peer_finder,
self.session.rate_limiter, self.session.blob_manager,
lbry_file_downloader = ManagedLBRYFileDownloader(rowid, stream_hash,
self.session.peer_finder,
self.session.rate_limiter,
self.session.blob_manager,
self.stream_info_manager, self,
payment_rate_manager, self.session.wallet,
self.download_directory,
upload_allowed)
download_directory,
upload_allowed,
file_name=file_name)
self.lbry_files.append(lbry_file_downloader)
d = self.save_lbry_file_options(stream_hash, blob_data_rate)
d.addCallback(lambda _: lbry_file_downloader.set_stream_info())
d = lbry_file_downloader.set_stream_info()
d.addCallback(lambda _: lbry_file_downloader)
return d
def delete_lbry_file(self, stream_hash):
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
download_directory=None, file_name=None):
d = self._save_lbry_file(stream_hash, blob_data_rate)
d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
blob_data_rate, upload_allowed, download_directory, file_name))
return d
def delete_lbry_file(self, lbry_file):
for l in self.lbry_files:
if l.stream_hash == stream_hash:
if l == lbry_file:
lbry_file = l
break
else:
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
stream_hash)))
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
def wait_for_finished(count=2):
if count <= 0 or lbry_file.saving_status is False:
@@ -162,26 +147,19 @@ class LBRYFileManager(object):
self.lbry_files.remove(lbry_file)
d.addCallback(lambda _: remove_from_list())
d.addCallback(lambda _: self.delete_lbry_file_options(stream_hash))
d.addCallback(lambda _: self.delete_lbry_file_status(stream_hash))
d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
return d
def toggle_lbry_file_running(self, stream_hash):
def toggle_lbry_file_running(self, lbry_file):
"""Toggle whether a stream reader is currently running"""
for l in self.lbry_files:
if l.stream_hash == stream_hash:
if l == lbry_file:
return l.toggle_running()
else:
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
stream_hash)))
def get_stream_hash_from_name(self, lbry_file_name):
for l in self.lbry_files:
if l.file_name == lbry_file_name:
return l.stream_hash
return None
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
def stop(self):
ds = []
def wait_for_finished(lbry_file, count=2):
@@ -206,50 +184,62 @@ class LBRYFileManager(object):
dl.addCallback(lambda _: close_db())
return dl
def get_count_for_stream_hash(self, stream_hash):
return self._get_count_for_stream_hash(stream_hash)
######### database calls #########
def _open_db(self):
self.db = leveldb.LevelDB(os.path.join(self.session.db_dir, "lbryfiles.db"))
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
check_same_thread=False)
return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
" blob_data_rate real, " +
" status text," +
" stream_hash text,"
" foreign key(stream_hash) references lbry_files(stream_hash)" +
")")
def _save_payment_rate(self, rate_type, rate):
if rate is not None:
self.db.Put(json.dumps((self.SETTING, rate_type)), json.dumps(rate), sync=True)
else:
self.db.Delete(json.dumps((self.SETTING, rate_type)), sync=True)
@rerun_if_locked
def _save_lbry_file(self, stream_hash, data_payment_rate):
def do_save(db_transaction):
db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
(data_payment_rate, ManagedLBRYFileDownloader.STATUS_STOPPED,
stream_hash))
return db_transaction.lastrowid
return self.sql_db.runInteraction(do_save)
def _save_lbry_file_options(self, stream_hash, blob_data_rate):
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((blob_data_rate,)),
sync=True)
@rerun_if_locked
def _delete_lbry_file_options(self, rowid):
return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
(rowid,))
def _get_lbry_file_options(self, stream_hash):
try:
return json.loads(self.db.Get(json.dumps((self.LBRYFILE_OPTIONS, stream_hash))))
except KeyError:
return None, None
@rerun_if_locked
def _set_lbry_file_payment_rate(self, rowid, new_rate):
return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
(new_rate, rowid))
def _delete_lbry_file_options(self, stream_hash):
self.db.Delete(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), sync=True)
@rerun_if_locked
def _get_all_lbry_files(self):
d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
return d
def _set_lbry_file_payment_rate(self, stream_hash, new_rate):
@rerun_if_locked
def _change_file_status(self, rowid, new_status):
return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
(new_status, rowid))
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((new_rate, )), sync=True)
@rerun_if_locked
def _get_lbry_file_status(self, rowid):
d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
(rowid,))
d.addCallback(lambda r: r[0][0] if len(r) else ManagedLBRYFileDownloader.STATUS_STOPPED)
return d
def _get_all_lbry_file_stream_hashes(self):
hashes = []
for k, v in self.db.RangeIter():
key_type, stream_hash = json.loads(k)
if key_type == self.LBRYFILE_STATUS:
hashes.append(stream_hash)
return hashes
def _change_file_status(self, stream_hash, new_status):
self.db.Put(json.dumps((self.LBRYFILE_STATUS, stream_hash)), new_status, sync=True)
def _get_lbry_file_status(self, stream_hash):
try:
return self.db.Get(json.dumps((self.LBRYFILE_STATUS, stream_hash)))
except KeyError:
return ManagedLBRYFileDownloader.STATUS_STOPPED
def _delete_lbry_file_status(self, stream_hash):
self.db.Delete(json.dumps((self.LBRYFILE_STATUS, stream_hash)), sync=True)
@rerun_if_locked
def _get_count_for_stream_hash(self, stream_hash):
return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
(stream_hash,))
+8 -4
View File
@@ -1,3 +1,6 @@
# pylint: skip-file
# This file is not maintained, but might be used in the future
#
import logging
import sys
from lbrynet.lbrylive.LiveStreamCreator import StdOutLiveStreamCreator
@@ -15,7 +18,8 @@ from twisted.internet import defer, task
class LBRYStdinUploader():
"""This class reads from standard in, creates a stream, and makes it available on the network."""
def __init__(self, peer_port, dht_node_port, known_dht_nodes):
def __init__(self, peer_port, dht_node_port, known_dht_nodes,
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
"""
@param peer_port: the network port on which to listen for peers
@@ -25,8 +29,8 @@ class LBRYStdinUploader():
"""
self.peer_port = peer_port
self.lbry_server_port = None
self.session = LBRYSession(blob_manager_class=TempBlobManager,
stream_info_manager_class=DBLiveStreamMetadataManager,
self.session = LBRYSession(blob_manager_class=blob_manager_class,
stream_info_manager_class=stream_info_manager_class,
dht_node_class=Node, dht_node_port=dht_node_port,
known_dht_nodes=known_dht_nodes, peer_port=self.peer_port,
use_upnp=False)
@@ -114,4 +118,4 @@ def launch_stdin_uploader():
d.addCallback(lambda _: start_stdin_uploader())
d.addCallback(lambda _: shut_down())
reactor.addSystemEventTrigger('before', 'shutdown', uploader.shut_down)
reactor.run()
reactor.run()
+10 -5
View File
@@ -1,7 +1,10 @@
# pylint: skip-file
# This file is not maintained, but might be used in the future
#
import logging
import sys
from lbrynet.lbrynet_console.plugins.LBRYLive.LBRYLiveStreamDownloader import LBRYLiveStreamDownloader
from lbrynet.lbrylive.client.LiveStreamDownloader import LBRYLiveStreamDownloader
from lbrynet.core.BlobManager import TempBlobManager
from lbrynet.core.Session import LBRYSession
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
@@ -15,15 +18,17 @@ from twisted.internet import task
class LBRYStdoutDownloader():
"""This class downloads a live stream from the network and outputs it to standard out."""
def __init__(self, dht_node_port, known_dht_nodes):
def __init__(self, dht_node_port, known_dht_nodes,
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
"""
@param dht_node_port: the network port on which to listen for DHT node requests
@param known_dht_nodes: a list of (ip_address, dht_port) which will be used to join the DHT network
"""
self.session = LBRYSession(blob_manager_class=TempBlobManager,
stream_info_manager_class=DBLiveStreamMetadataManager,
self.session = LBRYSession(blob_manager_class=blob_manager_class,
stream_info_manager_class=stream_info_manager_class,
dht_node_class=Node, dht_node_port=dht_node_port, known_dht_nodes=known_dht_nodes,
use_upnp=False)
self.payment_rate_manager = BaseLiveStreamPaymentRateManager()
@@ -93,4 +98,4 @@ def launch_stdout_downloader():
d.addErrback(print_error)
d.addCallback(lambda _: shut_down())
reactor.addSystemEventTrigger('before', 'shutdown', downloader.shut_down)
reactor.run()
reactor.run()
+9 -22
View File
@@ -1,20 +1,20 @@
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter
from lbrynet.lbrylive.StreamDescriptor import get_sd_info, LiveStreamType, LBRYLiveStreamDescriptorValidator
from lbrynet.lbrylive.StreamDescriptor import get_sd_info
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
from lbrynet.lbrylive.LiveBlob import LiveStreamBlobMaker
from lbrynet.lbrylive.PaymentRateManager import BaseLiveStreamPaymentRateManager
from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_pass_phrase
from Crypto import Random
import binascii
import logging
from lbrynet.conf import CRYPTSD_FILE_EXTENSION
from lbrynet.conf import MIN_BLOB_INFO_PAYMENT_RATE
from lbrynet.lbrylive.client.LiveStreamDownloader import FullLiveStreamDownloaderFactory
from twisted.internet import interfaces, defer
from twisted.protocols.basic import FileSender
from zope.interface import implements
log = logging.getLogger(__name__)
class LiveStreamCreator(CryptStreamCreator):
def __init__(self, blob_manager, stream_info_manager, name=None, key=None, iv_generator=None,
delete_after_num=None, secret_pass_phrase=None):
@@ -33,8 +33,8 @@ class LiveStreamCreator(CryptStreamCreator):
return d
def _blob_finished(self, blob_info):
logging.debug("In blob_finished")
logging.debug("length: %s", str(blob_info.length))
log.debug("In blob_finished")
log.debug("length: %s", str(blob_info.length))
sig_hash = get_lbry_hash_obj()
sig_hash.update(self.stream_hash)
if blob_info.length != 0:
@@ -51,11 +51,11 @@ class LiveStreamCreator(CryptStreamCreator):
d = self.stream_info_manager.add_blobs_to_stream(self.stream_hash, [blob_info])
def log_add_error(err):
logging.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
log.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
return err
d.addErrback(log_add_error)
logging.debug("returning from blob_finished")
log.debug("returning from blob_finished")
return d
def setup(self):
@@ -173,17 +173,4 @@ class StdinStreamProducer(object):
self.consumer.write(data)
def childConnectionLost(self, fd, reason):
self.stopProducing()
def add_live_stream_to_sd_identifier(session, stream_info_manager, sd_identifier):
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
session.rate_limiter,
session.blob_manager,
stream_info_manager,
session.wallet,
BaseLiveStreamPaymentRateManager(
MIN_BLOB_INFO_PAYMENT_RATE
))
sd_identifier.add_stream_info_validator(LiveStreamType, LBRYLiveStreamDescriptorValidator)
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
self.stopProducing()
+177 -116
View File
@@ -1,11 +1,16 @@
import time
import logging
import leveldb
import json
from twisted.enterprise import adbapi
import os
from twisted.internet import threads, defer
import sqlite3
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
from lbrynet.core.Error import DuplicateStreamHashError
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
from lbrynet.core.sqlite_helpers import rerun_if_locked
log = logging.getLogger(__name__)
class DBLiveStreamMetadataManager(DHTHashSupplier):
@@ -14,26 +19,22 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
def __init__(self, db_dir, hash_announcer):
DHTHashSupplier.__init__(self, hash_announcer)
self.db_dir = db_dir
self.stream_info_db = None
self.stream_blob_db = None
self.stream_desc_db = None
self.db_conn = None
def setup(self):
return threads.deferToThread(self._open_db)
return self._open_db()
def stop(self):
self.stream_info_db = None
self.stream_blob_db = None
self.stream_desc_db = None
self.db_conn = None
return defer.succeed(True)
def get_all_streams(self):
return threads.deferToThread(self._get_all_streams)
return self._get_all_streams()
def save_stream(self, stream_hash, pub_key, file_name, key, blobs):
next_announce_time = time.time() + self.hash_reannounce_time
d = threads.deferToThread(self._store_stream, stream_hash, pub_key, file_name, key,
next_announce_time=next_announce_time)
d = self._store_stream(stream_hash, pub_key, file_name, key,
next_announce_time=next_announce_time)
def save_blobs():
return self.add_blobs_to_stream(stream_hash, blobs)
@@ -48,169 +49,229 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
return d
def get_stream_info(self, stream_hash):
return threads.deferToThread(self._get_stream_info, stream_hash)
return self._get_stream_info(stream_hash)
def check_if_stream_exists(self, stream_hash):
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
return self._check_if_stream_exists(stream_hash)
def delete_stream(self, stream_hash):
return threads.deferToThread(self._delete_stream, stream_hash)
return self._delete_stream(stream_hash)
def add_blobs_to_stream(self, stream_hash, blobs):
def add_blobs():
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
return threads.deferToThread(add_blobs)
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
logging.info("Getting blobs for a stream. Count is %s", str(count))
log.info("Getting blobs for a stream. Count is %s", str(count))
def get_positions_of_start_and_end():
if start_blob is not None:
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
else:
start_num = None
d1 = defer.succeed(None)
if end_blob is not None:
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
else:
d2 = defer.succeed(None)
dl = defer.DeferredList([d1, d2])
def get_positions(results):
start_num = None
end_num = None
return start_num, end_num
if results[0][0] is True:
start_num = results[0][1]
if results[1][0] is True:
end_num = results[1][1]
return start_num, end_num
dl.addCallback(get_positions)
return dl
def get_blob_infos(nums):
start_num, end_num = nums
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
count, reverse)
return self._get_further_blob_infos(stream_hash, start_num, end_num,
count, reverse)
d = threads.deferToThread(get_positions_of_start_and_end)
d = get_positions_of_start_and_end()
d.addCallback(get_blob_infos)
return d
def get_stream_of_blob(self, blob_hash):
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
return self._get_stream_of_blobhash(blob_hash)
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
def get_sd_blob_hashes_for_stream(self, stream_hash):
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
return self._get_sd_blob_hashes_for_stream(stream_hash)
def hashes_to_announce(self):
next_announce_time = time.time() + self.hash_reannounce_time
return threads.deferToThread(self._get_streams_to_announce, next_announce_time)
return self._get_streams_to_announce(next_announce_time)
######### database calls #########
def _open_db(self):
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_info.db"))
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_blob.db"))
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_desc.db"))
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "live_stream.db"),
check_same_thread=False)
def create_tables(transaction):
transaction.execute("create table if not exists live_streams (" +
" stream_hash text primary key, " +
" public_key text, " +
" key text, " +
" stream_name text, " +
" next_announce_time real" +
")")
transaction.execute("create table if not exists live_stream_blobs (" +
" blob_hash text, " +
" stream_hash text, " +
" position integer, " +
" revision integer, " +
" iv text, " +
" length integer, " +
" signature text, " +
" foreign key(stream_hash) references live_streams(stream_hash)" +
")")
transaction.execute("create table if not exists live_stream_descriptors (" +
" sd_blob_hash TEXT PRIMARY KEY, " +
" stream_hash TEXT, " +
" foreign key(stream_hash) references live_streams(stream_hash)" +
")")
return self.db_conn.runInteraction(create_tables)
@rerun_if_locked
def _delete_stream(self, stream_hash):
desc_batch = leveldb.WriteBatch()
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
if stream_hash == s_h:
desc_batch.Delete(sd_blob_hash)
self.stream_desc_db.Write(desc_batch, sync=True)
blob_batch = leveldb.WriteBatch()
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if stream_hash == s_h:
blob_batch.Delete(blob_hash_stream_hash)
self.stream_blob_db.Write(blob_batch, sync=True)
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
stream_batch = leveldb.WriteBatch()
for s_h, stream_info in self.stream_info_db.RangeIter():
if stream_hash == s_h:
stream_batch.Delete(s_h)
self.stream_info_db.Write(stream_batch, sync=True)
def do_delete(transaction, s_h):
transaction.execute("delete from live_streams where stream_hash = ?", (s_h,))
transaction.execute("delete from live_stream_blobs where stream_hash = ?", (s_h,))
transaction.execute("delete from live_stream_descriptors where stream_hash = ?", (s_h,))
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
return d
@rerun_if_locked
def _store_stream(self, stream_hash, public_key, name, key, next_announce_time=None):
try:
self.stream_info_db.Get(stream_hash)
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
except KeyError:
pass
self.stream_info_db.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)), sync=True)
d = self.db_conn.runQuery("insert into live_streams values (?, ?, ?, ?, ?)",
(stream_hash, public_key, key, name, next_announce_time))
def check_duplicate(err):
if err.check(sqlite3.IntegrityError):
raise DuplicateStreamHashError(stream_hash)
return err
d.addErrback(check_duplicate)
return d
@rerun_if_locked
def _get_all_streams(self):
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
d = self.db_conn.runQuery("select stream_hash from live_streams")
d.addCallback(lambda results: [r[0] for r in results])
return d
@rerun_if_locked
def _get_stream_info(self, stream_hash):
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
d = self.db_conn.runQuery("select public_key, key, stream_name from live_streams where stream_hash = ?",
(stream_hash,))
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
return d
@rerun_if_locked
def _check_if_stream_exists(self, stream_hash):
try:
self.stream_info_db.Get(stream_hash)
return True
except KeyError:
return False
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
d.addCallback(lambda r: True if len(r) else False)
return d
@rerun_if_locked
def _get_streams_to_announce(self, next_announce_time):
# TODO: See if the following would be better for handling announce times:
# TODO: Have a separate db for them, and read the whole thing into memory
# TODO: on startup, and then write changes to db when they happen
stream_hashes = []
batch = leveldb.WriteBatch()
current_time = time.time()
for stream_hash, stream_info in self.stream_info_db.RangeIter():
public_key, key, name, announce_time = json.loads(stream_info)
if announce_time < current_time:
batch.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)))
stream_hashes.append(stream_hash)
self.stream_info_db.Write(batch, sync=True)
return stream_hashes
def get_and_update(transaction):
timestamp = time.time()
r = transaction.execute("select stream_hash from live_streams where" +
" (next_announce_time is null or next_announce_time < ?) " +
" and stream_hash is not null", (timestamp, ))
s_hs = [s_h for s_h, in r.fetchall()]
transaction.execute("update live_streams set next_announce_time = ? where " +
" (next_announce_time is null or next_announce_time < ?)",
(next_announce_time, timestamp))
return s_hs
return self.db_conn.runInteraction(get_and_update)
@rerun_if_locked
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
d = self.db_conn.runQuery("select position from live_stream_blobs where stream_hash = ? and blob_hash = ?",
(stream_hash, blob_hash))
d.addCallback(lambda r: r[0][0] if len(r) else None)
return d
@rerun_if_locked
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
blob_infos = []
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if stream_hash == s_h:
position, revision, iv, length, signature = json.loads(blob_info)
if (start_num is None) or (position > start_num):
if (end_num is None) or (position < end_num):
blob_infos.append((b_h, position, revision, iv, length, signature))
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
params = []
q_string = "select * from ("
q_string += " select blob_hash, position, revision, iv, length, signature from live_stream_blobs "
q_string += " where stream_hash = ? "
params.append(stream_hash)
if start_num is not None:
q_string += " and position > ? "
params.append(start_num)
if end_num is not None:
q_string += " and position < ? "
params.append(end_num)
q_string += " order by position "
if reverse is True:
q_string += " DESC "
if count is not None:
blob_infos = blob_infos[:count]
return blob_infos
q_string += " limit ? "
params.append(count)
q_string += ") order by position"
# Order by position is done twice so that it always returns them from lowest position to
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
return self.db_conn.runQuery(q_string, tuple(params))
@rerun_if_locked
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
batch = leveldb.WriteBatch()
for blob_info in blob_infos:
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
try:
self.stream_blob_db.Get(blob_hash_stream_hash)
if ignore_duplicate_error is False:
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
continue
except KeyError:
pass
batch.Put(blob_hash_stream_hash,
json.dumps((blob_info.blob_num,
blob_info.revision,
blob_info.iv,
blob_info.length,
blob_info.signature)))
self.stream_blob_db.Write(batch, sync=True)
def add_blobs(transaction):
for blob_info in blob_infos:
try:
transaction.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
blob_info.revision, blob_info.iv, blob_info.length,
blob_info.signature))
except sqlite3.IntegrityError:
if ignore_duplicate_error is False:
raise
return self.db_conn.runInteraction(add_blobs)
@rerun_if_locked
def _get_stream_of_blobhash(self, blob_hash):
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
b_h, s_h = json.loads(blob_hash_stream_hash)
if blob_hash == b_h:
return s_h
return None
d = self.db_conn.runQuery("select stream_hash from live_stream_blobs where blob_hash = ?",
(blob_hash,))
d.addCallback(lambda r: r[0][0] if len(r) else None)
return d
@rerun_if_locked
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
return self.db_conn.runQuery("insert into live_stream_descriptors values (?, ?)",
(sd_blob_hash, stream_hash))
@rerun_if_locked
def _get_sd_blob_hashes_for_stream(self, stream_hash):
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
d = self.db_conn.runQuery("select sd_blob_hash from live_stream_descriptors where stream_hash = ?",
(stream_hash,))
d.addCallback(lambda results: [r[0] for r in results])
return d
class TempLiveStreamMetadataManager(DHTHashSupplier):
+13 -7
View File
@@ -2,17 +2,20 @@ import binascii
import logging
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
from twisted.internet import defer, threads
from lbrynet.core.Error import DuplicateStreamHashError
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
from lbrynet.interfaces import IStreamDescriptorValidator
from zope.interface import implements
log = logging.getLogger(__name__)
LiveStreamType = "lbrylive"
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
logging.debug("Saving info for %s", str(sd_info['stream_name']))
log.debug("Saving info for %s", str(sd_info['stream_name']))
hex_stream_name = sd_info['stream_name']
public_key = sd_info['public_key']
key = sd_info['key']
@@ -30,7 +33,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
iv = blob['iv']
signature = blob['signature']
crypt_blobs.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
log.debug("Trying to save stream info for %s", str(hex_stream_name))
d = stream_info_manager.save_stream(stream_hash, public_key, hex_stream_name,
key, crypt_blobs)
@@ -88,7 +91,7 @@ class LBRYLiveStreamDescriptorValidator(object):
self.raw_info = raw_info
def validate(self):
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
hex_stream_name = self.raw_info['stream_name']
public_key = self.raw_info['public_key']
key = self.raw_info['key']
@@ -98,7 +101,7 @@ class LBRYLiveStreamDescriptorValidator(object):
h.update(public_key)
h.update(key)
if h.hexdigest() != stream_hash:
raise ValueError("Stream hash does not match stream metadata")
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
blobs = self.raw_info['blobs']
def check_blob_signatures():
@@ -121,11 +124,14 @@ class LBRYLiveStreamDescriptorValidator(object):
hashsum.update(iv)
hashsum.update(str(length))
if not verify_signature(hashsum.digest(), signature, public_key):
raise ValueError("Invalid signature in stream descriptor")
raise InvalidStreamDescriptorError("Invalid signature in stream descriptor")
return threads.deferToThread(check_blob_signatures)
def info_to_show(self):
info = []
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
return info
return info
def get_length_of_stream(self):
return None
+30 -28
View File
@@ -1,5 +1,5 @@
import binascii
from lbrynet.core.DownloadOption import DownloadOption
from lbrynet.core.StreamDescriptor import StreamMetadata
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
from zope.interface import implements
from lbrynet.lbrylive.client.LiveStreamMetadataHandler import LiveStreamMetadataHandler
@@ -9,6 +9,7 @@ from lbrynet.lbrylive.StreamDescriptor import save_sd_info
from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
from twisted.internet import defer, threads # , process
from lbrynet.interfaces import IStreamDownloaderFactory
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType
class LiveStreamDownloader(CryptStreamDownloader):
@@ -85,9 +86,9 @@ class FullLiveStreamDownloader(LiveStreamDownloader):
d.addCallback(lambda _: set_file_name_if_unset())
return d
def stop(self):
def stop(self, err=None):
d = self._close_file()
d.addBoth(lambda _: LiveStreamDownloader.stop(self))
d.addBoth(lambda _: LiveStreamDownloader.stop(self, err))
return d
def _start(self):
@@ -138,34 +139,24 @@ class FullLiveStreamDownloaderFactory(object):
self.wallet = wallet
self.default_payment_rate_manager = default_payment_rate_manager
def get_downloader_options(self, sd_validator, payment_rate_manager):
options = [
DownloadOption(
[float, None],
"rate which will be paid for data (None means use application default)",
"data payment rate",
None
),
DownloadOption(
[float, None],
"rate which will be paid for metadata (None means use application default)",
"metadata payment rate",
None
),
DownloadOption(
[bool],
"allow reuploading data downloaded for this file",
"allow upload",
True
),
]
return options
def can_download(self, sd_validator):
return True
def make_downloader(self, sd_validator, options, payment_rate_manager):
def make_downloader(self, metadata, options, payment_rate_manager):
# TODO: check options for payment rate manager parameters
payment_rate_manager = LiveStreamPaymentRateManager(self.default_payment_rate_manager,
payment_rate_manager)
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
def save_source_if_blob(stream_hash):
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
else:
d = defer.succeed(True)
d.addCallback(lambda _: stream_hash)
return d
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
d.addCallback(save_source_if_blob)
def create_downloader(stream_hash):
stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter,
@@ -177,4 +168,15 @@ class FullLiveStreamDownloaderFactory(object):
return d
d.addCallback(create_downloader)
return d
return d
def add_full_live_stream_downloader_to_sd_identifier(session, stream_info_manager, sd_identifier,
base_live_stream_payment_rate_manager):
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
session.rate_limiter,
session.blob_manager,
stream_info_manager,
session.wallet,
base_live_stream_payment_rate_manager)
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
@@ -9,7 +9,10 @@ from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
from lbrynet.interfaces import IRequestCreator, IMetadataHandler
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
from lbrynet.core.Error import NoResponseError
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
log = logging.getLogger(__name__)
class LiveStreamMetadataHandler(object):
@@ -79,7 +82,7 @@ class LiveStreamMetadataHandler(object):
r = None
if self._finished_discovery() is False:
r = self.stream_hash
logging.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
log.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
return defer.succeed(r)
def _find_peers_for_hash(self, h):
@@ -101,7 +104,7 @@ class LiveStreamMetadataHandler(object):
if blob_hash is not None:
infos.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
else:
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
self._final_blob_num = blob_num - 1
return infos
@@ -134,7 +137,7 @@ class LiveStreamMetadataHandler(object):
further_blobs_request['count'] = count
else:
further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST
logging.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
r_dict = {'further_blobs': further_blobs_request}
response_identifier = 'further_blobs'
request = ClientPaidRequest(r_dict, response_identifier, further_blobs_request['count'])
@@ -142,7 +145,7 @@ class LiveStreamMetadataHandler(object):
return None
def _get_discovery_params(self):
logging.debug("In _get_discovery_params")
log.debug("In _get_discovery_params")
stream_position = self.download_manager.stream_position()
blobs = self.download_manager.blobs
if blobs:
@@ -153,7 +156,7 @@ class LiveStreamMetadataHandler(object):
if final_blob_num is not None:
last_blob_num = final_blob_num
if self.download_whole is False:
logging.debug("download_whole is False")
log.debug("download_whole is False")
if final_blob_num is not None:
for i in xrange(stream_position, final_blob_num + 1):
if not i in blobs:
@@ -171,7 +174,7 @@ class LiveStreamMetadataHandler(object):
return self.stream_hash, blobs[last_blob_num].blob_hash, 'end', -1 * self.max_before_skip_ahead
else:
return self.stream_hash, None, 'end', -1 * self.max_before_skip_ahead
logging.debug("download_whole is True")
log.debug("download_whole is True")
beginning = None
end = None
for i in xrange(stream_position, last_blob_num + 1):
@@ -187,17 +190,17 @@ class LiveStreamMetadataHandler(object):
break
if beginning is None:
if final_blob_num is not None:
logging.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
str(last_blob_num + 1))
log.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
str(last_blob_num + 1))
return None
else:
logging.debug("Discovery is not finished. final blob num is unknown.")
log.debug("Discovery is not finished. final blob num is unknown.")
if last_blob_num != -1:
return self.stream_hash, blobs[last_blob_num].blob_hash, None, None
else:
return self.stream_hash, 'beginning', None, None
else:
logging.info("Discovery is not finished. Not all blobs are known.")
log.info("Discovery is not finished. Not all blobs are known.")
return self.stream_hash, beginning, end, None
def _price_settled(self, protocol):
@@ -244,7 +247,7 @@ class LiveStreamMetadataHandler(object):
if response == "RATE_ACCEPTED":
return True
else:
logging.info("Rate offer has been rejected by %s", str(peer))
log.info("Rate offer has been rejected by %s", str(peer))
del self._protocol_prices[protocol]
self._price_disagreements.append(peer)
return True
@@ -263,8 +266,8 @@ class LiveStreamMetadataHandler(object):
if not 'blob_infos' in response:
return InvalidResponseError("Missing the required field 'blob_infos'")
raw_blob_infos = response['blob_infos']
logging.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
logging.debug("blobs: %s", str(raw_blob_infos))
log.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
log.debug("blobs: %s", str(raw_blob_infos))
for raw_blob_info in raw_blob_infos:
length = raw_blob_info['length']
if length != 0:
@@ -276,10 +279,10 @@ class LiveStreamMetadataHandler(object):
iv = raw_blob_info['iv']
signature = raw_blob_info['signature']
blob_info = LiveBlobInfo(blob_hash, num, length, iv, revision, signature)
logging.debug("Learned about a potential blob: %s", str(blob_hash))
log.debug("Learned about a potential blob: %s", str(blob_hash))
if self._verify_blob(blob_info):
if blob_hash is None:
logging.info("Setting _final_blob_num to %s", str(num - 1))
log.info("Setting _final_blob_num to %s", str(num - 1))
self._final_blob_num = num - 1
else:
blob_infos.append(blob_info)
@@ -289,7 +292,7 @@ class LiveStreamMetadataHandler(object):
def add_blobs_to_download_manager():
blob_nums = [b.blob_num for b in blob_infos]
logging.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
log.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
self.download_manager.add_blobs_to_download(blob_infos)
d.addCallback(lambda _: add_blobs_to_download_manager())
@@ -308,12 +311,12 @@ class LiveStreamMetadataHandler(object):
return d
def _verify_blob(self, blob):
logging.debug("Got an unverified blob to check:")
logging.debug("blob_hash: %s", blob.blob_hash)
logging.debug("blob_num: %s", str(blob.blob_num))
logging.debug("revision: %s", str(blob.revision))
logging.debug("iv: %s", blob.iv)
logging.debug("length: %s", str(blob.length))
log.debug("Got an unverified blob to check:")
log.debug("blob_hash: %s", blob.blob_hash)
log.debug("blob_num: %s", str(blob.blob_num))
log.debug("revision: %s", str(blob.revision))
log.debug("iv: %s", blob.iv)
log.debug("length: %s", str(blob.length))
hashsum = get_lbry_hash_obj()
hashsum.update(self.stream_hash)
if blob.length != 0:
@@ -322,12 +325,12 @@ class LiveStreamMetadataHandler(object):
hashsum.update(str(blob.revision))
hashsum.update(blob.iv)
hashsum.update(str(blob.length))
logging.debug("hexdigest to be verified: %s", hashsum.hexdigest())
log.debug("hexdigest to be verified: %s", hashsum.hexdigest())
if verify_signature(hashsum.digest(), blob.signature, self.stream_pub_key):
logging.debug("Blob info is valid")
log.debug("Blob info is valid")
return True
else:
logging.debug("The blob info is invalid")
log.debug("The blob info is invalid")
return False
def _request_failed(self, reason, peer):
@@ -335,8 +338,9 @@ class LiveStreamMetadataHandler(object):
return
if reason.check(NoResponseError):
self._incompatible_peers.append(peer)
return
logging.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
log.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
self._update_local_score(peer, -5.0)
peer.update_score(-10.0)
if reason.check(ConnectionClosedBeforeResponseError):
return
return reason
@@ -0,0 +1,73 @@
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType, LBRYLiveStreamDescriptorValidator
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
def add_live_stream_to_sd_identifier(sd_identifier, base_live_stream_payment_rate_manager):
sd_identifier.add_stream_type(LiveStreamType, LBRYLiveStreamDescriptorValidator,
LiveStreamOptions(base_live_stream_payment_rate_manager))
class LiveStreamOptions(object):
def __init__(self, base_live_stream_payment_rate_manager):
self.base_live_stream_prm = base_live_stream_payment_rate_manager
def get_downloader_options(self, sd_validator, payment_rate_manager):
prm = payment_rate_manager
def get_default_data_rate_description():
if prm.min_blob_data_payment_rate is None:
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
else:
return "%f LBC/MB" % prm.min_blob_data_payment_rate
options = [
DownloadOption(
[
DownloadOptionChoice(None,
"No change",
"No change"),
DownloadOptionChoice(None,
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
"Default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)),
DownloadOptionChoice(float,
"Rate in LBC/MB",
"Rate in LBC/MB")
],
"rate which will be paid for data",
"data payment rate",
prm.min_blob_data_payment_rate,
get_default_data_rate_description()
),
DownloadOption(
[
DownloadOptionChoice(None,
"No change",
"No change"),
DownloadOptionChoice(None,
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate),
"Default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)),
DownloadOptionChoice(float,
"Rate in LBC/MB",
"Rate in LBC/MB")
],
"rate which will be paid for metadata",
"metadata payment rate",
None,
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)
),
DownloadOption(
[
DownloadOptionChoice(True,
"Allow reuploading data downloaded for this file",
"Allow reuploading"),
DownloadOptionChoice(False,
"Disallow reuploading data downloaded for this file",
"Disallow reuploading")
],
"allow reuploading data downloaded for this file",
"allow upload",
True,
"Allow"
),
]
return options
@@ -3,6 +3,9 @@ from lbrynet.core.client.StreamProgressManager import StreamProgressManager
from twisted.internet import defer
log = logging.getLogger(__name__)
class LiveStreamProgressManager(StreamProgressManager):
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False,
download_whole=True, max_before_skip_ahead=5):
@@ -52,9 +55,9 @@ class LiveStreamProgressManager(StreamProgressManager):
return
blobs = self.download_manager.blobs
logging.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
log.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
if blobs:
logging.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
log.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
if self.outputting_d is None:
self.outputting_d = defer.Deferred()
@@ -71,15 +74,15 @@ class LiveStreamProgressManager(StreamProgressManager):
reactor.callLater(0, self._output_loop)
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
logging.info("Outputting blob %s", str(current_blob_num))
log.info("Outputting blob %s", str(current_blob_num))
self.provided_blob_nums.append(current_blob_num)
d = self.download_manager.handle_blob(current_blob_num)
d.addCallback(lambda _: finished_outputting_blob())
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
elif blobs and max(blobs.iterkeys()) > self.last_blob_outputted + self.max_before_skip_ahead - 1:
self.last_blob_outputted += 1
logging.info("Skipping blob number %s due to knowing about blob number %s",
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
log.info("Skipping blob number %s due to knowing about blob number %s",
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
self._finished_with_blob(current_blob_num)
reactor.callLater(0, self._output_loop)
else:
@@ -4,6 +4,9 @@ from zope.interface import implements
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
log = logging.getLogger(__name__)
class CryptBlobInfoQueryHandlerFactory(object):
implements(IQueryHandlerFactory)
@@ -54,7 +57,7 @@ class CryptBlobInfoQueryHandler(object):
if self.query_identifiers[1] in queries:
further_blobs_request = queries[self.query_identifiers[1]]
logging.debug("Received the client's request for additional blob information")
log.debug("Received the client's request for additional blob information")
if self.blob_info_payment_rate is None:
response['further_blobs'] = {'error': 'RATE_UNSET'}
@@ -62,7 +65,7 @@ class CryptBlobInfoQueryHandler(object):
def count_and_charge(blob_infos):
if len(blob_infos) != 0:
logging.info("Responding with %s infos", str(len(blob_infos)))
log.info("Responding with %s infos", str(len(blob_infos)))
expected_payment = 1.0 * len(blob_infos) * self.blob_info_payment_rate / 1000.0
self.wallet.add_expected_payment(self.peer, expected_payment)
self.peer.update_stats('uploaded_crypt_blob_infos', len(blob_infos))
+160 -44
View File
@@ -1,60 +1,176 @@
from twisted.protocols import basic
from twisted.internet import defer
import logging
log = logging.getLogger(__name__)
class ConsoleControl(basic.LineReceiver):
from os import linesep as delimiter
def __init__(self, control_handlers):
self.control_handlers = {}
self.categories = {}
categories = set([category for category, handler in control_handlers])
prompt_number = 0
for category in categories:
self.categories[prompt_number] = category
for handler in [handler for cat, handler in control_handlers if cat == category]:
self.control_handlers[prompt_number] = handler
prompt_number += 1
def __init__(self):
self.connected = False
self.buffer = []
self.command_handlers = {}
self.current_handler = None
def start(self, command_handlers):
self.command_handlers = {h.command: h for h in command_handlers}
self.current_handler = None
self.send_initial_prompt()
return defer.succeed(True)
def connectionMade(self):
self.connected = True
if self.buffer:
self.send(self.buffer)
self.buffer = []
def send_initial_prompt(self):
self.sendLine("")
self.sendLine("You should have received 1000 LBC the first time you ran\n"
"this program. If you did not, let us know! But first give\n"
"them a moment to show up. The block time is currently 30\n"
"seconds so they should show up within a couple of minutes.\n\n"
"Welcome to lbrynet-console!")
self.sendLine("")
self.sendLine("Enter a command. Try 'get wonderfullife' or 'help' to see more options.")
self.show_prompt()
def send(self, s):
self.transport.write(s)
def write(self, s):
if self.connected is False:
self.buffer.append(s)
else:
self.send(s)
def flush(self):
if self.connected is True and self.buffer:
self.send(self.buffer)
self.buffer = []
def show_prompt(self):
self.send("> ")
def show_quick_help(self):
self.sendLine("Available commands:")
self.sendLine("")
showed_help_all = False
sorted_handlers = sorted(self.command_handlers.items(), key=lambda x: x[0])
sorted_handlers = sorted(sorted_handlers, key=lambda x: x[1].priority, reverse=True)
for command, handler in sorted_handlers:
if handler.priority > 0:
if showed_help_all is False and handler.priority < 10:
self.sendLine("help-all - Show the full list of available commands")
showed_help_all = True
self.sendLine(command + " - " + handler.short_help)
self.sendLine("")
self.sendLine("For more information about any command type 'help <command>'")
def show_full_help(self):
self.sendLine("Available commands:")
self.sendLine("")
for command, handler in sorted(self.command_handlers.items(), key=lambda x: x[0]):
self.sendLine(command + " - " + handler.short_help)
self.sendLine("")
self.sendLine("For more information about any command type 'help <command>'")
def handler_done(self):
self.current_handler = None
self.show_prompt()
def handler_failed(self, err):
self.current_handler = None
self.sendLine("An error occurred:")
self.sendLine(err.getTraceback())
self.show_prompt()
def lineReceived(self, line):
def show_response(response):
if response is not None:
self.sendLine(response)
def show_error(err):
self.sendLine(err.getTraceback())
if not self.command_handlers:
return
if self.current_handler is None:
words = line.split()
if len(words) == 0:
self.show_prompt()
return
command, args = words[0], words[1:]
if command == "help":
if len(args) == 0:
self.show_quick_help()
self.show_prompt()
return
if args[0] in self.command_handlers:
self.sendLine(self.command_handlers[args[0]].full_help)
self.show_prompt()
return
if args[0] == "help-all":
self.sendLine("Show the full list of available commands!")
self.show_prompt()
return
self.sendLine("Can't help you with '%s'. Sorry!" % args[0])
self.show_prompt()
return
elif command == "help-all":
self.show_full_help()
self.show_prompt()
return
if command in self.command_handlers:
command_handler = self.command_handlers[command]
else:
candidates = [k for k in self.command_handlers.keys() if k.startswith(command)]
if len(candidates) == 0:
self.sendLine("Unknown command. Type 'help' for a list of commands.")
self.show_prompt()
return
if len(candidates) >= 2:
l = "Ambiguous command. Matches: "
for candidate in candidates:
l += candidate
l += ", "
l = l[:-2]
l += l
self.sendLine(l)
self.show_prompt()
return
else:
command_handler = self.command_handlers[candidates[0]]
try:
num = int(line)
except ValueError:
num = None
if num in self.control_handlers:
self.current_handler = self.control_handlers[num].get_handler()
line = None
if self.current_handler is not None:
try:
r = self.current_handler.handle_line(line)
done, ds = r[0], [d for d in r[1:] if d is not None]
except Exception as e:
done = True
ds = [defer.fail(e)]
if done is True:
self.current_handler = command_handler.get_handler(self)
except:
self.current_handler = None
map(lambda d: d.addCallbacks(show_response, show_error), ds)
if self.current_handler is None:
self.show_prompt()
def show_prompt(self):
self.sendLine("Options:")
for num, handler in self.control_handlers.iteritems():
if num in self.categories:
self.sendLine("")
self.sendLine(self.categories[num])
self.sendLine("")
self.sendLine("[" + str(num) + "] " + handler.get_prompt_description())
import traceback
self.sendLine(traceback.format_exc())
log.error(traceback.format_exc())
self.show_prompt()
return
try:
self.current_handler.start(*args)
except TypeError:
self.current_handler = None
self.sendLine("Invalid arguments. Type 'help <command>' for the argument list.")
import traceback
log.error(traceback.format_exc())
self.show_prompt()
return
except:
self.current_handler = None
import traceback
self.sendLine(traceback.format_exc())
log.error(traceback.format_exc())
self.show_prompt()
return
self.current_handler.finished_deferred.addCallbacks(lambda _: self.handler_done(),
self.handler_failed)
else:
try:
self.current_handler.handle_line(line)
except Exception as e:
self.current_handler = None
import traceback
self.sendLine(traceback.format_exc())
log.error(traceback.format_exc())
self.show_prompt()
return
File diff suppressed because it is too large. Load diff
+303 -131
View File
@@ -1,13 +1,22 @@
import logging
from lbrynet.core.Session import LBRYSession
import os.path
import argparse
import requests
import locale
import sys
import webbrowser
if sys.platform == "darwin":
from appdirs import user_data_dir
from yapsy.PluginManager import PluginManager
from twisted.internet import defer, threads, stdio, task
from twisted.internet import defer, threads, stdio, task, error
from jsonrpc.proxy import JSONRPCProxy
from lbrynet.core.Session import LBRYSession
from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE # , MIN_BLOB_INFO_PAYMENT_RATE
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, API_CONNECTION_STRING # , MIN_BLOB_INFO_PAYMENT_RATE
from lbrynet.core.utils import generate_id
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.core.PaymentRateManager import PaymentRateManager
@@ -15,11 +24,12 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
from lbrynet.core.PTCWallet import PTCWallet
from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileOpenerFactory
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager
#from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory
from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory
from lbrynet.lbrynet_console.ControlHandlers import LBRYFileStatusFactory, DeleteLBRYFileChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import ToggleLBRYFileRunningChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory
@@ -28,17 +38,23 @@ from lbrynet.lbrynet_console.ControlHandlers import ShowPublishedSDHashesChooser
from lbrynet.lbrynet_console.ControlHandlers import CreatePlainStreamDescriptorChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import ShowLBRYFileStreamHashChooserFactory, AddStreamFromHashFactory
from lbrynet.lbrynet_console.ControlHandlers import AddStreamFromSDFactory, AddStreamFromLBRYcrdNameFactory
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory, GetNewWalletAddressFactory
from lbrynet.lbrynet_console.ControlHandlers import ShowServerStatusFactory, ModifyServerSettingsFactory
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory, StatusFactory
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory, PublishFactory
from lbrynet.lbrynet_console.ControlHandlers import BlockchainStatusFactory
from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet
log = logging.getLogger(__name__)
alert = logging.getLogger("lbryalert." + __name__)
class LBRYConsole():
"""A class which can upload and download file streams to and from the network"""
def __init__(self, peer_port, dht_node_port, known_dht_nodes, control_class, wallet_type, lbrycrd_rpc_port,
use_upnp, conf_dir, data_dir):
def __init__(self, peer_port, dht_node_port, known_dht_nodes, fake_wallet,
lbrycrd_conf, lbrycrd_dir, use_upnp, data_dir, created_data_dir,
lbrycrdd_path):
"""
@param peer_port: the network port on which to listen for peers
@@ -49,34 +65,49 @@ class LBRYConsole():
self.peer_port = peer_port
self.dht_node_port = dht_node_port
self.known_dht_nodes = known_dht_nodes
self.wallet_type = wallet_type
self.wallet_rpc_port = lbrycrd_rpc_port
self.fake_wallet = fake_wallet
self.lbrycrd_conf = lbrycrd_conf
self.lbrycrd_dir = lbrycrd_dir
if not self.lbrycrd_dir:
if sys.platform == "darwin":
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), "Library/Application Support/lbrycrd")
else:
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
if not self.lbrycrd_conf:
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
self.lbrycrdd_path = lbrycrdd_path
self.use_upnp = use_upnp
self.lbry_server_port = None
self.control_class = control_class
self.session = None
self.lbry_file_metadata_manager = None
self.lbry_file_manager = None
self.conf_dir = conf_dir
self.data_dir = data_dir
self.db_dir = data_dir
self.current_db_revision = 1
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
self.created_data_dir = created_data_dir
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces([
os.path.join(self.conf_dir, "plugins"),
os.path.join(self.db_dir, "plugins"),
os.path.join(os.path.dirname(__file__), "plugins"),
])
self.control_handlers = []
self.command_handlers = []
self.query_handlers = {}
self.settings = LBRYSettings(self.conf_dir)
self.settings = LBRYSettings(self.db_dir)
self.blob_request_payment_rate_manager = None
self.lbryid = None
self.sd_identifier = StreamDescriptorIdentifier()
self.plugin_objects = []
self.db_migration_revisions = None
def start(self):
"""Initialize the session and restore everything to its saved state"""
d = threads.deferToThread(self._create_directory)
d = self._setup_controller()
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
d.addCallback(lambda _: self._check_db_migration())
d.addCallback(lambda _: self._get_settings())
d.addCallback(lambda _: self._get_session())
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
d.addCallback(lambda _: self._setup_lbry_file_manager())
d.addCallback(lambda _: self._setup_lbry_file_opener())
d.addCallback(lambda _: self._setup_control_handlers())
@@ -84,17 +115,30 @@ class LBRYConsole():
d.addCallback(lambda _: self._load_plugins())
d.addCallback(lambda _: self._setup_server())
d.addCallback(lambda _: self._start_controller())
d.addErrback(self._show_start_error)
return d
def _show_start_error(self, error):
print error.getTraceback()
log.error("An error occurred during start up: %s", error.getTraceback())
return error
def _show_shutdown_error(self, error):
print error.getErrorMessage()
log.error("An error occurred during shutdown: %s", error.getTraceback())
return error
def shut_down(self):
"""Stop the session, all currently running streams, and stop the server"""
d = self.session.shut_down()
d.addCallback(lambda _: self._shut_down())
d = self._shut_down()
if self.session is not None:
d.addCallback(lambda _: self.session.shut_down())
d.addErrback(self._show_shutdown_error)
return d
def add_control_handlers(self, control_handlers):
for control_handler in control_handlers:
self.control_handlers.append(control_handler)
self.command_handlers.append(control_handler)
def add_query_handlers(self, query_handlers):
@@ -111,13 +155,40 @@ class LBRYConsole():
dl.addCallback(_set_query_handlers)
return dl
def _create_directory(self):
if not os.path.exists(self.conf_dir):
os.makedirs(self.conf_dir)
logging.debug("Created the configuration directory: %s", str(self.conf_dir))
if not os.path.exists(self.data_dir):
os.makedirs(self.data_dir)
logging.debug("Created the data directory: %s", str(self.data_dir))
def _setup_data_directory(self):
alert.info("Loading databases...")
if self.created_data_dir:
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
db_revision.write(str(self.current_db_revision))
db_revision.close()
log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision")))
if not os.path.exists(self.blobfile_dir):
os.mkdir(self.blobfile_dir)
log.debug("Created the blobfile directory: %s", str(self.blobfile_dir))
def _check_db_migration(self):
old_revision = 0
db_revision_file = os.path.join(self.db_dir, "db_revision")
if os.path.exists(db_revision_file):
old_revision = int(open(db_revision_file).read().strip())
if old_revision < self.current_db_revision:
from lbrynet.db_migrator import dbmigrator
print "Upgrading your databases..."
d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision)
def print_success(old_dirs):
success_string = "Finished upgrading the databases. It is now safe to delete the"
success_string += " following directories, if you feel like it. It won't make any"
success_string += " difference.\nAnyway here they are: "
for i, old_dir in enumerate(old_dirs):
success_string += old_dir
if i + 1 < len(old_dir):
success_string += ", "
print success_string
d.addCallback(print_success)
return d
return defer.succeed(True)
def _get_settings(self):
d = self.settings.start()
@@ -137,28 +208,96 @@ class LBRYConsole():
return d
def _get_session(self):
d = self.settings.get_default_data_payment_rate()
def get_default_data_rate():
d = self.settings.get_default_data_payment_rate()
d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE})
return d
def create_session(default_data_payment_rate):
if default_data_payment_rate is None:
default_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
if self.wallet_type == "lbrycrd":
wallet = LBRYcrdWallet("rpcuser", "rpcpassword", "127.0.0.1", self.wallet_rpc_port)
def get_wallet():
if self.fake_wallet:
d = defer.succeed(PTCWallet(self.db_dir))
elif self.lbrycrdd_path is not None:
d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir,
wallet_conf=self.lbrycrd_conf,
lbrycrdd_path=self.lbrycrdd_path))
else:
wallet = PTCWallet(self.conf_dir)
self.session = LBRYSession(default_data_payment_rate, db_dir=self.conf_dir, lbryid=self.lbryid,
blob_dir=self.data_dir, dht_node_port=self.dht_node_port,
d = defer.succeed(LBRYumWallet(self.db_dir))
d.addCallback(lambda wallet: {"wallet": wallet})
return d
d1 = get_default_data_rate()
d2 = get_wallet()
def combine_results(results):
r = {}
for success, result in results:
if success is True:
r.update(result)
return r
def create_session(results):
alert.info("Databases loaded.")
self.session = LBRYSession(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port,
use_upnp=self.use_upnp, wallet=wallet)
use_upnp=self.use_upnp, wallet=results['wallet'])
d.addCallback(create_session)
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
d.addCallback(lambda _: self.session.setup())
dl.addCallback(combine_results)
dl.addCallback(create_session)
dl.addCallback(lambda _: self.session.setup())
dl.addCallback(lambda _: self.check_first_run())
dl.addCallback(self._show_first_run_result)
return dl
def check_first_run(self):
d = self.session.wallet.is_first_run()
d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else 0.0)
return d
def _do_first_run(self):
d = self.session.wallet.get_new_address()
def send_request(url, data):
r = requests.post(url, json=data)
if r.status_code == 200:
return r.json()['credits_sent']
return 0.0
def log_error(err):
log.warning("unable to request free credits. %s", err.getErrorMessage())
return 0.0
def request_credits(address):
url = "http://credreq.lbry.io/requestcredits"
data = {"address": address}
d = threads.deferToThread(send_request, url, data)
d.addErrback(log_error)
return d
d.addCallback(request_credits)
return d
@staticmethod
def _show_first_run_result(credits_received):
if credits_received != 0.0:
points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),),
grouping=True)
alert.info("\n\nThank you for testing the alpha version of LBRY!\n\n"
"You have been given %s for free because we love you.\n"
"Please give them a few minutes to show up while you\n"
"catch up with our blockchain.\n", points_string)
def _setup_lbry_file_manager(self):
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.conf_dir)
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
d = self.lbry_file_metadata_manager.setup()
def set_lbry_file_manager():
@@ -179,59 +318,46 @@ class LBRYConsole():
def _setup_control_handlers(self):
handlers = [
('General',
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node)),
('General',
GetWalletBalancesFactory(self.session.wallet)),
('General',
ModifyApplicationDefaultsFactory(self)),
('General',
ShutDownFactory(self)),
('General',
PeerStatsAndSettingsChooserFactory(self.session.peer_manager)),
('lbryfile',
LBRYFileStatusFactory(self.lbry_file_manager)),
('Stream Downloading',
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager)),
('lbryfile',
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
self.lbry_file_manager)),
('lbryfile',
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager)),
('lbryfile',
CreateLBRYFileFactory(self.session, self.lbry_file_manager)),
('lbryfile',
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
self.session.blob_manager,
self.lbry_file_manager)),
('lbryfile',
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
self.lbry_file_manager)),
('lbryfile',
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager)),
('lbryfile',
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager)),
('lbryfile',
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager)),
('Stream Downloading',
AddStreamFromHashFactory(self.sd_identifier, self.session))
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node),
GetWalletBalancesFactory(self.session.wallet),
ModifyApplicationDefaultsFactory(self),
ShutDownFactory(self),
PeerStatsAndSettingsChooserFactory(self.session.peer_manager),
LBRYFileStatusFactory(self.lbry_file_manager),
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager,
self.session.wallet),
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
self.lbry_file_manager),
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager),
CreateLBRYFileFactory(self.session, self.lbry_file_manager),
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
self.session.blob_manager),
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
self.lbry_file_manager),
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager),
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager),
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager),
AddStreamFromHashFactory(self.sd_identifier, self.session, self.session.wallet),
StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager,
self.session.blob_manager, self.session.wallet if not self.fake_wallet else None),
ImmediateAnnounceAllBlobsFactory(self.session.blob_manager)
]
self.add_control_handlers(handlers)
if self.wallet_type == 'lbrycrd':
if not self.fake_wallet:
lbrycrd_handlers = [
('Stream Downloading',
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
self.session.wallet)),
('General',
ClaimNameFactory(self.session.wallet)),
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
self.session.wallet),
ClaimNameFactory(self.session.wallet, self.lbry_file_manager,
self.session.blob_manager),
GetNewWalletAddressFactory(self.session.wallet),
PublishFactory(self.session, self.lbry_file_manager, self.session.wallet),
BlockchainStatusFactory(self.session.wallet)
]
self.add_control_handlers(lbrycrd_handlers)
if self.peer_port is not None:
server_handlers = [
('Server',
ShowServerStatusFactory(self)),
('Server',
ModifyServerSettingsFactory(self)),
ShowServerStatusFactory(self),
ModifyServerSettingsFactory(self),
]
self.add_control_handlers(server_handlers)
@@ -265,12 +391,19 @@ class LBRYConsole():
def setup_plugins():
ds = []
for plugin in self.plugin_manager.getAllPlugins():
self.plugin_objects.append(plugin.plugin_object)
ds.append(plugin.plugin_object.setup(self))
return defer.DeferredList(ds)
d.addCallback(lambda _: setup_plugins())
return d
def _stop_plugins(self):
ds = []
for plugin_object in self.plugin_objects:
ds.append(defer.maybeDeferred(plugin_object.stop))
return defer.DeferredList(ds)
def _setup_server(self):
def restore_running_status(running):
@@ -290,7 +423,12 @@ class LBRYConsole():
self.query_handlers,
self.session.peer_manager)
from twisted.internet import reactor
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
try:
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. %s", self.peer_port, traceback.format_exc())
raise ValueError("%s lbrynet may already be running on your computer.", str(e))
return defer.succeed(True)
def stop_server(self):
@@ -300,26 +438,35 @@ class LBRYConsole():
else:
return defer.succeed(True)
def _start_controller(self):
self.control_class(self.control_handlers)
def _setup_controller(self):
self.controller = ConsoleControl()
stdio.StandardIO(self.controller)
logger = logging.getLogger()
formatter = logging.Formatter("%(message)s")
alert_handler = logging.StreamHandler(self.controller)
alert_handler.setFormatter(formatter)
alert_handler.addFilter(logging.Filter("lbryalert"))
alert_handler.setLevel(logging.DEBUG)
logger.addHandler(alert_handler)
return defer.succeed(True)
def _start_controller(self):
return self.controller.start(self.command_handlers)
def _shut_down(self):
self.plugin_manager = None
d1 = self.lbry_file_metadata_manager.stop()
d1.addCallback(lambda _: self.lbry_file_manager.stop())
d2 = self.stop_server()
dl = defer.DeferredList([d1, d2])
ds = []
if self.lbry_file_metadata_manager is not None:
d = self.lbry_file_metadata_manager.stop()
d.addCallback(lambda _: self.lbry_file_manager.stop())
ds.append(d)
ds.append(self.stop_server())
ds.append(self._stop_plugins())
dl = defer.DeferredList(ds)
return dl
class StdIOControl():
def __init__(self, control_handlers):
stdio.StandardIO(ConsoleControl(control_handlers))
def launch_lbry_console():
from twisted.internet import reactor
parser = argparse.ArgumentParser(description="Launch a lbrynet console")
@@ -335,12 +482,9 @@ def launch_lbry_console():
parser.add_argument("--dht_node_port",
help="The port on which the console will listen for DHT connections.",
type=int, default=4444)
parser.add_argument("--wallet_type",
help="Either 'lbrycrd' or 'ptc'.",
type=str, default="lbrycrd")
parser.add_argument("--lbrycrd_wallet_rpc_port",
help="The rpc port on which the LBRYcrd wallet is listening",
type=int, default=8332)
parser.add_argument("--fake_wallet",
help="Testing purposes only. Use a non-blockchain wallet.",
action="store_true")
parser.add_argument("--no_dht_bootstrap",
help="Don't try to connect to the DHT",
action="store_true")
@@ -352,16 +496,22 @@ def launch_lbry_console():
help="The port of a known DHT node, to be used to bootstrap into the DHT. Must "
"be used with --dht_bootstrap_host",
type=int, default=4000)
parser.add_argument("--use_upnp",
help="Try to use UPnP to enable incoming connections through the firewall",
parser.add_argument("--disable_upnp",
help="Don't try to use UPnP to enable incoming connections through the firewall",
action="store_true")
parser.add_argument("--conf_dir",
help=("The full path to the directory in which to store configuration "
"options and user added plugins. Default: ~/.lbrynet"),
type=str)
parser.add_argument("--data_dir",
help=("The full path to the directory in which to store data chunks "
"downloaded from lbrynet. Default: <conf_dir>/blobfiles"),
help=("The full path to the directory in which lbrynet data and metadata will be stored. "
"Default: ~/.lbrynet on linux, ~/Library/Application Support/lbrynet on OS X"),
type=str)
parser.add_argument("--lbrycrdd_path",
help="The path to lbrycrdd, which will be launched if it isn't running. If"
"this option is chosen, lbrycrdd will be used as the interface to the"
"blockchain. By default, a lightweight interface is used.")
parser.add_argument("--lbrycrd_wallet_dir",
help="The directory in which lbrycrd data will stored. Used if lbrycrdd is "
"launched by this application.")
parser.add_argument("--lbrycrd_wallet_conf",
help="The configuration file for the LBRYcrd wallet. Default: ~/.lbrycrd/lbrycrd.conf",
type=str)
args = parser.parse_args()
@@ -381,27 +531,49 @@ def launch_lbry_console():
else:
dht_node_port = args.dht_node_port
if not args.conf_dir:
conf_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
conf_dir = args.conf_dir
if not os.path.exists(conf_dir):
os.mkdir(conf_dir)
created_data_dir = False
if not args.data_dir:
data_dir = os.path.join(conf_dir, "blobfiles")
if sys.platform == "darwin":
data_dir = user_data_dir("LBRY")
else:
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
data_dir = args.data_dir
if not os.path.exists(data_dir):
os.mkdir(data_dir)
created_data_dir = True
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
logging.basicConfig(level=logging.DEBUG, filename=os.path.join(conf_dir, "console.log"),
format=log_format)
daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
try:
daemon.is_running()
log.info("Attempt to start lbrynet-console while lbrynet-daemon is running")
print "lbrynet-daemon is running, you must turn it off before using lbrynet-console"
print "If you're running the app, quit before starting lbrynet-console"
print "If you're running lbrynet-daemon in a terminal, run 'stop-lbrynet-daemon' to turn it off"
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, StdIOControl, wallet_type=args.wallet_type,
lbrycrd_rpc_port=args.lbrycrd_wallet_rpc_port, use_upnp=args.use_upnp,
conf_dir=conf_dir, data_dir=data_dir)
webbrowser.open("http://localhost:5279")
d = task.deferLater(reactor, 0, console.start)
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
reactor.run()
except:
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
formatter = logging.Formatter(log_format)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler(os.path.join(data_dir, "console.log"))
file_handler.setFormatter(formatter)
file_handler.addFilter(logging.Filter("lbrynet"))
logger.addHandler(file_handler)
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, fake_wallet=args.fake_wallet,
lbrycrd_conf=args.lbrycrd_wallet_conf, lbrycrd_dir=args.lbrycrd_wallet_dir,
use_upnp=not args.disable_upnp, data_dir=data_dir,
created_data_dir=created_data_dir, lbrycrdd_path=args.lbrycrdd_path)
d = task.deferLater(reactor, 0, console.start)
d.addErrback(lambda _: reactor.stop())
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
reactor.run()
if __name__ == "__main__":
launch_lbry_console()
+3
View File
@@ -7,4 +7,7 @@ class LBRYPlugin(IPlugin):
IPlugin.__init__(self)
def setup(self, lbry_console):
raise NotImplementedError
def stop(self):
raise NotImplementedError
+55 -57
View File
@@ -1,60 +1,66 @@
import binascii
import functools
import json
import leveldb
import logging
import os
from twisted.internet import threads, defer
import unqlite
log = logging.getLogger(__name__)
def run_in_thread(fn):
@functools.wraps(fn)
def wrapped(*args, **kwargs):
return threads.deferToThread(fn, *args, **kwargs)
return wrapped
class LBRYSettings(object):
NAME = "settings.db"
def __init__(self, db_dir):
self.db_dir = db_dir
self.db = None
def start(self):
return threads.deferToThread(self._open_db)
return self._open_db()
def stop(self):
self.db.close()
self.db = None
return defer.succeed(True)
def _open_db(self):
logging.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "settings.db"))
filename = os.path.join(self.db_dir, self.NAME)
log.debug("Opening %s as the settings database", filename)
self.db = unqlite.UnQLite(filename)
return defer.succeed(True)
@run_in_thread
def save_lbryid(self, lbryid):
self.db['lbryid'] = binascii.hexlify(lbryid)
self.db.commit()
def save_lbryid():
self.db.Put("lbryid", binascii.hexlify(lbryid), sync=True)
return threads.deferToThread(save_lbryid)
@run_in_thread
def get_lbryid(self):
if 'lbryid' in self.db:
return binascii.unhexlify(self.db['lbryid'])
else:
return None
def get_lbryid():
try:
return binascii.unhexlify(self.db.Get("lbryid"))
except KeyError:
return None
return threads.deferToThread(get_lbryid)
@run_in_thread
def get_server_running_status(self):
if 'server_running' in self.db:
return json.loads(self.db['server_running'])
else:
return True
def get_status():
try:
return json.loads(self.db.Get("server_running"))
except KeyError:
return True
return threads.deferToThread(get_status)
@run_in_thread
def save_server_running_status(self, running):
def save_status():
self.db.Put("server_running", json.dumps(running), sync=True)
return threads.deferToThread(save_status)
self.db['server_running'] = json.dumps(running)
self.db.commit()
def get_default_data_payment_rate(self):
return self._get_payment_rate("default_data_payment_rate")
@@ -74,35 +80,27 @@ class LBRYSettings(object):
def save_server_crypt_info_payment_rate(self, rate):
return self._save_payment_rate("server_crypt_info_payment_rate", rate)
@run_in_thread
def _get_payment_rate(self, rate_type):
if rate_type in self.db:
return json.loads(self.db[rate_type])
else:
return None
def get_rate():
try:
return json.loads(self.db.Get(rate_type))
except KeyError:
return None
return threads.deferToThread(get_rate)
@run_in_thread
def _save_payment_rate(self, rate_type, rate):
if rate is not None:
self.db[rate_type] = json.dumps(rate)
elif rate_type in self.db:
del self.db[rate_type]
self.db.commit()
def save_rate():
if rate is not None:
self.db.Put(rate_type, json.dumps(rate), sync=True)
else:
self.db.Delete(rate_type, sync=True)
return threads.deferToThread(save_rate)
@run_in_thread
def get_query_handler_status(self, query_identifier):
def get_status():
try:
return json.loads(self.db.Get(json.dumps(('q_h', query_identifier))))
except KeyError:
return True
return threads.deferToThread(get_status)
if json.dumps(('q_h', query_identifier)) in self.db:
return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
else:
return True
def enable_query_handler(self, query_identifier):
return self._set_query_handler_status(query_identifier, True)
@@ -110,7 +108,7 @@ class LBRYSettings(object):
def disable_query_handler(self, query_identifier):
return self._set_query_handler_status(query_identifier, False)
@run_in_thread
def _set_query_handler_status(self, query_identifier, status):
def set_status():
self.db.Put(json.dumps(('q_h', query_identifier)), json.dumps(status), sync=True)
return threads.deferToThread(set_status)
self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
self.db.commit()
+10
View File
@@ -10,5 +10,15 @@ class IControlHandlerFactory(Interface):
class IControlHandler(Interface):
def handle_line(self, line):
pass
class ICommandHandlerFactory(Interface):
def get_handler(self):
pass
class ICommandHandler(Interface):
def handle_line(self, line):
pass
@@ -1,25 +1,43 @@
from twisted.internet import threads, defer
from twisted.internet import defer
from ValuableBlobInfo import ValuableBlobInfo
from db_keys import BLOB_INFO_TYPE
import json
import leveldb
import os
import sqlite3
from twisted.enterprise import adbapi
from lbrynet.core.sqlite_helpers import rerun_if_locked
class BlindInfoManager(object):
def __init__(self, db, peer_manager):
self.db = db
def __init__(self, db_dir, peer_manager):
self.db_dir = db_dir
self.db_conn = None
self.peer_manager = peer_manager
def setup(self):
return defer.succeed(True)
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_info.db"),
check_same_thread=False)
def set_up_table(transaction):
transaction.execute("create table if not exists valuable_blobs (" +
" blob_hash text primary key, " +
" blob_length integer, " +
" reference text, " +
" peer_host text, " +
" peer_port integer, " +
" peer_score text" +
")")
return self.db_conn.runInteraction(set_up_table)
def stop(self):
self.db = None
return defer.succeed(True)
def get_all_blob_infos(self):
d = threads.deferToThread(self._get_all_blob_infos)
d = self._get_all_blob_infos()
def make_blob_infos(blob_data):
blob_infos = []
@@ -42,21 +60,19 @@ class BlindInfoManager(object):
peer_port = blob_info.peer.port
peer_score = blob_info.peer_score
blobs.append((blob_hash, length, reference, peer_host, peer_port, peer_score))
return threads.deferToThread(self._save_blob_infos, blobs)
return self._save_blob_infos(blobs)
@rerun_if_locked
def _get_all_blob_infos(self):
blob_infos = []
for key, blob_info in self.db.RangeIter():
key_type, blob_hash = json.loads(key)
if key_type == BLOB_INFO_TYPE:
blob_infos.append([blob_hash] + json.loads(blob_info))
return blob_infos
return self.db_conn.runQuery("select * from valuable_blobs")
@rerun_if_locked
def _save_blob_infos(self, blobs):
batch = leveldb.WriteBatch()
for blob in blobs:
try:
self.db.Get(json.dumps((BLOB_INFO_TYPE, blob[0])))
except KeyError:
batch.Put(json.dumps((BLOB_INFO_TYPE, blob[0])), json.dumps(blob[1:]))
self.db.Write(batch, sync=True)
def save_infos(transaction):
for blob in blobs:
try:
transaction.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
blob)
except sqlite3.IntegrityError:
pass
return self.db_conn.runInteraction(save_infos)
@@ -2,7 +2,7 @@ from zope.interface import implements
from lbrynet.interfaces import IMetadataHandler, IRequestCreator
from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
from lbrynet.core.Error import NoResponseError
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
from ValuableBlobInfo import ValuableBlobInfo
import datetime
import logging
@@ -12,6 +12,9 @@ from twisted.python.failure import Failure
from collections import defaultdict
log = logging.getLogger(__name__)
class BlindMetadataHandler(object):
implements(IMetadataHandler, IRequestCreator)
@@ -195,7 +198,7 @@ class BlindMetadataHandler(object):
if not 'valuable_blob_hashes' in response:
return InvalidResponseError("Missing the required field 'valuable_blob_hashes'")
hashes = response['valuable_blob_hashes']
logging.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
log.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
expire_time = datetime.datetime.now() + datetime.timedelta(minutes=10)
reference = None
unique_hashes = set()
@@ -230,8 +233,8 @@ class BlindMetadataHandler(object):
if not 'blob_lengths' in response:
return InvalidResponseError("Missing the required field 'blob_lengths'")
raw_blob_lengths = response['blob_lengths']
logging.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
logging.debug("blobs: %s", str(raw_blob_lengths))
log.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
log.debug("blobs: %s", str(raw_blob_lengths))
infos = []
unique_hashes = set()
for blob_hash, length in raw_blob_lengths:
@@ -287,11 +290,12 @@ class BlindMetadataHandler(object):
return
if reason.check(NoResponseError):
self._incompatible_peers.append(peer)
return
logging.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
str(request_type), str(reason.getErrorMessage()))
log.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
str(request_type), str(reason.getErrorMessage()))
self._update_local_score(peer, -10.0)
peer.update_score(-5.0)
if reason.check(ConnectionClosedBeforeResponseError):
return
return reason
def _search_for_peers(self):
@@ -109,5 +109,5 @@ class BlindRepeater(object):
def _update_status(self, stopped=True):
self.stopped = stopped
def insufficient_funds(self):
def insufficient_funds(self, err):
return self.stop()
@@ -1,111 +1,121 @@
from lbrynet.lbrynet_console.ControlHandlers import ControlHandler, ControlHandlerFactory
from lbrynet.lbrynet_console.ControlHandlers import RecursiveControlHandler, ModifyPaymentRate
from lbrynet.lbrynet_console.ControlHandlers import CommandHandler, CommandHandlerFactory
from lbrynet.lbrynet_console.ControlHandlers import RecursiveCommandHandler, ModifyPaymentRate
from twisted.internet import defer
class StartRepeater(ControlHandler):
class StartRepeater(CommandHandler):
prompt_description = "Start the blind repeater"
def __init__(self, repeater, settings):
def __init__(self, console, repeater, settings):
CommandHandler.__init__(self, console)
self.repeater = repeater
self.settings = settings
def handle_line(self, line):
assert line is None, "Start repeater should not be passed any arguments"
def start(self):
#assert line is None, "Start repeater should not be passed any arguments"
d = self.settings.save_repeater_status(running=True)
d.addCallback(lambda _: self.repeater.start())
d.addCallback(lambda _: "Started the repeater")
return True, d
d.addCallback(lambda _: self.console.sendLine("Started the repeater"))
d.chainDeferred(self.finished_deferred)
class StartRepeaterFactory(ControlHandlerFactory):
class StartRepeaterFactory(CommandHandlerFactory):
control_handler_class = StartRepeater
class StopRepeater(ControlHandler):
class StopRepeater(CommandHandler):
prompt_description = "Stop the blind repeater"
def __init__(self, repeater, settings):
def __init__(self, console, repeater, settings):
CommandHandler.__init__(self, console)
self.repeater = repeater
self.settings = settings
def handle_line(self, line):
assert line is None, "Stop repeater should not be passed any arguments"
def start(self):
#assert line is None, "Stop repeater should not be passed any arguments"
d = self.settings.save_repeater_status(running=False)
d.addCallback(lambda _: self.repeater.stop())
d.addCallback(lambda _: "Stopped the repeater")
return True, d
d.addCallback(lambda _: self.console.sendLine("Stopped the repeater"))
d.chainDeferred(self.finished_deferred)
class StopRepeaterFactory(ControlHandlerFactory):
class StopRepeaterFactory(CommandHandlerFactory):
control_handler_class = StopRepeater
class UpdateMaxSpace(ControlHandler):
class UpdateMaxSpace(CommandHandler):
prompt_description = "Set the maximum space to be used by the blind repeater"
line_prompt = "Maximum space (in bytes):"
def __init__(self, repeater, settings):
def __init__(self, console, repeater, settings):
CommandHandler.__init__(self, console)
self.repeater = repeater
self.settings = settings
def start(self):
self.console.sendLine(self.line_prompt)
def handle_line(self, line):
if line is None:
return False, defer.succeed(self.line_prompt)
return True, self._set_max_space(line)
d = self._set_max_space(line)
d.chainDeferred(self.finished_deferred)
def _set_max_space(self, line):
max_space = int(line)
d = self.settings.save_max_space(max_space)
d.addCallback(lambda _: self.repeater.set_max_space(max_space))
d.addCallback(lambda _: "Set the maximum space to " + str(max_space) + " bytes")
d.addCallback(lambda _: self.console.sendLine("Set the maximum space to " + str(max_space) + " bytes"))
return d
class UpdateMaxSpaceFactory(ControlHandlerFactory):
class UpdateMaxSpaceFactory(CommandHandlerFactory):
control_handler_class = UpdateMaxSpace
class AddApprovedPeer(ControlHandler):
class AddApprovedPeer(CommandHandler):
prompt_description = "Add a peer to the approved list of peers to check for valuable blob hashes"
host_prompt = "Peer host in dotted quad (e.g. 127.0.0.1)"
port_prompt = "Peer port (e.g. 4444)"
def __init__(self, repeater, peer_manager, settings):
def __init__(self, console, repeater, peer_manager, settings):
CommandHandler.__init__(self, console)
self.repeater = repeater
self.peer_manager = peer_manager
self.settings = settings
self.host_to_add = None
def start(self):
self.console.sendLine(self.host_prompt)
def handle_line(self, line):
if line is None:
return False, defer.succeed(self.host_prompt)
elif self.host_to_add is None:
#if line is None:
# return False, defer.succeed(self.host_prompt)
if self.host_to_add is None:
self.host_to_add = line
return False, defer.succeed(self.port_prompt)
self.console.sendLine(self.port_prompt)
else:
self.host_to_add, host = None, self.host_to_add
return True, self._add_peer(host, line)
d = self._add_peer(host, line)
d.chainDeferred(self.finished_deferred)
def _add_peer(self, host, port):
peer = self.peer_manager.get_peer(host, int(port))
d = self.settings.save_approved_peer(host, int(port))
d.addCallback(lambda _: self.repeater.add_approved_peer(peer))
d.addCallback(lambda _: "Successfully added peer")
d.addCallback(lambda _: self.console.sendLine("Successfully added peer"))
return d
class AddApprovedPeerFactory(ControlHandlerFactory):
class AddApprovedPeerFactory(CommandHandlerFactory):
control_handler_class = AddApprovedPeer
class ApprovedPeerChooser(RecursiveControlHandler):
class ApprovedPeerChooser(RecursiveCommandHandler):
def __init__(self, repeater, factory_class, *args, **kwargs):
def __init__(self, console, repeater, factory_class, *args, **kwargs):
self.repeater = repeater
self.factory_class = factory_class
self.args = args
RecursiveControlHandler.__init__(self, **kwargs)
RecursiveCommandHandler.__init__(self, console, **kwargs)
def _get_control_handler_factories(self):
control_handler_factories = []
@@ -114,7 +124,7 @@ class ApprovedPeerChooser(RecursiveControlHandler):
return control_handler_factories
class ApprovedPeerChooserFactory(ControlHandlerFactory):
class ApprovedPeerChooserFactory(CommandHandlerFactory):
def get_prompt_description(self):
peer = self.args[0]
return str(peer)
@@ -123,30 +133,32 @@ class ApprovedPeerChooserFactory(ControlHandlerFactory):
class DeleteApprovedPeerChooser(ApprovedPeerChooser):
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
def __init__(self, repeater, settings):
ApprovedPeerChooser.__init__(self, repeater, DeleteApprovedPeerFactory, repeater, settings,
exit_after_one_done=True)
def __init__(self, console, repeater, settings):
ApprovedPeerChooser.__init__(self, console, repeater, DeleteApprovedPeerFactory, repeater,
settings, exit_after_one_done=True)
class DeleteApprovedPeerChooserFactory(ControlHandlerFactory):
class DeleteApprovedPeerChooserFactory(CommandHandlerFactory):
control_handler_class = DeleteApprovedPeerChooser
class DeleteApprovedPeer(ControlHandler):
class DeleteApprovedPeer(CommandHandler):
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
def __init__(self, peer, repeater, settings):
def __init__(self, console, peer, repeater, settings):
CommandHandler.__init__(self, console)
self.repeater = repeater
self.settings = settings
self.peer_to_remove = peer
def handle_line(self, line):
return True, self._remove_peer()
def start(self):
d = self._remove_peer()
d.chainDeferred(self.finished_deferred)
def _remove_peer(self):
d = self.settings.remove_approved_peer(self.peer_to_remove.host, int(self.peer_to_remove.port))
d.addCallback(lambda _: self.repeater.remove_approved_peer(self.peer_to_remove))
d.addCallback(lambda _: "Successfully removed peer")
d.addCallback(lambda _: self.console.sendLine("Successfully removed peer"))
return d
@@ -154,38 +166,43 @@ class DeleteApprovedPeerFactory(ApprovedPeerChooserFactory):
control_handler_class = DeleteApprovedPeer
class ShowApprovedPeers(ControlHandler):
class ShowApprovedPeers(CommandHandler):
prompt_description = "Show the list of peers approved to be checked for valuable blob hashes"
def __init__(self, repeater):
def __init__(self, console, repeater):
CommandHandler.__init__(self, console)
self.repeater = repeater
def handle_line(self, line):
assert line is None, "Show approved peers should not be passed any arguments"
return True, self._show_peers()
def start(self):
#assert line is None, "Show approved peers should not be passed any arguments"
d = self._show_peers()
d.chainDeferred(self.finished_deferred)
def _show_peers(self):
peer_string = "Approved peers:\n"
for peer in self.repeater.approved_peers:
peer_string += str(peer) + "\n"
return defer.succeed(peer_string)
self.console.sendLine(peer_string)
return defer.succeed(None)
class ShowApprovedPeersFactory(ControlHandlerFactory):
class ShowApprovedPeersFactory(CommandHandlerFactory):
control_handler_class = ShowApprovedPeers
class RepeaterStatus(ControlHandler):
class RepeaterStatus(CommandHandler):
prompt_description = "Show the repeater's status"
def __init__(self, repeater):
def __init__(self, console, repeater):
CommandHandler.__init__(self, console)
self.repeater = repeater
def handle_line(self, line):
assert line is None, "Show repeater status should not be passed any arguments"
return True, defer.maybeDeferred(self._get_status)
def start(self):
#assert line is None, "Show repeater status should not be passed any arguments"
self._show_status()
self.finished_deferred.callback(None)
def _get_status(self):
def _show_status(self):
status_string = "Repeater status: " + self.repeater.status() + "\n"
if self.repeater.stopped is False:
@@ -197,25 +214,25 @@ class RepeaterStatus(ControlHandler):
status_string += "Maximum space: " + str(max_space) + " bytes\n"
status_string += "Space used: " + str(space_used) + " bytes\n"
return defer.succeed(status_string)
self.console.sendLine(status_string)
class RepeaterStatusFactory(ControlHandlerFactory):
class RepeaterStatusFactory(CommandHandlerFactory):
control_handler_class = RepeaterStatus
class ModifyDataPaymentRate(ModifyPaymentRate):
prompt_description = "Modify Blind Repeater data payment rate"
def __init__(self, repeater, settings):
ModifyPaymentRate.__init__(self)
def __init__(self, console, repeater, settings):
ModifyPaymentRate.__init__(self, console)
self._prompt_choices['unset'] = (self._unset, "Use the application default data rate")
self.payment_rate_manager = repeater.payment_rate_manager
self.settings = settings
def _unset(self):
self._set_rate(None)
return True, defer.succeed("Using the application default data rate")
return defer.succeed("Using the application default data rate")
def _set_rate(self, rate):
@@ -237,15 +254,15 @@ class ModifyDataPaymentRate(ModifyPaymentRate):
return status
class ModifyDataPaymentRateFactory(ControlHandlerFactory):
class ModifyDataPaymentRateFactory(CommandHandlerFactory):
control_handler_class = ModifyDataPaymentRate
class ModifyInfoPaymentRate(ModifyPaymentRate):
prompt_description = "Modify Blind Repeater valuable info payment rate"
def __init__(self, repeater, settings):
ModifyPaymentRate.__init__(self)
def __init__(self, console, repeater, settings):
ModifyPaymentRate.__init__(self, console)
self.payment_rate_manager = repeater.payment_rate_manager
self.settings = settings
@@ -264,15 +281,15 @@ class ModifyInfoPaymentRate(ModifyPaymentRate):
return status
class ModifyInfoPaymentRateFactory(ControlHandlerFactory):
class ModifyInfoPaymentRateFactory(CommandHandlerFactory):
control_handler_class = ModifyInfoPaymentRate
class ModifyHashPaymentRate(ModifyPaymentRate):
prompt_description = "Modify Blind Repeater valuable hash payment rate"
def __init__(self, repeater, settings):
ModifyPaymentRate.__init__(self)
def __init__(self, console, repeater, settings):
ModifyPaymentRate.__init__(self, console)
self.payment_rate_manager = repeater.payment_rate_manager
self.settings = settings
@@ -291,18 +308,18 @@ class ModifyHashPaymentRate(ModifyPaymentRate):
return status
class ModifyHashPaymentRateFactory(ControlHandlerFactory):
class ModifyHashPaymentRateFactory(CommandHandlerFactory):
control_handler_class = ModifyHashPaymentRate
class ModifyRepeaterOptions(RecursiveControlHandler):
class ModifyRepeaterOptions(RecursiveCommandHandler):
prompt_description = "Modify Blind Repeater options"
def __init__(self, repeater, lbry_session, settings):
def __init__(self, console, repeater, lbry_session, settings):
self.repeater = repeater
self.lbry_session = lbry_session
self.settings = settings
RecursiveControlHandler.__init__(self)
RecursiveCommandHandler.__init__(self, console)
def _get_control_handler_factories(self):
return [ModifyDataPaymentRateFactory(self.repeater, self.settings),
@@ -314,5 +331,5 @@ class ModifyRepeaterOptions(RecursiveControlHandler):
]
class ModifyRepeaterOptionsFactory(ControlHandlerFactory):
class ModifyRepeaterOptionsFactory(CommandHandlerFactory):
control_handler_class = ModifyRepeaterOptions
@@ -1,67 +1,79 @@
from db_keys import SETTING_TYPE, PEER_TYPE
from twisted.internet import threads
from twisted.internet import threads, defer
import json
import unqlite
import os
from twisted.enterprise import adbapi
from lbrynet.core.sqlite_helpers import rerun_if_locked
class BlindRepeaterSettings(object):
def __init__(self, db):
self.db = db
def __init__(self, db_dir):
self.db_dir = db_dir
self.unq_db = None
self.sql_db = None
def setup(self):
self.unq_db = unqlite.UnQLite(os.path.join(self.db_dir, "blind_settings.db"))
# check_same_thread=False is solely to quiet a spurious error that appears to be due
# to a bug in twisted, where the connection is closed by a different thread than the
# one that opened it. The individual connections in the pool are not used in multiple
# threads.
self.sql_db = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_peers.db"),
check_same_thread=False)
return self.sql_db.runQuery("create table if not exists approved_peers (" +
" ip_address text, " +
" port integer" +
")")
def stop(self):
self.unq_db = None
self.sql_db = None
return defer.succeed(True)
def save_repeater_status(self, running):
def save_status():
self.db.Put(json.dumps((SETTING_TYPE, "running")), json.dumps(running), sync=True)
self.unq_db["running"] = json.dumps(running)
return threads.deferToThread(save_status)
def get_repeater_saved_status(self):
def get_status():
try:
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, "running"))))
except KeyError:
if "running" in self.unq_db:
return json.loads(self.unq_db['running'])
else:
return False
return threads.deferToThread(get_status)
def save_max_space(self, max_space):
def save_space():
self.db.Put(json.dumps((SETTING_TYPE, "max_space")), str(max_space), sync=True)
self.unq_db['max_space'] = json.dumps(max_space)
return threads.deferToThread(save_space)
def get_saved_max_space(self):
def get_space():
try:
return int(self.db.Get(json.dumps((SETTING_TYPE, "max_space"))))
except KeyError:
if 'max_space' in self.unq_db:
return json.loads(self.unq_db['max_space'])
else:
return 0
return threads.deferToThread(get_space)
@rerun_if_locked
def save_approved_peer(self, host, port):
def add_peer():
peer_string = json.dumps((PEER_TYPE, (host, port)))
self.db.Put(peer_string, "", sync=True)
return threads.deferToThread(add_peer)
return self.sql_db.runQuery("insert into approved_peers values (?, ?)",
(host, port))
@rerun_if_locked
def remove_approved_peer(self, host, port):
def remove_peer():
peer_string = json.dumps((PEER_TYPE, (host, port)))
self.db.Delete(peer_string, sync=True)
return threads.deferToThread(remove_peer)
return self.sql_db.runQuery("delete from approved_peers where ip_address = ? and port = ?",
(host, port))
@rerun_if_locked
def get_approved_peers(self):
def get_peers():
peers = []
for k, v in self.db.RangeIter():
key_type, peer_info = json.loads(k)
if key_type == PEER_TYPE:
peers.append(peer_info)
return peers
return threads.deferToThread(get_peers)
return self.sql_db.runQuery("select * from approved_peers")
def get_data_payment_rate(self):
return threads.deferToThread(self._get_rate, "data_payment_rate")
@@ -82,13 +94,13 @@ class BlindRepeaterSettings(object):
return threads.deferToThread(self._save_rate, "valuable_hash_rate", rate)
def _get_rate(self, rate_type):
try:
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, rate_type))))
except KeyError:
if rate_type in self.unq_db:
return json.loads(self.unq_db[rate_type])
else:
return None
def _save_rate(self, rate_type, rate):
if rate is not None:
self.db.Put(json.dumps((SETTING_TYPE, rate_type)), json.dumps(rate), sync=True)
else:
self.db.Delete(json.dumps((SETTING_TYPE, rate_type)), sync=True)
self.unq_db[rate_type] = json.dumps(rate)
elif rate_type in self.unq_db:
del self.unq_db[rate_type]
@@ -4,6 +4,9 @@ from twisted.internet import defer
import logging
log = logging.getLogger(__name__)
class ValuableQueryHandler(object):
implements(IQueryHandler)
@@ -98,8 +101,8 @@ class ValuableBlobHashQueryHandler(ValuableQueryHandler):
for blob_hash, count in valuable_hashes:
hashes_and_scores.append((blob_hash, 1.0 * count / 10.0))
if len(hashes_and_scores) != 0:
logging.info("Responding to a valuable blob hashes request with %s blob hashes: %s",
str(len(hashes_and_scores)))
log.info("Responding to a valuable blob hashes request with %s blob hashes",
str(len(hashes_and_scores)))
expected_payment = 1.0 * len(hashes_and_scores) * self.valuable_blob_hash_payment_rate / 1000.0
self.wallet.add_expected_payment(self.peer, expected_payment)
self.peer.update_stats('uploaded_valuable_blob_hashes', len(hashes_and_scores))
@@ -190,10 +193,10 @@ class ValuableBlobLengthQueryHandler(ValuableQueryHandler):
if success is True:
lengths.append(response_pair)
if len(lengths) > 0:
logging.info("Responding with %s blob lengths: %s", str(len(lengths)))
log.info("Responding with %s blob lengths", str(len(lengths)))
expected_payment = 1.0 * len(lengths) * self.blob_length_payment_rate / 1000.0
self.wallet.add_expected_payment(self.peer, expected_payment)
self.peer.update_stats('uploaded_valuable_blob_infos', len(lengths))
return {'blob_length': {'blob_lengths': lengths}}
dl.addCallback(make_response)
dl.addCallback(make_response)
@@ -1,7 +1,5 @@
import leveldb
import os
from lbrynet.lbrynet_console import LBRYPlugin
from twisted.internet import defer, threads
from twisted.internet import defer
from lbrynet.conf import MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE, MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE
from BlindRepeater import BlindRepeater
from BlindInfoManager import BlindInfoManager
@@ -19,6 +17,7 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
def __init__(self):
LBRYPlugin.LBRYPlugin.__init__(self)
self.enabled = False
self.blind_info_manager = None
self.valuable_blob_length_query_handler = None
self.valuable_blob_hash_query_handler = None
@@ -26,14 +25,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
self.control_handlers = None
self.payment_rate_manager = None
self.settings = None
self.db = None
def setup(self, lbry_console):
if not self.enabled:
return defer.succeed(True)
lbry_session = lbry_console.session
d = threads.deferToThread(self._setup_db, lbry_session.db_dir)
d.addCallback(lambda _: self._setup_settings())
d = self._setup_settings(lbry_session.db_dir)
d.addCallback(lambda _: self._get_payment_rate_manager(lbry_session.base_payment_rate_manager))
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager))
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager, lbry_session.db_dir))
d.addCallback(lambda _: self._setup_blind_repeater(lbry_session))
d.addCallback(lambda _: self._setup_valuable_blob_query_handler(lbry_session))
d.addCallback(lambda _: self._create_control_handlers(lbry_session))
@@ -41,11 +40,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
d.addCallback(lambda _: self._add_to_lbry_console(lbry_console))
return d
def _setup_db(self, db_dir):
self.db = leveldb.LevelDB(os.path.join(db_dir, "valuable_blobs.db"))
def stop(self):
if self.settings is not None:
return self.settings.stop()
return defer.succeed(True)
def _setup_settings(self):
self.settings = BlindRepeaterSettings(self.db)
def _setup_settings(self, db_dir):
self.settings = BlindRepeaterSettings(db_dir)
return self.settings.setup()
def _get_payment_rate_manager(self, default_payment_rate_manager):
d1 = self.settings.get_data_payment_rate()
@@ -67,8 +69,8 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
dl.addCallback(get_payment_rate_manager)
return dl
def _setup_blind_info_manager(self, peer_manager):
self.blind_info_manager = BlindInfoManager(self.db, peer_manager)
def _setup_blind_info_manager(self, peer_manager, db_dir):
self.blind_info_manager = BlindInfoManager(db_dir, peer_manager)
return self.blind_info_manager.setup()
def _setup_valuable_blob_query_handler(self, lbry_session):
@@ -125,4 +127,4 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
def _add_to_lbry_console(self, lbry_console):
lbry_console.add_control_handlers(self.control_handlers)
lbry_console.add_query_handlers([self.valuable_blob_length_query_handler,
self.valuable_blob_hash_query_handler])
self.valuable_blob_hash_query_handler])
@@ -1,3 +0,0 @@
BLOB_INFO_TYPE = 'b'
SETTING_TYPE = 's'
PEER_TYPE = 'p'
File diff suppressed because it is too large. Load diff
+59
View File
@@ -0,0 +1,59 @@
import sys
import json
from lbrynet.conf import API_CONNECTION_STRING
from jsonrpc.proxy import JSONRPCProxy
help_msg = "Usage: lbrynet-cli method json-args\n" \
+ "Examples: " \
+ "lbrynet-cli resolve_name '{\"name\": \"what\"}'\n" \
+ "lbrynet-cli get_balance\n" \
+ "lbrynet-cli help '{\"function\": \"resolve_name\"}'\n" \
+ "\n******lbrynet-cli functions******\n"
def main():
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
try:
s = api.is_running()
except:
print "lbrynet-daemon isn't running"
sys.exit(1)
args = sys.argv[1:]
meth = args[0]
msg = help_msg
for f in api.help():
msg += f + "\n"
if meth in ['--help', '-h', 'help']:
print msg
sys.exit(1)
if len(args) > 1:
if isinstance(args[1], dict):
params = args[1]
elif isinstance(args[1], basestring):
params = json.loads(args[1])
else:
params = None
if meth in api.help():
try:
if params:
r = api.call(meth, params)
else:
r = api.call(meth)
print r
except:
print "Something went wrong, here's the usage for %s:" % meth
print api.help({'function': meth})
else:
print "Unknown function"
print msg
if __name__ == '__main__':
main()
+127
View File
@@ -0,0 +1,127 @@
import argparse
import logging
import logging.handlers
import os
import webbrowser
import sys
import socket
from appdirs import user_data_dir
from twisted.web import server
from twisted.internet import reactor, defer
from jsonrpc.proxy import JSONRPCProxy
from lbrynet.core import log_support
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \
UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: stop it!
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)
REMOTE_SERVER = "www.google.com"
def test_internet_connection():
try:
host = socket.gethostbyname(REMOTE_SERVER)
s = socket.create_connection((host, 80), 2)
return True
except:
return False
def stop():
def _disp_shutdown():
print "Shutting down lbrynet-daemon from command line"
log.info("Shutting down lbrynet-daemon from command line")
def _disp_not_running():
print "Attempt to shut down lbrynet-daemon from command line when daemon isn't running"
log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
d = defer.Deferred(None)
d.addCallback(lambda _: JSONRPCProxy.from_url(API_CONNECTION_STRING).stop())
d.addCallbacks(lambda _: _disp_shutdown(), lambda _: _disp_not_running())
d.callback(None)
def start():
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
parser.add_argument("--wallet",
help="lbrycrd or lbryum, default lbryum",
type=str,
default='')
parser.add_argument("--ui",
help="path to custom UI folder",
default=None)
parser.add_argument("--branch",
help="Branch of lbry-web-ui repo to use, defaults on master")
parser.add_argument('--no-launch', dest='launchui', action="store_false")
parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
parser.add_argument('--quiet', dest='quiet', action="store_true")
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args()
log_support.configure_file_handler(lbrynet_log)
log_support.configure_loggly_handler()
if args.logtoconsole:
log_support.configure_console(level='DEBUG')
log_support.disable_third_party_loggers()
log_support.disable_noisy_loggers()
try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
log.info("lbrynet-daemon is already running")
if not args.logtoconsole:
print "lbrynet-daemon is already running"
if args.launchui:
webbrowser.open(UI_ADDRESS)
return
except:
pass
log.info("Starting lbrynet-daemon from command line")
if not args.logtoconsole and not args.quiet:
print "Starting lbrynet-daemon from command line"
print "To view activity, view the log file here: " + lbrynet_log
print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT)
print "JSONRPC API is available at " + API_CONNECTION_STRING
print "To quit press ctrl-c or call 'stop' via the API"
if test_internet_connection():
lbry = LBRYDaemonServer()
d = lbry.start(branch=args.branch if args.branch else DEFAULT_UI_BRANCH,
user_specified=args.ui,
wallet=args.wallet,
branch_specified=True if args.branch else False)
if args.launchui:
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
lbrynet_server = server.Site(lbry.root)
lbrynet_server.requestFactory = LBRYDaemonRequest
reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
reactor.run()
if not args.logtoconsole and not args.quiet:
print "\nClosing lbrynet-daemon"
else:
log.info("Not connected to internet, unable to start")
if not args.logtoconsole:
print "Not connected to internet, unable to start"
return
if __name__ == "__main__":
start()
+398
View File
@@ -0,0 +1,398 @@
import logging
import os
import shutil
import json
import sys
import mimetypes
import mimetools
import tempfile
import time
import cgi
from appdirs import user_data_dir
from twisted.web import server, static, resource
from twisted.internet import defer, interfaces, error, reactor, threads
from zope.interface import implements
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
from lbrynet.conf import API_ADDRESS, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: omg, this code is essentially duplicated in LBRYDaemon
if sys.platform != "darwin":
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
data_dir = user_data_dir("LBRY")
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)
class LBRYDaemonRequest(server.Request):
"""
For LBRY specific request functionality. Currently just provides
handling for large multipart POST requests, taken from here:
http://sammitch.ca/2013/07/handling-large-requests-in-twisted/
For multipart POST requests, this populates self.args with temp
file objects instead of strings. Note that these files don't auto-delete
on close because we want to be able to move and rename them.
"""
# max amount of memory to allow any ~single~ request argument [ie: POSTed file]
# note: this value seems to be taken with a grain of salt, memory usage may spike
# FAR above this value in some cases.
# eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will
# have spiked to 8MB before the data is rolled to disk after the
# second write completes.
memorylimit = 1024*1024*100
# enable/disable debug logging
do_log = False
# re-defined only for debug/logging purposes
def gotLength(self, length):
if self.do_log:
print '%f Headers received, Content-Length: %d' % (time.time(), length)
server.Request.gotLength(self, length)
# re-definition of twisted.web.server.Request.requestreceived, the only difference
# is that self.parse_multipart() is used rather than cgi.parse_multipart()
def requestReceived(self, command, path, version):
from twisted.web.http import parse_qs
if self.do_log:
print '%f Request Received' % time.time()
self.content.seek(0,0)
self.args = {}
self.stack = []
self.method, self.uri = command, path
self.clientproto = version
x = self.uri.split(b'?', 1)
if len(x) == 1:
self.path = self.uri
else:
self.path, argstring = x
self.args = parse_qs(argstring, 1)
# cache the client and server information, we'll need this later to be
# serialized and sent with the request so CGIs will work remotely
self.client = self.channel.transport.getPeer()
self.host = self.channel.transport.getHost()
# Argument processing
args = self.args
ctype = self.requestHeaders.getRawHeaders(b'content-type')
if ctype is not None:
ctype = ctype[0]
if self.method == b"POST" and ctype:
mfd = b'multipart/form-data'
key, pdict = cgi.parse_header(ctype)
if key == b'application/x-www-form-urlencoded':
args.update(parse_qs(self.content.read(), 1))
elif key == mfd:
try:
self.content.seek(0,0)
args.update(self.parse_multipart(self.content, pdict))
#args.update(cgi.parse_multipart(self.content, pdict))
except KeyError as e:
if e.args[0] == b'content-disposition':
# Parse_multipart can't cope with missing
# content-dispostion headers in multipart/form-data
# parts, so we catch the exception and tell the client
# it was a bad request.
self.channel.transport.write(
b"HTTP/1.1 400 Bad Request\r\n\r\n")
self.channel.transport.loseConnection()
return
raise
self.content.seek(0, 0)
self.process()
# re-definition of cgi.parse_multipart that uses a single temporary file to store
# data rather than storing 2 to 3 copies in various lists.
def parse_multipart(self, fp, pdict):
if self.do_log:
print '%f Parsing Multipart data: ' % time.time()
rewind = fp.tell() #save cursor
fp.seek(0,0) #reset cursor
boundary = ""
if 'boundary' in pdict:
boundary = pdict['boundary']
if not cgi.valid_boundary(boundary):
raise ValueError, ('Invalid boundary in multipart form: %r'
% (boundary,))
nextpart = "--" + boundary
lastpart = "--" + boundary + "--"
partdict = {}
terminator = ""
while terminator != lastpart:
c_bytes = -1
data = tempfile.NamedTemporaryFile(delete=False)
if terminator:
# At start of next part. Read headers first.
headers = mimetools.Message(fp)
clength = headers.getheader('content-length')
if clength:
try:
c_bytes = int(clength)
except ValueError:
pass
if c_bytes > 0:
data.write(fp.read(c_bytes))
# Read lines until end of part.
while 1:
line = fp.readline()
if not line:
terminator = lastpart # End outer loop
break
if line[:2] == "--":
terminator = line.strip()
if terminator in (nextpart, lastpart):
break
data.write(line)
# Done with part.
if data.tell() == 0:
continue
if c_bytes < 0:
# if a Content-Length header was not supplied with the MIME part
# then the trailing line break must be removed.
# we have data, read the last 2 bytes
rewind = min(2, data.tell())
data.seek(-rewind, os.SEEK_END)
line = data.read(2)
if line[-2:] == "\r\n":
data.seek(-2, os.SEEK_END)
data.truncate()
elif line[-1:] == "\n":
data.seek(-1, os.SEEK_END)
data.truncate()
line = headers['content-disposition']
if not line:
continue
key, params = cgi.parse_header(line)
if key != 'form-data':
continue
if 'name' in params:
name = params['name']
# kludge in the filename
if 'filename' in params:
fname_index = name + '_filename'
if fname_index in partdict:
partdict[fname_index].append(params['filename'])
else:
partdict[fname_index] = [params['filename']]
else:
# Unnamed parts are not returned at all.
continue
data.seek(0,0)
if name in partdict:
partdict[name].append(data)
else:
partdict[name] = [data]
fp.seek(rewind) # Restore cursor
return partdict
class LBRYindex(resource.Resource):
def __init__(self, ui_dir):
resource.Resource.__init__(self)
self.ui_dir = ui_dir
isLeaf = False
def _delayed_render(self, request, results):
request.write(str(results))
request.finish()
def getChild(self, name, request):
if name == '':
return self
return resource.Resource.getChild(self, name, request)
def render_GET(self, request):
request.setHeader('cache-control','no-cache, no-store, must-revalidate')
request.setHeader('expires', '0')
return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
class LBRYFileStreamer(object):
"""
Writes downloaded LBRY file to request as the download comes in, pausing and resuming as requested
used for Chrome
"""
implements(interfaces.IPushProducer)
def __init__(self, request, path, start, stop, size):
self._request = request
self._fileObject = file(path)
self._content_type = mimetypes.guess_type(path)[0]
self._stop_pos = size - 1 if stop == '' else int(stop) #chrome and firefox send range requests for "0-"
self._cursor = self._start_pos = int(start)
self._file_size = size
self._depth = 0
self._paused = self._sent_bytes = self._stopped = False
self._delay = 0.25
self._deferred = defer.succeed(None)
self._request.setResponseCode(206)
self._request.setHeader('accept-ranges', 'bytes')
self._request.setHeader('content-type', self._content_type)
self.resumeProducing()
def pauseProducing(self):
self._paused = True
log.info("Pausing producer")
return defer.succeed(None)
def resumeProducing(self):
def _check_for_new_data():
self._depth += 1
self._fileObject.seek(self._start_pos, os.SEEK_END)
readable_bytes = self._fileObject.tell()
self._fileObject.seek(self._cursor)
self._sent_bytes = False
if (readable_bytes > self._cursor) and not (self._stopped or self._paused):
read_length = min(readable_bytes, self._stop_pos) - self._cursor + 1
self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length - 1, self._file_size))
self._request.setHeader('content-length', str(read_length))
start_cur = self._cursor
for i in range(read_length):
if self._paused or self._stopped:
break
else:
data = self._fileObject.read(1)
self._request.write(data)
self._cursor += 1
log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" %
(start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth))
self._sent_bytes = True
if self._cursor == self._stop_pos + 1:
self.stopProducing()
return defer.succeed(None)
elif self._paused or self._stopped:
return defer.succeed(None)
else:
self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data))
return defer.succeed(None)
log.info("Resuming producer")
self._paused = False
self._deferred.addCallback(lambda _: _check_for_new_data())
def stopProducing(self):
log.info("Stopping producer")
self._stopped = True
# self._fileObject.close()
self._deferred.addErrback(lambda err: err.trap(defer.CancelledError))
self._deferred.addErrback(lambda err: err.trap(error.ConnectionDone))
self._deferred.cancel()
# self._request.finish()
self._request.unregisterProducer()
return defer.succeed(None)
class HostedLBRYFile(resource.Resource):
def __init__(self, api):
self._api = api
self._producer = None
resource.Resource.__init__(self)
# todo: fix LBRYFileStreamer and use it instead of static.File
# def makeProducer(self, request, stream):
# def _save_producer(producer):
# self._producer = producer
# return defer.succeed(None)
#
# range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
# start, stop = int(range_header[0]), range_header[1]
# log.info("GET range %s-%s" % (start, stop))
# path = os.path.join(self._api.download_directory, stream.file_name)
#
# d = stream.get_total_bytes()
# d.addCallback(lambda size: _save_producer(LBRYFileStreamer(request, path, start, stop, size)))
# d.addCallback(lambda _: request.registerProducer(self._producer, streaming=True))
# # request.notifyFinish().addCallback(lambda _: self._producer.stopProducing())
# request.notifyFinish().addErrback(self._responseFailed, d)
# return d
def render_GET(self, request):
if 'name' in request.args.keys():
if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
d = self._api._download_name(request.args['name'][0])
# d.addCallback(lambda stream: self.makeProducer(request, stream))
d.addCallback(lambda stream: static.File(os.path.join(self._api.download_directory,
stream.file_name)).render_GET(request))
elif request.args['name'][0] in self._api.waiting_on.keys():
request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
request.finish()
else:
request.redirect(UI_ADDRESS)
request.finish()
return server.NOT_DONE_YET
# def _responseFailed(self, err, call):
# call.addErrback(lambda err: err.trap(error.ConnectionDone))
# call.addErrback(lambda err: err.trap(defer.CancelledError))
# call.addErrback(lambda err: log.info("Error: " + str(err)))
# call.cancel()
class LBRYFileUpload(resource.Resource):
"""
Accepts a file sent via the file upload widget in the web UI, saves
it into a temporary dir, and responds with a JSON string containing
the path of the newly created file.
"""
def __init__(self, api):
self._api = api
def render_POST(self, request):
origfilename = request.args['file_filename'][0]
uploaded_file = request.args['file'][0] # Temp file created by request
# Move to a new temporary dir and restore the original file name
newdirpath = tempfile.mkdtemp()
newpath = os.path.join(newdirpath, origfilename)
shutil.move(uploaded_file.name, newpath)
self._api.uploaded_temp_files.append(newpath)
return json.dumps(newpath)
class LBRYDaemonServer(object):
def _setup_server(self, wallet):
self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active"))
self._api = LBRYDaemon(self.root, wallet_type=wallet)
self.root.putChild("view", HostedLBRYFile(self._api))
self.root.putChild("upload", LBRYFileUpload(self._api))
self.root.putChild(API_ADDRESS, self._api)
return defer.succeed(True)
def start(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, wallet=None):
d = self._setup_server(wallet)
d.addCallback(lambda _: self._api.setup(branch, user_specified, branch_specified))
return d
+167
View File
@@ -0,0 +1,167 @@
import json
import logging
import os
import sys
from copy import deepcopy
from appdirs import user_data_dir
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.core.LBRYMetadata import LBRYFeeValidator
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
DOWNLOAD_TIMEOUT_CODE = 'timeout'
DOWNLOAD_RUNNING_CODE = 'running'
DOWNLOAD_STOPPED_CODE = 'stopped'
STREAM_STAGES = [
(INITIALIZING_CODE, 'Initializing...'),
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
]
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)
class GetStream(object):
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager,
max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
self.wallet = wallet
self.resolved_name = None
self.description = None
self.fee = None
self.data_rate = data_rate
self.name = None
self.file_name = file_name
self.session = session
self.exchange_rate_manager = exchange_rate_manager
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
self.lbry_file_manager = lbry_file_manager
self.sd_identifier = sd_identifier
self.stream_hash = None
self.max_key_fee = max_key_fee
self.stream_info = None
self.stream_info_manager = None
self.d = defer.Deferred(None)
self.timeout = timeout
self.timeout_counter = 0
self.download_directory = download_directory
self.download_path = None
self.downloader = None
self.finished = defer.Deferred(None)
self.checker = LoopingCall(self.check_status)
self.code = STREAM_STAGES[0]
def check_status(self):
self.timeout_counter += 1
# TODO: Why is this the stopping condition for the finished callback?
if self.download_path:
self.checker.stop()
self.finished.callback((self.stream_hash, self.download_path))
elif self.timeout_counter >= self.timeout:
log.info("Timeout downloading lbry://%s" % self.resolved_name)
self.checker.stop()
self.d.cancel()
self.code = STREAM_STAGES[4]
self.finished.callback(False)
def _convert_max_fee(self):
if isinstance(self.max_key_fee, dict):
max_fee = LBRYFeeValidator(self.max_key_fee)
if max_fee.currency_symbol == "LBC":
return max_fee.amount
return self.exchange_rate_manager.to_lbc(self.fee).amount
elif isinstance(self.max_key_fee, float):
return float(self.max_key_fee)
def start(self, stream_info, name):
def _cause_timeout(err):
log.error(err)
log.debug('Forcing a timeout')
self.timeout_counter = self.timeout * 2
def _set_status(x, status):
log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
return x
def get_downloader_factory(metadata):
for factory in metadata.factories:
if isinstance(factory, ManagedLBRYFileDownloaderFactory):
return factory, metadata
raise Exception('No suitable factory was found in {}'.format(metadata.factories))
def make_downloader(args):
factory, metadata = args
return factory.make_downloader(metadata,
[self.data_rate, True],
self.payment_rate_manager,
download_directory=self.download_directory,
file_name=self.file_name)
self.resolved_name = name
self.stream_info = deepcopy(stream_info)
self.description = self.stream_info['description']
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
if 'fee' in self.stream_info:
self.fee = LBRYFeeValidator(self.stream_info['fee'])
max_key_fee = self._convert_max_fee()
if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee:
log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount,
self.max_key_fee,
self.resolved_name))
return defer.fail(KeyFeeAboveMaxAllowed())
log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee),
max_key_fee,
self.resolved_name))
self.checker.start(1)
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
self.d.addCallback(get_downloader_factory)
self.d.addCallback(make_downloader)
self.d.addCallbacks(self._start_download, _cause_timeout)
self.d.callback(None)
return self.finished
def _start_download(self, downloader):
log.info('Starting download for %s', self.name)
self.downloader = downloader
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
d = self._pay_key_fee()
d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name))
d.addCallback(lambda _: self.downloader.start())
def _pay_key_fee(self):
if self.fee is not None:
fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount
reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc)
if reserved_points is None:
return defer.fail(InsufficientFundsError())
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
return defer.succeed(None)
@@ -0,0 +1,215 @@
import time
import requests
import logging
import json
import googlefinance
from twisted.internet import defer, reactor
from twisted.internet.task import LoopingCall
from lbrynet.core.LBRYMetadata import LBRYFeeValidator
log = logging.getLogger(__name__)
CURRENCY_PAIRS = ["USDBTC", "BTCLBC"]
BITTREX_FEE = 0.0025
COINBASE_FEE = 0.0 #add fee
class ExchangeRate(object):
def __init__(self, market, spot, ts):
assert int(time.time()) - ts < 600
self.currency_pair = (market[0:3], market[3:6])
self.spot = spot
self.ts = ts
def as_dict(self):
return {'spot': self.spot, 'ts': self.ts}
class MarketFeed(object):
def __init__(self, market, name, url, params, fee):
self.market = market
self.name = name
self.url = url
self.params = params
self.fee = fee
self.rate = None
self._updater = LoopingCall(self._update_price)
def _make_request(self):
r = requests.get(self.url, self.params)
return r.text
def _handle_response(self, response):
return NotImplementedError
def _subtract_fee(self, from_amount):
return defer.succeed(from_amount / (1.0 - self.fee))
def _save_price(self, price):
log.info("Saving price update %f for %s" % (price, self.market))
self.rate = ExchangeRate(self.market, price, int(time.time()))
def _update_price(self):
d = defer.succeed(self._make_request())
d.addCallback(self._handle_response)
d.addCallback(self._subtract_fee)
d.addCallback(self._save_price)
def start(self):
if not self._updater.running:
self._updater.start(300)
def stop(self):
if self._updater.running:
self._updater.stop()
class BittrexFeed(MarketFeed):
def __init__(self):
MarketFeed.__init__(
self,
"BTCLBC",
"Bittrex",
"https://bittrex.com/api/v1.1/public/getmarkethistory",
{'market': 'BTC-LBC', 'count': 50},
BITTREX_FEE
)
def _handle_response(self, response):
trades = json.loads(response)['result']
vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades])
return defer.succeed(float(1.0 / vwap))
class GoogleBTCFeed(MarketFeed):
def __init__(self):
MarketFeed.__init__(
self,
"USDBTC",
"Coinbase via Google finance",
None,
None,
COINBASE_FEE
)
def _make_request(self):
return googlefinance.getQuotes('CURRENCY:USDBTC')[0]
def _handle_response(self, response):
return float(response['LastTradePrice'])
def get_default_market_feed(currency_pair):
currencies = None
if isinstance(currency_pair, str):
currencies = (currency_pair[0:3], currency_pair[3:6])
elif isinstance(currency_pair, tuple):
currencies = currency_pair
assert currencies is not None
if currencies == ("USD", "BTC"):
return GoogleBTCFeed()
elif currencies == ("BTC", "LBC"):
return BittrexFeed()
class ExchangeRateManager(object):
def __init__(self):
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS]
def start(self):
log.info("Starting exchange rate manager")
for feed in self.market_feeds:
feed.start()
def stop(self):
log.info("Stopping exchange rate manager")
for source in self.market_feeds:
source.stop()
def convert_currency(self, from_currency, to_currency, amount):
log.info("Converting %f %s to %s" % (amount, from_currency, to_currency))
if from_currency == to_currency:
return amount
for market in self.market_feeds:
if market.rate.currency_pair == (from_currency, to_currency):
return amount * market.rate.spot
for market in self.market_feeds:
if market.rate.currency_pair[0] == from_currency:
return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
raise Exception('Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
def fee_dict(self):
return {market: market.rate.as_dict() for market in self.market_feeds}
def to_lbc(self, fee):
if fee is None:
return None
if not isinstance(fee, LBRYFeeValidator):
fee_in = LBRYFeeValidator(fee)
else:
fee_in = fee
return LBRYFeeValidator({fee_in.currency_symbol:
{
'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
'address': fee_in.address
}
})
class DummyBTCLBCFeed(MarketFeed):
def __init__(self):
MarketFeed.__init__(
self,
"BTCLBC",
"market name",
"derp.com",
None,
0.0
)
class DummyUSDBTCFeed(MarketFeed):
def __init__(self):
MarketFeed.__init__(
self,
"USDBTC",
"market name",
"derp.com",
None,
0.0
)
class DummyExchangeRateManager(object):
def __init__(self, rates):
self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()]
for feed in self.market_feeds:
feed.rate = ExchangeRate(feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
def convert_currency(self, from_currency, to_currency, amount):
log.info("Converting %f %s to %s" % (amount, from_currency, to_currency))
for market in self.market_feeds:
if market.rate.currency_pair == (from_currency, to_currency):
return amount * market.rate.spot
for market in self.market_feeds:
if market.rate.currency_pair[0] == from_currency:
return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
def to_lbc(self, fee):
if fee is None:
return None
if not isinstance(fee, LBRYFeeValidator):
fee_in = LBRYFeeValidator(fee)
else:
fee_in = fee
return LBRYFeeValidator({fee_in.currency_symbol:
{
'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
'address': fee_in.address
}
})
+159
View File
@@ -0,0 +1,159 @@
import logging
import mimetypes
import os
import sys
import random
from appdirs import user_data_dir
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
from lbrynet import reflector
from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS
from twisted.internet import threads, defer, reactor
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)
class Publisher(object):
def __init__(self, session, lbry_file_manager, wallet):
self.session = session
self.lbry_file_manager = lbry_file_manager
self.wallet = wallet
self.received_file_name = False
self.file_path = None
self.file_name = None
self.publish_name = None
self.bid_amount = None
self.verified = False
self.lbry_file = None
self.txid = None
self.stream_hash = None
reflector_server = random.choice(REFLECTOR_SERVERS)
self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1]
self.metadata = {}
def start(self, name, file_path, bid, metadata):
log.info('Starting publish for %s', name)
def _show_result():
log.info("Success! Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
return defer.succeed(self.txid)
self.publish_name = name
self.file_path = file_path
self.bid_amount = bid
self.metadata = metadata
d = self._check_file_path(self.file_path)
d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
self.file_name, open(self.file_path)))
d.addCallback(self.add_to_lbry_files)
d.addCallback(lambda _: self._create_sd_blob())
d.addCallback(lambda _: self._claim_name())
d.addCallback(lambda _: self.set_status())
d.addCallback(lambda _: self.start_reflector())
d.addCallbacks(lambda _: _show_result(), self._show_publish_error)
return d
def start_reflector(self):
reflector_server = random.choice(REFLECTOR_SERVERS)
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
log.info("Reflecting new publication")
factory = reflector.ClientFactory(
self.session.blob_manager,
self.lbry_file_manager.stream_info_manager,
self.stream_hash
)
d = reactor.resolve(reflector_address)
d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory))
d.addCallback(lambda _: factory.finished_deferred)
return d
def _check_file_path(self, file_path):
def check_file_threaded():
f = open(file_path)
f.close()
self.file_name = os.path.basename(self.file_path)
return True
return threads.deferToThread(check_file_threaded)
def set_lbry_file(self, lbry_file_downloader):
self.lbry_file = lbry_file_downloader
return defer.succeed(None)
def add_to_lbry_files(self, stream_hash):
self.stream_hash = stream_hash
prm = PaymentRateManager(self.session.base_payment_rate_manager)
d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
d.addCallback(self.set_lbry_file)
return d
def _create_sd_blob(self):
log.debug('Creating stream descriptor blob')
d = publish_sd_blob(self.lbry_file_manager.stream_info_manager,
self.session.blob_manager,
self.lbry_file.stream_hash)
def set_sd_hash(sd_hash):
log.debug('stream descriptor hash: %s', sd_hash)
if 'sources' not in self.metadata:
self.metadata['sources'] = {}
self.metadata['sources']['lbry_sd_hash'] = sd_hash
d.addCallback(set_sd_hash)
return d
def set_status(self):
log.debug('Setting status')
d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
d.addCallback(lambda _: self.lbry_file.restore())
return d
def _claim_name(self):
log.debug('Claiming name')
self._update_metadata()
m = Metadata(self.metadata)
def set_tx_hash(txid):
log.debug('Name claimed using txid: %s', txid)
self.txid = txid
d = self.wallet.claim_name(self.publish_name, self.bid_amount, m)
d.addCallback(set_tx_hash)
return d
def _update_metadata(self):
filename = os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name)
self.metadata['content-type'] = get_content_type(filename)
self.metadata['ver'] = CURRENT_METADATA_VERSION
def _show_publish_error(self, err):
log.info(err.getTraceback())
message = "An error occurred publishing %s to %s. Error: %s."
if err.check(InsufficientFundsError):
error_message = "Insufficient funds"
else:
error_message = err.getErrorMessage()
log.error(error_message)
log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())
return defer.fail(Exception("Publish failed"))
def get_content_type(filename):
return mimetypes.guess_type(filename)[0]
+239
View File
@@ -0,0 +1,239 @@
import os
import logging
import shutil
import sys
import json
from urllib2 import urlopen
from StringIO import StringIO
from twisted.web import static
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.conf import DEFAULT_UI_BRANCH, LOG_FILE_NAME
from lbrynet import __version__ as lbrynet_version
from lbryum.version import LBRYUM_VERSION as lbryum_version
from zipfile import ZipFile
from appdirs import user_data_dir
if sys.platform != "darwin":
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
log_dir = user_data_dir("LBRY")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
log = logging.getLogger(__name__)
class LBRYUIManager(object):
def __init__(self, root):
if sys.platform != "darwin":
self.data_dir = os.path.join(os.path.expanduser("~"), '.lbrynet')
else:
self.data_dir = user_data_dir("LBRY")
self.ui_root = os.path.join(self.data_dir, "lbry-ui")
self.active_dir = os.path.join(self.ui_root, "active")
self.update_dir = os.path.join(self.ui_root, "update")
if not os.path.isdir(self.data_dir):
os.mkdir(self.data_dir)
if not os.path.isdir(self.ui_root):
os.mkdir(self.ui_root)
if not os.path.isdir(self.active_dir):
os.mkdir(self.active_dir)
if not os.path.isdir(self.update_dir):
os.mkdir(self.update_dir)
self.config = os.path.join(self.ui_root, "active.json")
self.update_requires = os.path.join(self.update_dir, "requirements.txt")
self.requirements = {}
self.check_requirements = True
self.ui_dir = self.active_dir
self.git_version = None
self.root = root
self.branch = None
self.update_checker = LoopingCall(self.setup)
if not os.path.isfile(os.path.join(self.config)):
self.loaded_git_version = None
self.loaded_branch = None
self.loaded_requirements = None
else:
try:
f = open(self.config, "r")
loaded_ui = json.loads(f.read())
f.close()
self.loaded_git_version = loaded_ui['commit']
self.loaded_branch = loaded_ui['branch']
self.loaded_requirements = loaded_ui['requirements']
except:
self.loaded_git_version = None
self.loaded_branch = None
self.loaded_requirements = None
def setup(self, branch=DEFAULT_UI_BRANCH, user_specified=None, branch_specified=False, check_requirements=None):
if check_requirements is not None:
self.check_requirements = check_requirements
if self.branch is not None:
self.branch = branch
if user_specified:
if os.path.isdir(user_specified):
log.info("Checking user specified UI directory: " + str(user_specified))
self.branch = "user-specified"
self.loaded_git_version = "user-specified"
d = self.migrate_ui(source=user_specified)
d.addCallback(lambda _: self._load_ui())
return d
else:
log.info("User specified UI directory doesn't exist, using " + self.branch)
elif self.loaded_branch == "user-specified" and not branch_specified:
log.info("Loading user provided UI")
d = self._load_ui()
return d
else:
log.info("Checking for updates for UI branch: " + branch)
self._git_url = "https://s3.amazonaws.com/lbry-ui/{}/data.json".format(branch)
self._dist_url = "https://s3.amazonaws.com/lbry-ui/{}/dist.zip".format(branch)
d = self._up_to_date()
d.addCallback(lambda r: self._download_ui() if not r else self._load_ui())
return d
def _up_to_date(self):
def _get_git_info():
response = urlopen(self._git_url)
data = json.loads(response.read())
return defer.succeed(data['sha'])
def _set_git(version):
self.git_version = version.replace('\n', '')
if self.git_version == self.loaded_git_version:
log.info("UI is up to date")
return defer.succeed(True)
else:
log.info("UI updates available, checking if installation meets requirements")
return defer.succeed(False)
def _use_existing():
log.info("Failed to check for new ui version, trying to use cached ui")
return defer.succeed(True)
d = _get_git_info()
d.addCallbacks(_set_git, lambda _: _use_existing)
return d
def migrate_ui(self, source=None):
if not source:
requires_file = self.update_requires
source_dir = self.update_dir
delete_source = True
else:
requires_file = os.path.join(source, "requirements.txt")
source_dir = source
delete_source = False
def _skip_requirements():
log.info("Skipping ui requirement check")
return defer.succeed(True)
def _check_requirements():
if not os.path.isfile(requires_file):
log.info("No requirements.txt file, rejecting request to migrate this UI")
return defer.succeed(False)
f = open(requires_file, "r")
for requirement in [line for line in f.read().split('\n') if line]:
t = requirement.split('=')
if len(t) == 3:
self.requirements[t[0]] = {'version': t[1], 'operator': '=='}
elif t[0][-1] == ">":
self.requirements[t[0][:-1]] = {'version': t[1], 'operator': '>='}
elif t[0][-1] == "<":
self.requirements[t[0][:-1]] = {'version': t[1], 'operator': '<='}
f.close()
passed_requirements = True
for r in self.requirements:
if r == 'lbrynet':
c = lbrynet_version
elif r == 'lbryum':
c = lbryum_version
else:
c = None
if c:
if self.requirements[r]['operator'] == '==':
if not self.requirements[r]['version'] == c:
passed_requirements = False
log.info("Local version %s of %s does not meet UI requirement for version %s" % (
c, r, self.requirements[r]['version']))
else:
log.info("Local version of %s meets ui requirement" % r)
if self.requirements[r]['operator'] == '>=':
if not self.requirements[r]['version'] <= c:
passed_requirements = False
log.info("Local version %s of %s does not meet UI requirement for version %s" % (
c, r, self.requirements[r]['version']))
else:
log.info("Local version of %s meets ui requirement" % r)
if self.requirements[r]['operator'] == '<=':
if not self.requirements[r]['version'] >= c:
passed_requirements = False
log.info("Local version %s of %s does not meet UI requirement for version %s" % (
c, r, self.requirements[r]['version']))
else:
log.info("Local version of %s meets ui requirement" % r)
return defer.succeed(passed_requirements)
def _disp_failure():
log.info("Failed to satisfy requirements for branch '%s', update was not loaded" % self.branch)
return defer.succeed(False)
def _do_migrate():
if os.path.isdir(self.active_dir):
shutil.rmtree(self.active_dir)
shutil.copytree(source_dir, self.active_dir)
if delete_source:
shutil.rmtree(source_dir)
log.info("Loaded UI update")
f = open(self.config, "w")
loaded_ui = {'commit': self.git_version, 'branch': self.branch, 'requirements': self.requirements}
f.write(json.dumps(loaded_ui))
f.close()
self.loaded_git_version = loaded_ui['commit']
self.loaded_branch = loaded_ui['branch']
self.loaded_requirements = loaded_ui['requirements']
return defer.succeed(True)
d = _check_requirements() if self.check_requirements else _skip_requirements()
d.addCallback(lambda r: _do_migrate() if r else _disp_failure())
return d
def _download_ui(self):
def _delete_update_dir():
if os.path.isdir(self.update_dir):
shutil.rmtree(self.update_dir)
return defer.succeed(None)
def _dl_ui():
url = urlopen(self._dist_url)
z = ZipFile(StringIO(url.read()))
names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
z.extractall(self.update_dir, members=names)
log.info("Downloaded files for UI commit " + str(self.git_version).replace("\n", ""))
return self.branch
d = _delete_update_dir()
d.addCallback(lambda _: _dl_ui())
d.addCallback(lambda _: self.migrate_ui())
d.addCallback(lambda _: self._load_ui())
return d
def _load_ui(self):
for d in [i[0] for i in os.walk(self.active_dir) if os.path.dirname(i[0]) == self.active_dir]:
self.root.putChild(os.path.basename(d), static.File(d))
return defer.succeed(True)
Loaded 100 of 169 files, more files were not shown because too many files have changed in this diff. Show more